mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-09-01 05:03:10 +00:00
[Backend][UsageResource] Add PUT to enable updating Usage #6117 :::::: and allow adding space to tagCAtegory and Tags (#6119)
* [Backend][UsageResource] Add PUT to enable updating Usage #6117 * [Backend][UsageResource] Allow space in tags category and primary tags #6121 * [Backend][UsageResource] Update in more genric way to make use of space in resourcePath * ui changes for allowing space in tag and tag category name * [Backend][Improvement]add test for put and post Co-authored-by: Shailesh Parmar <shailesh.parmar.webdev@gmail.com>
This commit is contained in:
parent
da559150c1
commit
e7dca141ed
@ -13,7 +13,11 @@
|
|||||||
|
|
||||||
package org.openmetadata.catalog.jdbi3;
|
package org.openmetadata.catalog.jdbi3;
|
||||||
|
|
||||||
|
import static org.openmetadata.catalog.Entity.CHART;
|
||||||
|
import static org.openmetadata.catalog.Entity.DASHBOARD;
|
||||||
import static org.openmetadata.catalog.Entity.FIELD_USAGE_SUMMARY;
|
import static org.openmetadata.catalog.Entity.FIELD_USAGE_SUMMARY;
|
||||||
|
import static org.openmetadata.catalog.Entity.MLMODEL;
|
||||||
|
import static org.openmetadata.catalog.Entity.TABLE;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
@ -26,6 +30,7 @@ import org.jdbi.v3.core.mapper.RowMapper;
|
|||||||
import org.jdbi.v3.core.statement.StatementContext;
|
import org.jdbi.v3.core.statement.StatementContext;
|
||||||
import org.jdbi.v3.sqlobject.transaction.Transaction;
|
import org.jdbi.v3.sqlobject.transaction.Transaction;
|
||||||
import org.openmetadata.catalog.Entity;
|
import org.openmetadata.catalog.Entity;
|
||||||
|
import org.openmetadata.catalog.EntityInterface;
|
||||||
import org.openmetadata.catalog.entity.data.Chart;
|
import org.openmetadata.catalog.entity.data.Chart;
|
||||||
import org.openmetadata.catalog.entity.data.Dashboard;
|
import org.openmetadata.catalog.entity.data.Dashboard;
|
||||||
import org.openmetadata.catalog.entity.data.MlModel;
|
import org.openmetadata.catalog.entity.data.MlModel;
|
||||||
@ -47,6 +52,8 @@ import org.openmetadata.catalog.util.RestUtil;
|
|||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class UsageRepository {
|
public class UsageRepository {
|
||||||
|
private static final String PUT = "createOrUpdate";
|
||||||
|
private static final String POST = "createNew";
|
||||||
private final CollectionDAO dao;
|
private final CollectionDAO dao;
|
||||||
|
|
||||||
public UsageRepository(CollectionDAO dao) {
|
public UsageRepository(CollectionDAO dao) {
|
||||||
@ -71,14 +78,28 @@ public class UsageRepository {
|
|||||||
public RestUtil.PutResponse create(String entityType, String id, DailyCount usage) throws IOException {
|
public RestUtil.PutResponse create(String entityType, String id, DailyCount usage) throws IOException {
|
||||||
// Validate data entity for which usage is being collected
|
// Validate data entity for which usage is being collected
|
||||||
Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED);
|
Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED);
|
||||||
return addUsage(entityType, id, usage);
|
return addUsage(POST, entityType, id, usage);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transaction
|
@Transaction
|
||||||
public RestUtil.PutResponse createByName(String entityType, String fullyQualifiedName, DailyCount usage)
|
public RestUtil.PutResponse createByName(String entityType, String fullyQualifiedName, DailyCount usage)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
EntityReference ref = Entity.getEntityReferenceByName(entityType, fullyQualifiedName, Include.NON_DELETED);
|
EntityReference ref = Entity.getEntityReferenceByName(entityType, fullyQualifiedName, Include.NON_DELETED);
|
||||||
return addUsage(entityType, ref.getId().toString(), usage);
|
return addUsage(POST, entityType, ref.getId().toString(), usage);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transaction
|
||||||
|
public RestUtil.PutResponse createOrUpdate(String entityType, String id, DailyCount usage) throws IOException {
|
||||||
|
// Validate data entity for which usage is being collected
|
||||||
|
Entity.getEntityReferenceById(entityType, UUID.fromString(id), Include.NON_DELETED);
|
||||||
|
return addUsage(PUT, entityType, id, usage);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transaction
|
||||||
|
public RestUtil.PutResponse createOrUpdateByName(String entityType, String fullyQualifiedName, DailyCount usage)
|
||||||
|
throws IOException {
|
||||||
|
EntityReference ref = Entity.getEntityReferenceByName(entityType, fullyQualifiedName, Include.NON_DELETED);
|
||||||
|
return addUsage(PUT, entityType, ref.getId().toString(), usage);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transaction
|
@Transaction
|
||||||
@ -86,121 +107,112 @@ public class UsageRepository {
|
|||||||
dao.usageDAO().computePercentile(entityType, date);
|
dao.usageDAO().computePercentile(entityType, date);
|
||||||
}
|
}
|
||||||
|
|
||||||
private RestUtil.PutResponse addUsage(String entityType, String entityId, DailyCount usage) throws IOException {
|
private RestUtil.PutResponse addUsage(String method, String entityType, String entityId, DailyCount usage)
|
||||||
|
throws IOException {
|
||||||
Fields fields = new Fields(List.of("usageSummary"));
|
Fields fields = new Fields(List.of("usageSummary"));
|
||||||
// If table usage was reported, add the usage count to schema and database
|
// If table usage was reported, add the usage count to schema and database
|
||||||
if (entityType.equalsIgnoreCase(Entity.TABLE)) {
|
String type = entityType.toLowerCase();
|
||||||
// we accept usage for deleted entities
|
switch (type) {
|
||||||
Table table = Entity.getEntity(Entity.TABLE, UUID.fromString(entityId), fields, Include.ALL);
|
case TABLE:
|
||||||
// Insert usage record
|
return tableEntityUsage(method, fields, entityId, entityType, usage);
|
||||||
dao.usageDAO().insert(usage.getDate(), entityId, entityType, usage.getCount());
|
case DASHBOARD:
|
||||||
Table updated = Entity.getEntity(Entity.TABLE, UUID.fromString(entityId), fields, Include.ALL);
|
return dashboardEntityUsage(method, fields, entityId, entityType, usage);
|
||||||
dao.usageDAO()
|
case CHART:
|
||||||
.insertOrUpdateCount(
|
return chartEntityUsage(method, fields, entityId, entityType, usage);
|
||||||
usage.getDate(), table.getDatabaseSchema().getId().toString(), Entity.DATABASE_SCHEMA, usage.getCount());
|
case MLMODEL:
|
||||||
dao.usageDAO()
|
return mlModelEntityUsage(method, fields, entityId, entityType, usage);
|
||||||
.insertOrUpdateCount(
|
default:
|
||||||
usage.getDate(), table.getDatabase().getId().toString(), Entity.DATABASE, usage.getCount());
|
LOG.error("Invalid Usage Entity Type");
|
||||||
dao.usageDAO().computePercentile(entityType, usage.getDate());
|
throw new UnhandledServerException(CatalogExceptionMessage.entityTypeNotSupported(entityType));
|
||||||
ChangeDescription change = new ChangeDescription().withPreviousVersion(table.getVersion());
|
|
||||||
change
|
|
||||||
.getFieldsUpdated()
|
|
||||||
.add(
|
|
||||||
new FieldChange()
|
|
||||||
.withName(FIELD_USAGE_SUMMARY)
|
|
||||||
.withNewValue(updated.getUsageSummary())
|
|
||||||
.withOldValue(table.getUsageSummary()));
|
|
||||||
ChangeEvent changeEvent =
|
|
||||||
new ChangeEvent()
|
|
||||||
.withEntity(updated)
|
|
||||||
.withChangeDescription(change)
|
|
||||||
.withEventType(EventType.ENTITY_UPDATED)
|
|
||||||
.withEntityType(entityType)
|
|
||||||
.withEntityId(updated.getId())
|
|
||||||
.withEntityFullyQualifiedName(updated.getFullyQualifiedName())
|
|
||||||
.withUserName(updated.getUpdatedBy())
|
|
||||||
.withTimestamp(System.currentTimeMillis())
|
|
||||||
.withCurrentVersion(updated.getVersion())
|
|
||||||
.withPreviousVersion(table.getVersion());
|
|
||||||
|
|
||||||
return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
|
|
||||||
} else if (entityType.equalsIgnoreCase(Entity.DASHBOARD)) {
|
|
||||||
Dashboard dashboard = Entity.getEntity(Entity.DASHBOARD, UUID.fromString(entityId), fields, Include.ALL);
|
|
||||||
dao.usageDAO().insert(usage.getDate(), entityId, entityType, usage.getCount());
|
|
||||||
Dashboard updated = Entity.getEntity(Entity.DASHBOARD, UUID.fromString(entityId), fields, Include.ALL);
|
|
||||||
ChangeDescription change = new ChangeDescription().withPreviousVersion(dashboard.getVersion());
|
|
||||||
change
|
|
||||||
.getFieldsUpdated()
|
|
||||||
.add(
|
|
||||||
new FieldChange()
|
|
||||||
.withName(FIELD_USAGE_SUMMARY)
|
|
||||||
.withNewValue(updated.getUsageSummary())
|
|
||||||
.withOldValue(dashboard.getUsageSummary()));
|
|
||||||
ChangeEvent changeEvent =
|
|
||||||
new ChangeEvent()
|
|
||||||
.withEntity(updated)
|
|
||||||
.withChangeDescription(change)
|
|
||||||
.withEventType(EventType.ENTITY_UPDATED)
|
|
||||||
.withEntityType(entityType)
|
|
||||||
.withEntityId(updated.getId())
|
|
||||||
.withEntityFullyQualifiedName(dashboard.getFullyQualifiedName())
|
|
||||||
.withUserName(updated.getUpdatedBy())
|
|
||||||
.withTimestamp(System.currentTimeMillis())
|
|
||||||
.withCurrentVersion(updated.getVersion())
|
|
||||||
.withPreviousVersion(dashboard.getVersion());
|
|
||||||
return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
|
|
||||||
} else if (entityType.equalsIgnoreCase(Entity.CHART)) {
|
|
||||||
Chart chart = Entity.getEntity(Entity.CHART, UUID.fromString(entityId), fields, Include.ALL);
|
|
||||||
dao.usageDAO().insert(usage.getDate(), entityId, entityType, usage.getCount());
|
|
||||||
Chart updated = Entity.getEntity(Entity.CHART, UUID.fromString(entityId), fields, Include.ALL);
|
|
||||||
ChangeDescription change = new ChangeDescription().withPreviousVersion(chart.getVersion());
|
|
||||||
change
|
|
||||||
.getFieldsUpdated()
|
|
||||||
.add(
|
|
||||||
new FieldChange()
|
|
||||||
.withName(FIELD_USAGE_SUMMARY)
|
|
||||||
.withNewValue(updated.getUsageSummary())
|
|
||||||
.withOldValue(chart.getUsageSummary()));
|
|
||||||
ChangeEvent changeEvent =
|
|
||||||
new ChangeEvent()
|
|
||||||
.withEntity(updated)
|
|
||||||
.withChangeDescription(change)
|
|
||||||
.withEventType(EventType.ENTITY_UPDATED)
|
|
||||||
.withEntityType(entityType)
|
|
||||||
.withEntityId(updated.getId())
|
|
||||||
.withEntityFullyQualifiedName(updated.getFullyQualifiedName())
|
|
||||||
.withUserName(updated.getUpdatedBy())
|
|
||||||
.withTimestamp(System.currentTimeMillis())
|
|
||||||
.withCurrentVersion(updated.getVersion())
|
|
||||||
.withPreviousVersion(chart.getVersion());
|
|
||||||
return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
|
|
||||||
} else if (entityType.equalsIgnoreCase(Entity.MLMODEL)) {
|
|
||||||
MlModel mlModel = Entity.getEntity(Entity.MLMODEL, UUID.fromString(entityId), fields, Include.ALL);
|
|
||||||
dao.usageDAO().insert(usage.getDate(), entityId, entityType, usage.getCount());
|
|
||||||
MlModel updated = Entity.getEntity(Entity.CHART, UUID.fromString(entityId), fields, Include.ALL);
|
|
||||||
ChangeDescription change = new ChangeDescription().withPreviousVersion(mlModel.getVersion());
|
|
||||||
change
|
|
||||||
.getFieldsUpdated()
|
|
||||||
.add(
|
|
||||||
new FieldChange()
|
|
||||||
.withName(FIELD_USAGE_SUMMARY)
|
|
||||||
.withNewValue(updated.getUsageSummary())
|
|
||||||
.withOldValue(mlModel.getUsageSummary()));
|
|
||||||
ChangeEvent changeEvent =
|
|
||||||
new ChangeEvent()
|
|
||||||
.withEntity(updated)
|
|
||||||
.withChangeDescription(change)
|
|
||||||
.withEventType(EventType.ENTITY_UPDATED)
|
|
||||||
.withEntityType(entityType)
|
|
||||||
.withEntityId(updated.getId())
|
|
||||||
.withEntityFullyQualifiedName(updated.getFullyQualifiedName())
|
|
||||||
.withUserName(updated.getUpdatedBy())
|
|
||||||
.withTimestamp(System.currentTimeMillis())
|
|
||||||
.withCurrentVersion(updated.getVersion())
|
|
||||||
.withPreviousVersion(mlModel.getVersion());
|
|
||||||
return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
|
|
||||||
}
|
}
|
||||||
throw new UnhandledServerException(CatalogExceptionMessage.entityTypeNotSupported(entityType));
|
}
|
||||||
|
|
||||||
|
private RestUtil.PutResponse tableEntityUsage(
|
||||||
|
String method, Fields fields, String entityId, String entityType, DailyCount usage) throws IOException {
|
||||||
|
// we accept usage for deleted entities
|
||||||
|
Table table = Entity.getEntity(Entity.TABLE, UUID.fromString(entityId), fields, Include.ALL);
|
||||||
|
// Insert usage record
|
||||||
|
insertToUsageRepository(method, entityId, entityType, usage);
|
||||||
|
Table updated = Entity.getEntity(Entity.TABLE, UUID.fromString(entityId), fields, Include.ALL);
|
||||||
|
insertToUsageRepository(method, table.getDatabaseSchema().getId().toString(), Entity.DATABASE_SCHEMA, usage);
|
||||||
|
insertToUsageRepository(method, table.getDatabase().getId().toString(), Entity.DATABASE, usage);
|
||||||
|
dao.usageDAO().computePercentile(entityType, usage.getDate());
|
||||||
|
|
||||||
|
ChangeDescription change =
|
||||||
|
getChangeDescription(table.getVersion(), updated.getUsageSummary(), table.getUsageSummary());
|
||||||
|
ChangeEvent changeEvent = getChangeEvent(updated, change, entityType, table.getVersion());
|
||||||
|
|
||||||
|
return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
|
||||||
|
}
|
||||||
|
|
||||||
|
private RestUtil.PutResponse dashboardEntityUsage(
|
||||||
|
String method, Fields fields, String entityId, String entityType, DailyCount usage) throws IOException {
|
||||||
|
Dashboard dashboard = Entity.getEntity(Entity.DASHBOARD, UUID.fromString(entityId), fields, Include.ALL);
|
||||||
|
insertToUsageRepository(method, entityId, entityType, usage);
|
||||||
|
Dashboard updated = Entity.getEntity(Entity.DASHBOARD, UUID.fromString(entityId), fields, Include.ALL);
|
||||||
|
|
||||||
|
ChangeDescription change =
|
||||||
|
getChangeDescription(dashboard.getVersion(), updated.getUsageSummary(), dashboard.getUsageSummary());
|
||||||
|
ChangeEvent changeEvent = getChangeEvent(updated, change, entityType, dashboard.getVersion());
|
||||||
|
|
||||||
|
return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
|
||||||
|
}
|
||||||
|
|
||||||
|
private RestUtil.PutResponse chartEntityUsage(
|
||||||
|
String method, Fields fields, String entityId, String entityType, DailyCount usage) throws IOException {
|
||||||
|
Chart chart = Entity.getEntity(Entity.CHART, UUID.fromString(entityId), fields, Include.ALL);
|
||||||
|
insertToUsageRepository(method, entityId, entityType, usage);
|
||||||
|
Chart updated = Entity.getEntity(Entity.CHART, UUID.fromString(entityId), fields, Include.ALL);
|
||||||
|
|
||||||
|
ChangeDescription change =
|
||||||
|
getChangeDescription(chart.getVersion(), updated.getUsageSummary(), chart.getUsageSummary());
|
||||||
|
ChangeEvent changeEvent = getChangeEvent(updated, change, entityType, chart.getVersion());
|
||||||
|
|
||||||
|
return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
|
||||||
|
}
|
||||||
|
|
||||||
|
private RestUtil.PutResponse mlModelEntityUsage(
|
||||||
|
String method, Fields fields, String entityId, String entityType, DailyCount usage) throws IOException {
|
||||||
|
MlModel mlModel = Entity.getEntity(Entity.MLMODEL, UUID.fromString(entityId), fields, Include.ALL);
|
||||||
|
insertToUsageRepository(method, entityId, entityType, usage);
|
||||||
|
MlModel updated = Entity.getEntity(Entity.CHART, UUID.fromString(entityId), fields, Include.ALL);
|
||||||
|
|
||||||
|
ChangeDescription change =
|
||||||
|
getChangeDescription(mlModel.getVersion(), updated.getUsageSummary(), mlModel.getUsageSummary());
|
||||||
|
ChangeEvent changeEvent = getChangeEvent(updated, change, entityType, mlModel.getVersion());
|
||||||
|
|
||||||
|
return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, RestUtil.ENTITY_FIELDS_CHANGED);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void insertToUsageRepository(String method, String entityId, String entityType, DailyCount usage) {
|
||||||
|
if (method.equals(POST)) {
|
||||||
|
dao.usageDAO().insert(usage.getDate(), entityId, entityType, usage.getCount());
|
||||||
|
} else if (method.equals(PUT)) {
|
||||||
|
dao.usageDAO().insertOrUpdateCount(usage.getDate(), entityId, entityType, usage.getCount());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private ChangeEvent getChangeEvent(
|
||||||
|
EntityInterface updated, ChangeDescription change, String entityType, Double prevVersion) {
|
||||||
|
return new ChangeEvent()
|
||||||
|
.withEntity(updated)
|
||||||
|
.withChangeDescription(change)
|
||||||
|
.withEventType(EventType.ENTITY_UPDATED)
|
||||||
|
.withEntityType(entityType)
|
||||||
|
.withEntityId(updated.getId())
|
||||||
|
.withEntityFullyQualifiedName(updated.getFullyQualifiedName())
|
||||||
|
.withUserName(updated.getUpdatedBy())
|
||||||
|
.withTimestamp(System.currentTimeMillis())
|
||||||
|
.withCurrentVersion(updated.getVersion())
|
||||||
|
.withPreviousVersion(prevVersion);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ChangeDescription getChangeDescription(Double version, Object newValue, Object oldValue) {
|
||||||
|
FieldChange fieldChange =
|
||||||
|
new FieldChange().withName(FIELD_USAGE_SUMMARY).withNewValue(newValue).withOldValue(oldValue);
|
||||||
|
ChangeDescription change = new ChangeDescription().withPreviousVersion(version);
|
||||||
|
change.getFieldsUpdated().add(fieldChange);
|
||||||
|
return change;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class UsageDetailsMapper implements RowMapper<UsageDetails> {
|
public static class UsageDetailsMapper implements RowMapper<UsageDetails> {
|
||||||
|
@ -27,6 +27,7 @@ import javax.validation.Valid;
|
|||||||
import javax.ws.rs.Consumes;
|
import javax.ws.rs.Consumes;
|
||||||
import javax.ws.rs.GET;
|
import javax.ws.rs.GET;
|
||||||
import javax.ws.rs.POST;
|
import javax.ws.rs.POST;
|
||||||
|
import javax.ws.rs.PUT;
|
||||||
import javax.ws.rs.Path;
|
import javax.ws.rs.Path;
|
||||||
import javax.ws.rs.PathParam;
|
import javax.ws.rs.PathParam;
|
||||||
import javax.ws.rs.Produces;
|
import javax.ws.rs.Produces;
|
||||||
@ -176,6 +177,37 @@ public class UsageResource {
|
|||||||
return dao.create(entity, id, usage).toResponse();
|
return dao.create(entity, id, usage).toResponse();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PUT
|
||||||
|
@Path("/{entity}/{id}")
|
||||||
|
@Operation(
|
||||||
|
operationId = "reportEntityUsageWithID",
|
||||||
|
summary = "Report usage",
|
||||||
|
tags = "usage",
|
||||||
|
description =
|
||||||
|
"Report usage information for an entity on a given date. System stores last 30 days of usage "
|
||||||
|
+ "information. Usage information older than 30 days is deleted.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "Usage information",
|
||||||
|
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityUsage.class))),
|
||||||
|
@ApiResponse(responseCode = "400", description = "Bad request")
|
||||||
|
})
|
||||||
|
public Response createOrUpdate(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Parameter(
|
||||||
|
description = "Entity type for which usage is reported",
|
||||||
|
required = true,
|
||||||
|
schema = @Schema(type = "string", example = "table, report, metrics, or dashboard"))
|
||||||
|
@PathParam("entity")
|
||||||
|
String entity,
|
||||||
|
@Parameter(description = "Entity id", required = true, schema = @Schema(type = "string")) @PathParam("id")
|
||||||
|
String id,
|
||||||
|
@Parameter(description = "Usage information a given date") @Valid DailyCount usage)
|
||||||
|
throws IOException {
|
||||||
|
return dao.createOrUpdate(entity, id, usage).toResponse();
|
||||||
|
}
|
||||||
|
|
||||||
@POST
|
@POST
|
||||||
@Path("/{entity}/name/{fqn}")
|
@Path("/{entity}/name/{fqn}")
|
||||||
@Operation(
|
@Operation(
|
||||||
@ -211,6 +243,41 @@ public class UsageResource {
|
|||||||
return dao.createByName(entity, fullyQualifiedName, usage).toResponse();
|
return dao.createByName(entity, fullyQualifiedName, usage).toResponse();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PUT
|
||||||
|
@Path("/{entity}/name/{fqn}")
|
||||||
|
@Operation(
|
||||||
|
operationId = "reportEntityUsageWithFQN",
|
||||||
|
summary = "Report usage by name",
|
||||||
|
tags = "usage",
|
||||||
|
description =
|
||||||
|
"Report usage information for an entity by name on a given date. System stores last 30 days "
|
||||||
|
+ "of usage information. Usage information older than 30 days is deleted.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
responseCode = "200",
|
||||||
|
description = "Usage information",
|
||||||
|
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityUsage.class))),
|
||||||
|
@ApiResponse(responseCode = "400", description = "Bad request")
|
||||||
|
})
|
||||||
|
public Response createorUpdateByName(
|
||||||
|
@Context UriInfo uriInfo,
|
||||||
|
@Parameter(
|
||||||
|
description = "Entity type for which usage is reported",
|
||||||
|
required = true,
|
||||||
|
schema = @Schema(type = "string", example = "table, report, metrics, or dashboard"))
|
||||||
|
@PathParam("entity")
|
||||||
|
String entity,
|
||||||
|
@Parameter(
|
||||||
|
description = "Fully qualified name of the entity that uniquely identifies an entity",
|
||||||
|
required = true,
|
||||||
|
schema = @Schema(type = "string"))
|
||||||
|
@PathParam("fqn")
|
||||||
|
String fullyQualifiedName,
|
||||||
|
@Parameter(description = "Usage information a given date") @Valid DailyCount usage)
|
||||||
|
throws IOException {
|
||||||
|
return dao.createOrUpdateByName(entity, fullyQualifiedName, usage).toResponse();
|
||||||
|
}
|
||||||
|
|
||||||
@POST
|
@POST
|
||||||
@Path("/compute.percentile/{entity}/{date}")
|
@Path("/compute.percentile/{entity}/{date}")
|
||||||
@Operation(
|
@Operation(
|
||||||
|
@ -69,9 +69,15 @@ public final class RestUtil {
|
|||||||
|
|
||||||
public static URI getHref(URI parent, String child) {
|
public static URI getHref(URI parent, String child) {
|
||||||
child = removeSlashes(child);
|
child = removeSlashes(child);
|
||||||
|
child = replaceSpaces(child);
|
||||||
return URI.create(parent.toString() + "/" + child);
|
return URI.create(parent.toString() + "/" + child);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String replaceSpaces(String s) {
|
||||||
|
s = s.replaceAll(" ", "%20");
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
public static URI getHref(UriInfo uriInfo, String collectionPath, String resourcePath) {
|
public static URI getHref(UriInfo uriInfo, String collectionPath, String resourcePath) {
|
||||||
collectionPath = removeSlashes(collectionPath);
|
collectionPath = removeSlashes(collectionPath);
|
||||||
resourcePath = removeSlashes(resourcePath);
|
resourcePath = removeSlashes(resourcePath);
|
||||||
|
@ -33,6 +33,8 @@ import java.util.Map;
|
|||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import javax.ws.rs.client.WebTarget;
|
import javax.ws.rs.client.WebTarget;
|
||||||
|
import javax.ws.rs.core.Response;
|
||||||
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.http.client.HttpResponseException;
|
import org.apache.http.client.HttpResponseException;
|
||||||
import org.junit.jupiter.api.Assertions;
|
import org.junit.jupiter.api.Assertions;
|
||||||
@ -58,6 +60,8 @@ import org.openmetadata.catalog.util.TestUtils;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||||
class UsageResourceTest extends CatalogApplicationTest {
|
class UsageResourceTest extends CatalogApplicationTest {
|
||||||
|
private static final String PUT = "PUT";
|
||||||
|
private static final String POST = "POST";
|
||||||
public static final List<Table> TABLES = new ArrayList<>();
|
public static final List<Table> TABLES = new ArrayList<>();
|
||||||
public static final int TABLE_COUNT = 10;
|
public static final int TABLE_COUNT = 10;
|
||||||
public static final int DAYS_OF_USAGE = 32;
|
public static final int DAYS_OF_USAGE = 32;
|
||||||
@ -81,6 +85,14 @@ class UsageResourceTest extends CatalogApplicationTest {
|
|||||||
entityNotFound(TABLE, NON_EXISTENT_ENTITY));
|
entityNotFound(TABLE, NON_EXISTENT_ENTITY));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void put_usageWithNonExistentEntityId_4xx() {
|
||||||
|
assertResponse(
|
||||||
|
() -> reportUsagePut(TABLE, NON_EXISTENT_ENTITY, usageReport(), ADMIN_AUTH_HEADERS),
|
||||||
|
NOT_FOUND,
|
||||||
|
entityNotFound(TABLE, NON_EXISTENT_ENTITY));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void post_usageInvalidEntityName_4xx() {
|
void post_usageInvalidEntityName_4xx() {
|
||||||
String invalidEntityType = "invalid";
|
String invalidEntityType = "invalid";
|
||||||
@ -90,6 +102,15 @@ class UsageResourceTest extends CatalogApplicationTest {
|
|||||||
entityTypeNotFound(invalidEntityType));
|
entityTypeNotFound(invalidEntityType));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void put_usageInvalidEntityName_4xx() {
|
||||||
|
String invalidEntityType = "invalid";
|
||||||
|
assertResponse(
|
||||||
|
() -> reportUsagePut(invalidEntityType, UUID.randomUUID(), usageReport(), ADMIN_AUTH_HEADERS),
|
||||||
|
NOT_FOUND,
|
||||||
|
entityTypeNotFound(invalidEntityType));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void post_usageWithNegativeCountName_4xx() {
|
void post_usageWithNegativeCountName_4xx() {
|
||||||
DailyCount dailyCount = usageReport().withCount(-1); // Negative usage count
|
DailyCount dailyCount = usageReport().withCount(-1); // Negative usage count
|
||||||
@ -99,6 +120,15 @@ class UsageResourceTest extends CatalogApplicationTest {
|
|||||||
"[count must be greater than or equal to 0]");
|
"[count must be greater than or equal to 0]");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void put_usageWithNegativeCountName_4xx() {
|
||||||
|
DailyCount dailyCount = usageReport().withCount(-1); // Negative usage count
|
||||||
|
assertResponse(
|
||||||
|
() -> reportUsagePut(TABLE, UUID.randomUUID(), dailyCount, ADMIN_AUTH_HEADERS),
|
||||||
|
BAD_REQUEST,
|
||||||
|
"[count must be greater than or equal to 0]");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void post_usageWithoutDate_4xx() {
|
void post_usageWithoutDate_4xx() {
|
||||||
DailyCount usageReport = usageReport().withDate(null); // Negative usage count
|
DailyCount usageReport = usageReport().withDate(null); // Negative usage count
|
||||||
@ -108,17 +138,41 @@ class UsageResourceTest extends CatalogApplicationTest {
|
|||||||
"[date must not be null]");
|
"[date must not be null]");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void put_usageWithoutDate_4xx() {
|
||||||
|
DailyCount usageReport = usageReport().withDate(null); // Negative usage count
|
||||||
|
assertResponse(
|
||||||
|
() -> reportUsagePut(TABLE, UUID.randomUUID(), usageReport, ADMIN_AUTH_HEADERS),
|
||||||
|
BAD_REQUEST,
|
||||||
|
"[date must not be null]");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void post_validUsageByName_200_OK(TestInfo test) throws HttpResponseException {
|
void post_validUsageByName_200_OK(TestInfo test) throws HttpResponseException {
|
||||||
|
testValidUsageByName(test, POST);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void put_validUsageByName_200_OK(TestInfo test) throws HttpResponseException {
|
||||||
|
testValidUsageByName(test, PUT);
|
||||||
|
}
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
void testValidUsageByName(TestInfo test, String methodType) {
|
||||||
TableResourceTest tableResourceTest = new TableResourceTest();
|
TableResourceTest tableResourceTest = new TableResourceTest();
|
||||||
Table table = tableResourceTest.createEntity(tableResourceTest.createRequest(test), ADMIN_AUTH_HEADERS);
|
Table table = tableResourceTest.createEntity(tableResourceTest.createRequest(test), ADMIN_AUTH_HEADERS);
|
||||||
DailyCount usageReport = usageReport().withCount(100).withDate(RestUtil.DATE_FORMAT.format(new Date()));
|
DailyCount usageReport = usageReport().withCount(100).withDate(RestUtil.DATE_FORMAT.format(new Date()));
|
||||||
reportUsageByNameAndCheck(TABLE, table.getFullyQualifiedName(), usageReport, 100, 100, ADMIN_AUTH_HEADERS);
|
reportUsageByNameAndCheckPut(TABLE, table.getFullyQualifiedName(), usageReport, 100, 100, ADMIN_AUTH_HEADERS);
|
||||||
|
// a put request updates the data again
|
||||||
|
if (methodType.equals(PUT)) {
|
||||||
|
reportUsageByNamePut(TABLE, table.getFullyQualifiedName(), usageReport, ADMIN_AUTH_HEADERS);
|
||||||
|
checkUsageByName(usageReport.getDate(), TABLE, table.getFullyQualifiedName(), 200, 200, 200, ADMIN_AUTH_HEADERS);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Order(1) // Run this method first before other usage records are created
|
@Order(1) // Run this method first before other usage records are created
|
||||||
@Test
|
@Test
|
||||||
void post_validUsageForTables_200_OK() throws HttpResponseException {
|
void put_validUsageForTables_200_OK() throws HttpResponseException {
|
||||||
// This test creates TABLE_COUNT of tables.
|
// This test creates TABLE_COUNT of tables.
|
||||||
// For these tables, publish usage data for DAYS_OF_USAGE number of days starting from today.
|
// For these tables, publish usage data for DAYS_OF_USAGE number of days starting from today.
|
||||||
// For 100 tables send usage report for last 30 days
|
// For 100 tables send usage report for last 30 days
|
||||||
@ -150,7 +204,7 @@ class UsageResourceTest extends CatalogApplicationTest {
|
|||||||
// Report usage
|
// Report usage
|
||||||
int weeklyCount = Math.min(day + 1, 7) * usageCount; // Expected cumulative weekly count
|
int weeklyCount = Math.min(day + 1, 7) * usageCount; // Expected cumulative weekly count
|
||||||
int monthlyCount = Math.min(day + 1, 30) * usageCount; // Expected cumulative monthly count
|
int monthlyCount = Math.min(day + 1, 30) * usageCount; // Expected cumulative monthly count
|
||||||
reportUsageAndCheck(TABLE, id, usageReport, weeklyCount, monthlyCount, ADMIN_AUTH_HEADERS);
|
reportUsageAndCheckPut(TABLE, id, usageReport, weeklyCount, monthlyCount, ADMIN_AUTH_HEADERS);
|
||||||
|
|
||||||
// Database has cumulative count of all the table usage
|
// Database has cumulative count of all the table usage
|
||||||
databaseDailyCount += usageCount;
|
databaseDailyCount += usageCount;
|
||||||
@ -251,6 +305,13 @@ class UsageResourceTest extends CatalogApplicationTest {
|
|||||||
checkUsageByName(usage.getDate(), entity, fqn, usage.getCount(), weeklyCount, monthlyCount, authHeaders);
|
checkUsageByName(usage.getDate(), entity, fqn, usage.getCount(), weeklyCount, monthlyCount, authHeaders);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void reportUsageByNameAndCheckPut(
|
||||||
|
String entity, String fqn, DailyCount usage, int weeklyCount, int monthlyCount, Map<String, String> authHeaders)
|
||||||
|
throws HttpResponseException {
|
||||||
|
reportUsageByNamePut(entity, fqn, usage, authHeaders);
|
||||||
|
checkUsageByName(usage.getDate(), entity, fqn, usage.getCount(), weeklyCount, monthlyCount, authHeaders);
|
||||||
|
}
|
||||||
|
|
||||||
public static void reportUsageAndCheck(
|
public static void reportUsageAndCheck(
|
||||||
String entity, UUID id, DailyCount usage, int weeklyCount, int monthlyCount, Map<String, String> authHeaders)
|
String entity, UUID id, DailyCount usage, int weeklyCount, int monthlyCount, Map<String, String> authHeaders)
|
||||||
throws HttpResponseException {
|
throws HttpResponseException {
|
||||||
@ -258,18 +319,37 @@ class UsageResourceTest extends CatalogApplicationTest {
|
|||||||
checkUsage(usage.getDate(), entity, id, usage.getCount(), weeklyCount, monthlyCount, authHeaders);
|
checkUsage(usage.getDate(), entity, id, usage.getCount(), weeklyCount, monthlyCount, authHeaders);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void reportUsageAndCheckPut(
|
||||||
|
String entity, UUID id, DailyCount usage, int weeklyCount, int monthlyCount, Map<String, String> authHeaders)
|
||||||
|
throws HttpResponseException {
|
||||||
|
reportUsagePut(entity, id, usage, authHeaders);
|
||||||
|
checkUsage(usage.getDate(), entity, id, usage.getCount(), weeklyCount, monthlyCount, authHeaders);
|
||||||
|
}
|
||||||
|
|
||||||
public static void reportUsageByName(String entity, String name, DailyCount usage, Map<String, String> authHeaders)
|
public static void reportUsageByName(String entity, String name, DailyCount usage, Map<String, String> authHeaders)
|
||||||
throws HttpResponseException {
|
throws HttpResponseException {
|
||||||
WebTarget target = getResource("usage/" + entity + "/name/" + name);
|
WebTarget target = getResource("usage/" + entity + "/name/" + name);
|
||||||
TestUtils.post(target, usage, authHeaders);
|
TestUtils.post(target, usage, authHeaders);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void reportUsageByNamePut(String entity, String name, DailyCount usage, Map<String, String> authHeaders)
|
||||||
|
throws HttpResponseException {
|
||||||
|
WebTarget target = getResource("usage/" + entity + "/name/" + name);
|
||||||
|
TestUtils.put(target, usage, Response.Status.CREATED, authHeaders);
|
||||||
|
}
|
||||||
|
|
||||||
public static void reportUsage(String entity, UUID id, DailyCount usage, Map<String, String> authHeaders)
|
public static void reportUsage(String entity, UUID id, DailyCount usage, Map<String, String> authHeaders)
|
||||||
throws HttpResponseException {
|
throws HttpResponseException {
|
||||||
WebTarget target = getResource("usage/" + entity + "/" + id);
|
WebTarget target = getResource("usage/" + entity + "/" + id);
|
||||||
TestUtils.post(target, usage, authHeaders);
|
TestUtils.post(target, usage, authHeaders);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void reportUsagePut(String entity, UUID id, DailyCount usage, Map<String, String> authHeaders)
|
||||||
|
throws HttpResponseException {
|
||||||
|
WebTarget target = getResource("usage/" + entity + "/" + id);
|
||||||
|
TestUtils.put(target, usage, Response.Status.CREATED, authHeaders);
|
||||||
|
}
|
||||||
|
|
||||||
public static void computePercentile(String entity, String date, Map<String, String> authHeaders)
|
public static void computePercentile(String entity, String date, Map<String, String> authHeaders)
|
||||||
throws HttpResponseException {
|
throws HttpResponseException {
|
||||||
WebTarget target = getResource("usage/compute.percentile/" + entity + "/" + date);
|
WebTarget target = getResource("usage/compute.percentile/" + entity + "/" + date);
|
||||||
|
@ -49,6 +49,11 @@ class RestUtilTest {
|
|||||||
assertEquals(URI.create("http://base/collection/path"), RestUtil.getHref(uriInfo, "/collection", "/path"));
|
assertEquals(URI.create("http://base/collection/path"), RestUtil.getHref(uriInfo, "/collection", "/path"));
|
||||||
assertEquals(URI.create("http://base/collection/path"), RestUtil.getHref(uriInfo, "collection/", "path/"));
|
assertEquals(URI.create("http://base/collection/path"), RestUtil.getHref(uriInfo, "collection/", "path/"));
|
||||||
assertEquals(URI.create("http://base/collection/path"), RestUtil.getHref(uriInfo, "/collection/", "/path/"));
|
assertEquals(URI.create("http://base/collection/path"), RestUtil.getHref(uriInfo, "/collection/", "/path/"));
|
||||||
|
|
||||||
|
assertEquals(URI.create("http://base/collection/path%201"), RestUtil.getHref(uriInfo, "collection", "path 1"));
|
||||||
|
assertEquals(URI.create("http://base/collection/path%201"), RestUtil.getHref(uriInfo, "/collection", "/path 1"));
|
||||||
|
assertEquals(URI.create("http://base/collection/path%201"), RestUtil.getHref(uriInfo, "collection/", "path 1/"));
|
||||||
|
assertEquals(URI.create("http://base/collection/path%201"), RestUtil.getHref(uriInfo, "/collection/", "/path 1/"));
|
||||||
}
|
}
|
||||||
|
|
||||||
private UriInfo mockUriInfo(String uri) throws URISyntaxException {
|
private UriInfo mockUriInfo(String uri) throws URISyntaxException {
|
||||||
|
@ -44,7 +44,7 @@ import ConfirmationModal from '../../components/Modals/ConfirmationModal/Confirm
|
|||||||
import FormModal from '../../components/Modals/FormModal';
|
import FormModal from '../../components/Modals/FormModal';
|
||||||
import { ModalWithMarkdownEditor } from '../../components/Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor';
|
import { ModalWithMarkdownEditor } from '../../components/Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor';
|
||||||
import { TITLE_FOR_NON_ADMIN_ACTION } from '../../constants/constants';
|
import { TITLE_FOR_NON_ADMIN_ACTION } from '../../constants/constants';
|
||||||
import { delimiterRegex, nameWithSpace } from '../../constants/regex.constants';
|
import { delimiterRegex } from '../../constants/regex.constants';
|
||||||
import {
|
import {
|
||||||
CreateTagCategory,
|
CreateTagCategory,
|
||||||
TagCategoryType,
|
TagCategoryType,
|
||||||
@ -154,8 +154,6 @@ const TagsPage = () => {
|
|||||||
const errData: { [key: string]: string } = {};
|
const errData: { [key: string]: string } = {};
|
||||||
if (!data.name.trim()) {
|
if (!data.name.trim()) {
|
||||||
errData['name'] = 'Name is required';
|
errData['name'] = 'Name is required';
|
||||||
} else if (nameWithSpace.test(data.name)) {
|
|
||||||
errData['name'] = 'Name with space is not allowed';
|
|
||||||
} else if (delimiterRegex.test(data.name)) {
|
} else if (delimiterRegex.test(data.name)) {
|
||||||
errData['name'] = 'Name with delimiters are not allowed';
|
errData['name'] = 'Name with delimiters are not allowed';
|
||||||
} else if (
|
} else if (
|
||||||
@ -315,8 +313,6 @@ const TagsPage = () => {
|
|||||||
const errData: { [key: string]: string } = {};
|
const errData: { [key: string]: string } = {};
|
||||||
if (!data.name.trim()) {
|
if (!data.name.trim()) {
|
||||||
errData['name'] = 'Name is required';
|
errData['name'] = 'Name is required';
|
||||||
} else if (nameWithSpace.test(data.name)) {
|
|
||||||
errData['name'] = 'Name with space is not allowed';
|
|
||||||
} else if (delimiterRegex.test(data.name)) {
|
} else if (delimiterRegex.test(data.name)) {
|
||||||
errData['name'] = 'Name with delimiters are not allowed';
|
errData['name'] = 'Name with delimiters are not allowed';
|
||||||
} else if (
|
} else if (
|
||||||
|
Loading…
x
Reference in New Issue
Block a user