yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-30 08:57:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #3463 - Prepare Custom Metric definition (#3467)

Browse Source 
Fix #3463 - Prepare Custom Metric definition (#3467)
This commit is contained in:
Pere Miquel Brull 2022-03-22 15:29:41 +01:00 committed by GitHub
parent 32b80ef329
commit dfd6286cb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 347 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -52,6 +52,7 @@ import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface;
import org.openmetadata.catalog.resources.databases.TableResource;
import org.openmetadata.catalog.tests.ColumnTest;
import org.openmetadata.catalog.tests.CustomMetric;
import org.openmetadata.catalog.tests.TableTest;
import org.openmetadata.catalog.tests.type.TestCaseResult;
import org.openmetadata.catalog.type.ChangeDescription;
@ -115,6 +116,7 @@ public class TableRepository extends EntityRepository<Table> {
table.setProfileSample(fields.contains("profileSample") ? table.getProfileSample() : null);
table.setTableTests(fields.contains("tests") ? getTableTests(table) : null);
getColumnTests(fields.contains("tests"), table);
getCustomMetrics(fields.contains("customMetrics"), table);
return table;
}
@ -402,6 +404,79 @@ public class TableRepository extends EntityRepository<Table> {
return table;
}
@Transaction
public Table addCustomMetric(UUID tableId, CustomMetric customMetric) throws IOException, ParseException {
// Validate the request content
Table table = daoCollection.tableDAO().findEntityById(tableId);
String columnName = customMetric.getColumnName();
validateColumn(table, columnName);
// Override any custom metric definition with the same name
List<CustomMetric> storedCustomMetrics = getCustomMetrics(table, columnName);
Map<String, CustomMetric> storedMapCustomMetrics = new HashMap<>();
if (storedCustomMetrics != null) {
for (CustomMetric cm : storedCustomMetrics) {
storedMapCustomMetrics.put(cm.getName(), cm);
}
}
// existing metric use the previous UUID
if (storedMapCustomMetrics.containsKey(customMetric.getName())) {
CustomMetric prevMetric = storedMapCustomMetrics.get(customMetric.getName());
customMetric.setId(prevMetric.getId());
}
storedMapCustomMetrics.put(customMetric.getName(), customMetric);
List<CustomMetric> updatedMetrics = new ArrayList<>(storedMapCustomMetrics.values());
String extension = "table.column." + columnName + ".customMetrics";
daoCollection
.entityExtensionDAO()
.insert(table.getId().toString(), extension, "customMetric", JsonUtils.pojoToJson(updatedMetrics));
setFields(table, Fields.EMPTY_FIELDS);
// return the newly created/updated custom metric only
for (Column column : table.getColumns()) {
if (column.getName().equals(columnName)) {
column.setCustomMetrics(List.of(customMetric));
}
}
return table;
}
@Transaction
public Table deleteCustomMetric(UUID tableId, String columnName, String metricName) throws IOException {
// Validate the request content
Table table = daoCollection.tableDAO().findEntityById(tableId);
validateColumn(table, columnName);
// Override any custom metric definition with the same name
List<CustomMetric> storedCustomMetrics = getCustomMetrics(table, columnName);
Map<String, CustomMetric> storedMapCustomMetrics = new HashMap<>();
if (storedCustomMetrics != null) {
for (CustomMetric cm : storedCustomMetrics) {
storedMapCustomMetrics.put(cm.getName(), cm);
}
}
if (!storedMapCustomMetrics.containsKey(metricName)) {
throw new EntityNotFoundException(String.format("Failed to find %s for %s", metricName, table.getName()));
}
CustomMetric deleteCustomMetric = storedMapCustomMetrics.get(metricName);
storedMapCustomMetrics.remove(metricName);
List<CustomMetric> updatedMetrics = new ArrayList<>(storedMapCustomMetrics.values());
String extension = "table.column." + columnName + ".customMetrics";
daoCollection
.entityExtensionDAO()
.insert(table.getId().toString(), extension, "customMetric", JsonUtils.pojoToJson(updatedMetrics));
// return the newly created/updated custom metric test only
for (Column column : table.getColumns()) {
if (column.getName().equals(columnName)) {
column.setCustomMetrics(List.of(deleteCustomMetric));
}
}
return table;
}
@Transaction
public Table addDataModel(UUID tableId, DataModel dataModel) throws IOException, ParseException {
Table table = daoCollection.tableDAO().findEntityById(tableId);
@ -806,6 +881,20 @@ public class TableRepository extends EntityRepository<Table> {
}
}
private List<CustomMetric> getCustomMetrics(Table table, String columnName) throws IOException {
String extension = "table.column." + columnName + ".customMetrics";
return JsonUtils.readObjects(
daoCollection.entityExtensionDAO().getExtension(table.getId().toString(), extension), CustomMetric.class);
}
private void getCustomMetrics(boolean setMetrics, Table table) throws IOException {
// Add custom metrics info to columns if requested
List<Column> columns = table.getColumns();
for (Column c : listOrEmpty(columns)) {
c.setCustomMetrics(setMetrics ? getCustomMetrics(table, c.getName()) : null);
}
}
public static class TableEntityInterface implements EntityInterface<Table> {
private final Table entity;

55
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java
View File

@ -52,6 +52,7 @@ import javax.ws.rs.core.UriInfo;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateTable;
import org.openmetadata.catalog.api.tests.CreateColumnTest;
import org.openmetadata.catalog.api.tests.CreateCustomMetric;
import org.openmetadata.catalog.api.tests.CreateTableTest;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
@ -62,6 +63,7 @@ import org.openmetadata.catalog.resources.EntityResource;
import org.openmetadata.catalog.security.Authorizer;
import org.openmetadata.catalog.security.SecurityUtil;
import org.openmetadata.catalog.tests.ColumnTest;
import org.openmetadata.catalog.tests.CustomMetric;
import org.openmetadata.catalog.tests.TableTest;
import org.openmetadata.catalog.type.DataModel;
import org.openmetadata.catalog.type.EntityHistory;
@ -96,6 +98,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
public TableResource(CollectionDAO dao, Authorizer authorizer) {
super(Table.class, new TableRepository(dao), authorizer);
allowedFields.add("tests");
allowedFields.add("customMetrics");
}
public static class TableList extends ResultList<Table> {
@ -110,13 +113,14 @@ public class TableResource extends EntityResource<Table, TableRepository> {
}
static final String FIELDS =
"tableConstraints,usageSummary,owner,profileSample,"
"tableConstraints,usageSummary,owner,profileSample,customMetrics,"
+ "tags,followers,joins,sampleData,viewDefinition,tableProfile,location,tableQueries,dataModel,tests";
public static final List<String> ALLOWED_FIELDS;
static {
List<String> list = new ArrayList<>(Entity.getEntityFields(Table.class));
list.add("tests"); // Add a field parameter called tests that represent the fields - tableTests and columnTests
list.add("customMetrics"); // Add a field parameter to add customMetrics information to the columns
ALLOWED_FIELDS = Collections.unmodifiableList(list);
}
@ -539,7 +543,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
@PUT
@Path("/{id}/columnTest")
@Operation(summary = "Add table test cases", tags = "tables", description = "Add test cases to the table.")
@Operation(summary = "Add column test cases", tags = "tables", description = "Add column test cases to the table.")
public Table addColumnTest(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@ -552,6 +556,21 @@ public class TableResource extends EntityResource<Table, TableRepository> {
return addHref(uriInfo, table);
}
@PUT
@Path("/{id}/customMetric")
@Operation(summary = "Add column custom metrics", tags = "tables", description = "Add column custom metrics.")
public Table addCustomMetric(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id,
CreateCustomMetric createCustomMetric)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
CustomMetric customMetric = getCustomMetric(securityContext, createCustomMetric);
Table table = dao.addCustomMetric(UUID.fromString(id), customMetric);
return addHref(uriInfo, table);
}
@DELETE
@Path("/{id}/columnTest/{columnName}/{columnTestType}")
@Operation(
@ -572,6 +591,26 @@ public class TableResource extends EntityResource<Table, TableRepository> {
return addHref(uriInfo, table);
}
@DELETE
@Path("/{id}/customMetric/{columnName}/{customMetricName}")
@Operation(
summary = "delete custom metric from a column",
tags = "tables",
description = "Delete a custom metric from a column.")
public Table deleteCustomMetric(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id,
@Parameter(description = "column of the table", schema = @Schema(type = "string")) @PathParam("columnName")
String columnName,
@Parameter(description = "column Test Type", schema = @Schema(type = "string")) @PathParam("customMetricName")
String customMetricName)
throws IOException, ParseException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
Table table = dao.deleteCustomMetric(UUID.fromString(id), columnName, customMetricName);
return addHref(uriInfo, table);
}
@DELETE
@Path("/{id}/followers/{userId}")
@Operation(
@ -654,4 +693,16 @@ public class TableResource extends EntityResource<Table, TableRepository> {
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}
private CustomMetric getCustomMetric(SecurityContext securityContext, CreateCustomMetric create) {
return new CustomMetric()
.withId(UUID.randomUUID())
.withDescription(create.getDescription())
.withName(create.getName())
.withColumnName(create.getColumnName())
.withOwner(create.getOwner())
.withExpression(create.getExpression())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(System.currentTimeMillis());
}
}

42
catalog-rest-service/src/main/resources/json/schema/api/tests/createCustomMetric.json Normal file
View File

@ -0,0 +1,42 @@
{
"$id": "https://open-metadata.org/schema/api/tests/customMetric.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CreateCustomMetricRequest",
"description": "Custom Metric definition that we will associate with a column.",
"type": "object",
"properties": {
"description": {
"description": "Description of the custom metric.",
"type": "string"
},
"name": {
"description": "Name that identifies this Custom Metric.",
"type": "string",
"minLength": 1,
"maxLength": 128
},
"columnName": {
"description": "Name of the column in a table.",
"type": "string"
},
"expression": {
"description": "SQL expression to compute the Metric. It should return a single numerical value.",
"type": "string"
},
"owner": {
"description": "Owner of this Pipeline.",
"$ref": "../../type/entityReference.json",
"default": null
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
}
},
"required": ["name", "columnName", "expression"],
"additionalProperties": false
}

32
catalog-rest-service/src/main/resources/json/schema/entity/data/table.json
View File

@ -198,6 +198,14 @@
"$ref": "../../tests/columnTest.json"
},
"default": null
},
"customMetrics": {
"description": "List of Custom Metrics registered for a column.",
"type": "array",
"items": {
"$ref": "../../tests/customMetric.json"
},
"default": null
}
},
"required": ["name", "dataType"],
@ -275,6 +283,22 @@
},
"additionalProperties": false
},
"customMetricProfile": {
"type": "object",
"javaType": "org.openmetadata.catalog.type.CustomMetricProfile",
"description": "Profiling results of a Custom Metric.",
"properties": {
"name": {
"description": "Custom metric name.",
"type": "string"
},
"value": {
"description": "Profiling results for the metric.",
"type": "number"
}
},
"additionalProperties": false
},
"columnProfile": {
"type": "object",
"javaType": "org.openmetadata.catalog.type.ColumnProfile",
@ -377,6 +401,14 @@
}
},
"additionalProperties": false
},
"customMetricsProfile": {
"description": "Custom Metrics profile list bound to a column.",
"type": "array",
"items": {
"$ref": "#/definitions/customMetricProfile"
},
"default": null
}
},
"additionalProperties": false

2
catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json
View File

@ -97,6 +97,6 @@
"type": "string"
}
},
"required": ["name", "column", "testCase"],
"required": ["name", "columnName", "testCase"],
"additionalProperties": false
}

47
catalog-rest-service/src/main/resources/json/schema/tests/customMetric.json Normal file
View File

@ -0,0 +1,47 @@
{
"$id": "https://open-metadata.org/schema/tests/customMetric.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CustomMetric",
"description": "Custom Metric definition that we will associate with a column.",
"type": "object",
"javaType": "org.openmetadata.catalog.tests.CustomMetric",
"properties": {
"id": {
"description": "Unique identifier of this Custom Metric instance.",
"$ref": "../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Name that identifies this Custom Metric.",
"type": "string",
"minLength": 1,
"maxLength": 128
},
"description": {
"description": "Description of the Metric.",
"type": "string"
},
"columnName": {
"description": "Name of the column in a table.",
"type": "string"
},
"expression": {
"description": "SQL expression to compute the Metric. It should return a single numerical value.",
"type": "string"
},
"owner": {
"description": "Owner of this Custom Metric.",
"$ref": "../type/entityReference.json",
"default": null
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
}
},
"required": ["name", "columnName", "expression"],
"additionalProperties": false
}

82
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java
View File

@ -76,6 +76,7 @@ import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.data.CreateLocation;
import org.openmetadata.catalog.api.data.CreateTable;
import org.openmetadata.catalog.api.tests.CreateColumnTest;
import org.openmetadata.catalog.api.tests.CreateCustomMetric;
import org.openmetadata.catalog.api.tests.CreateTableTest;
import org.openmetadata.catalog.entity.data.Database;
import org.openmetadata.catalog.entity.data.Location;
@ -91,6 +92,7 @@ import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest;
import org.openmetadata.catalog.resources.tags.TagResourceTest;
import org.openmetadata.catalog.tests.ColumnTest;
import org.openmetadata.catalog.tests.ColumnTestCase;
import org.openmetadata.catalog.tests.CustomMetric;
import org.openmetadata.catalog.tests.TableTest;
import org.openmetadata.catalog.tests.TableTestCase;
import org.openmetadata.catalog.tests.column.ColumnValueLengthsToBeBetween;
@ -1052,6 +1054,49 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
assertEquals(expected.getGeneratedAt(), actual.getGeneratedAt());
}
@Test
void createUpdateDelete_tableCustomMetrics_200(TestInfo test) throws IOException {
Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS);
Column c1 = table.getColumns().get(0);
CreateCustomMetric createMetric =
new CreateCustomMetric()
.withName("custom")
.withColumnName(c1.getName())
.withExpression("SELECT SUM(xyz) FROM abc");
Table putResponse = putCustomMetric(table.getId(), createMetric, ADMIN_AUTH_HEADERS);
verifyCustomMetrics(putResponse, c1, List.of(createMetric));
table = getEntity(table.getId(), "customMetrics", ADMIN_AUTH_HEADERS);
verifyCustomMetrics(table, c1, List.of(createMetric));
// Update Custom Metric
CreateCustomMetric updatedMetric =
new CreateCustomMetric()
.withName("custom")
.withColumnName(c1.getName())
.withExpression("Another select statement");
putResponse = putCustomMetric(table.getId(), updatedMetric, ADMIN_AUTH_HEADERS);
verifyCustomMetrics(putResponse, c1, List.of(updatedMetric));
// Add another Custom Metric
CreateCustomMetric createMetric2 =
new CreateCustomMetric()
.withName("custom2")
.withColumnName(c1.getName())
.withExpression("Yet another statement");
putResponse = putCustomMetric(table.getId(), createMetric2, ADMIN_AUTH_HEADERS);
verifyCustomMetrics(putResponse, c1, List.of(createMetric2));
table = getEntity(table.getId(), "customMetrics", ADMIN_AUTH_HEADERS);
verifyCustomMetrics(table, c1, List.of(updatedMetric, createMetric2));
// Delete Custom Metric
putResponse = deleteCustomMetric(table.getId(), c1.getName(), updatedMetric.getName(), ADMIN_AUTH_HEADERS);
table = getEntity(table.getId(), "customMetrics", ADMIN_AUTH_HEADERS);
verifyCustomMetrics(table, c1, List.of(createMetric2));
}
@Test
void createUpdateDelete_tableColumnTests_200(TestInfo test) throws IOException {
Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS);
@ -1706,6 +1751,20 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
return TestUtils.delete(target, Table.class, authHeaders);
}
public static Table putCustomMetric(UUID tableId, CreateCustomMetric data, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/customMetric");
return TestUtils.put(target, data, Table.class, OK, authHeaders);
}
public static Table deleteCustomMetric(
UUID tableId, String columnName, String metricName, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target =
CatalogApplicationTest.getResource("tables/" + tableId + "/customMetric/" + columnName + "/" + metricName);
return TestUtils.delete(target, Table.class, authHeaders);
}
private static int getTagUsageCount(String tagFQN, Map<String, String> authHeaders) throws HttpResponseException {
return TagResourceTest.getTag(tagFQN, "usageCount", authHeaders).getUsageCount();
}
@ -1807,6 +1866,29 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
}
private void verifyCustomMetrics(Table table, Column column, List<CreateCustomMetric> expectedMetrics) {
List<CustomMetric> actualMetrics = new ArrayList<>();
for (Column c : table.getColumns()) {
if (c.getName().equals(column.getName())) {
actualMetrics = c.getCustomMetrics();
}
}
assertEquals(actualMetrics.size(), expectedMetrics.size());
Map<String, CustomMetric> columnMetricMap = new HashMap<>();
for (CustomMetric metric : actualMetrics) {
columnMetricMap.put(metric.getName(), metric);
}
for (CreateCustomMetric metric : expectedMetrics) {
CustomMetric storedMetric = columnMetricMap.get(metric.getName());
assertNotNull(storedMetric);
assertEquals(metric.getDescription(), storedMetric.getDescription());
assertEquals(metric.getOwner(), storedMetric.getOwner());
assertEquals(metric.getExpression(), storedMetric.getExpression());
}
}
private void verifyColumnTestCase(ColumnTestCase expected, ColumnTestCase actual) {
assertEquals(expected.getColumnTestType(), actual.getColumnTestType());
if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_TO_BE_UNIQUE) {

2
ingestion-core/src/metadata/_version.py
View File

@ -7,5 +7,5 @@ Provides metadata version information.
from incremental import Version
__version__ = Version("metadata", 0, 9, 0, dev=2)
__version__ = Version("metadata", 0, 9, 0, dev=3)
__all__ = ["__version__"]