diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index 1f3255c2a8d..f46f04e9f24 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -52,6 +52,7 @@ import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.jdbi3.DatabaseRepository.DatabaseEntityInterface; import org.openmetadata.catalog.resources.databases.TableResource; import org.openmetadata.catalog.tests.ColumnTest; +import org.openmetadata.catalog.tests.CustomMetric; import org.openmetadata.catalog.tests.TableTest; import org.openmetadata.catalog.tests.type.TestCaseResult; import org.openmetadata.catalog.type.ChangeDescription; @@ -115,6 +116,7 @@ public class TableRepository extends EntityRepository { table.setProfileSample(fields.contains("profileSample") ? table.getProfileSample() : null); table.setTableTests(fields.contains("tests") ? getTableTests(table) : null); getColumnTests(fields.contains("tests"), table); + getCustomMetrics(fields.contains("customMetrics"), table); return table; } @@ -402,6 +404,79 @@ public class TableRepository extends EntityRepository
{ return table; } + @Transaction + public Table addCustomMetric(UUID tableId, CustomMetric customMetric) throws IOException, ParseException { + // Validate the request content + Table table = daoCollection.tableDAO().findEntityById(tableId); + String columnName = customMetric.getColumnName(); + validateColumn(table, columnName); + + // Override any custom metric definition with the same name + List storedCustomMetrics = getCustomMetrics(table, columnName); + Map storedMapCustomMetrics = new HashMap<>(); + if (storedCustomMetrics != null) { + for (CustomMetric cm : storedCustomMetrics) { + storedMapCustomMetrics.put(cm.getName(), cm); + } + } + + // existing metric use the previous UUID + if (storedMapCustomMetrics.containsKey(customMetric.getName())) { + CustomMetric prevMetric = storedMapCustomMetrics.get(customMetric.getName()); + customMetric.setId(prevMetric.getId()); + } + + storedMapCustomMetrics.put(customMetric.getName(), customMetric); + List updatedMetrics = new ArrayList<>(storedMapCustomMetrics.values()); + String extension = "table.column." + columnName + ".customMetrics"; + daoCollection + .entityExtensionDAO() + .insert(table.getId().toString(), extension, "customMetric", JsonUtils.pojoToJson(updatedMetrics)); + setFields(table, Fields.EMPTY_FIELDS); + // return the newly created/updated custom metric only + for (Column column : table.getColumns()) { + if (column.getName().equals(columnName)) { + column.setCustomMetrics(List.of(customMetric)); + } + } + return table; + } + + @Transaction + public Table deleteCustomMetric(UUID tableId, String columnName, String metricName) throws IOException { + // Validate the request content + Table table = daoCollection.tableDAO().findEntityById(tableId); + validateColumn(table, columnName); + + // Override any custom metric definition with the same name + List storedCustomMetrics = getCustomMetrics(table, columnName); + Map storedMapCustomMetrics = new HashMap<>(); + if (storedCustomMetrics != null) { + for (CustomMetric cm : storedCustomMetrics) { + storedMapCustomMetrics.put(cm.getName(), cm); + } + } + + if (!storedMapCustomMetrics.containsKey(metricName)) { + throw new EntityNotFoundException(String.format("Failed to find %s for %s", metricName, table.getName())); + } + + CustomMetric deleteCustomMetric = storedMapCustomMetrics.get(metricName); + storedMapCustomMetrics.remove(metricName); + List updatedMetrics = new ArrayList<>(storedMapCustomMetrics.values()); + String extension = "table.column." + columnName + ".customMetrics"; + daoCollection + .entityExtensionDAO() + .insert(table.getId().toString(), extension, "customMetric", JsonUtils.pojoToJson(updatedMetrics)); + // return the newly created/updated custom metric test only + for (Column column : table.getColumns()) { + if (column.getName().equals(columnName)) { + column.setCustomMetrics(List.of(deleteCustomMetric)); + } + } + return table; + } + @Transaction public Table addDataModel(UUID tableId, DataModel dataModel) throws IOException, ParseException { Table table = daoCollection.tableDAO().findEntityById(tableId); @@ -806,6 +881,20 @@ public class TableRepository extends EntityRepository
{ } } + private List getCustomMetrics(Table table, String columnName) throws IOException { + String extension = "table.column." + columnName + ".customMetrics"; + return JsonUtils.readObjects( + daoCollection.entityExtensionDAO().getExtension(table.getId().toString(), extension), CustomMetric.class); + } + + private void getCustomMetrics(boolean setMetrics, Table table) throws IOException { + // Add custom metrics info to columns if requested + List columns = table.getColumns(); + for (Column c : listOrEmpty(columns)) { + c.setCustomMetrics(setMetrics ? getCustomMetrics(table, c.getName()) : null); + } + } + public static class TableEntityInterface implements EntityInterface
{ private final Table entity; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java index 064e3c98ba6..46246e67687 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java @@ -52,6 +52,7 @@ import javax.ws.rs.core.UriInfo; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.data.CreateTable; import org.openmetadata.catalog.api.tests.CreateColumnTest; +import org.openmetadata.catalog.api.tests.CreateCustomMetric; import org.openmetadata.catalog.api.tests.CreateTableTest; import org.openmetadata.catalog.entity.data.Table; import org.openmetadata.catalog.jdbi3.CollectionDAO; @@ -62,6 +63,7 @@ import org.openmetadata.catalog.resources.EntityResource; import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.security.SecurityUtil; import org.openmetadata.catalog.tests.ColumnTest; +import org.openmetadata.catalog.tests.CustomMetric; import org.openmetadata.catalog.tests.TableTest; import org.openmetadata.catalog.type.DataModel; import org.openmetadata.catalog.type.EntityHistory; @@ -96,6 +98,7 @@ public class TableResource extends EntityResource { public TableResource(CollectionDAO dao, Authorizer authorizer) { super(Table.class, new TableRepository(dao), authorizer); allowedFields.add("tests"); + allowedFields.add("customMetrics"); } public static class TableList extends ResultList
{ @@ -110,13 +113,14 @@ public class TableResource extends EntityResource { } static final String FIELDS = - "tableConstraints,usageSummary,owner,profileSample," + "tableConstraints,usageSummary,owner,profileSample,customMetrics," + "tags,followers,joins,sampleData,viewDefinition,tableProfile,location,tableQueries,dataModel,tests"; public static final List ALLOWED_FIELDS; static { List list = new ArrayList<>(Entity.getEntityFields(Table.class)); list.add("tests"); // Add a field parameter called tests that represent the fields - tableTests and columnTests + list.add("customMetrics"); // Add a field parameter to add customMetrics information to the columns ALLOWED_FIELDS = Collections.unmodifiableList(list); } @@ -539,7 +543,7 @@ public class TableResource extends EntityResource { @PUT @Path("/{id}/columnTest") - @Operation(summary = "Add table test cases", tags = "tables", description = "Add test cases to the table.") + @Operation(summary = "Add column test cases", tags = "tables", description = "Add column test cases to the table.") public Table addColumnTest( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @@ -552,6 +556,21 @@ public class TableResource extends EntityResource { return addHref(uriInfo, table); } + @PUT + @Path("/{id}/customMetric") + @Operation(summary = "Add column custom metrics", tags = "tables", description = "Add column custom metrics.") + public Table addCustomMetric( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id, + CreateCustomMetric createCustomMetric) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + CustomMetric customMetric = getCustomMetric(securityContext, createCustomMetric); + Table table = dao.addCustomMetric(UUID.fromString(id), customMetric); + return addHref(uriInfo, table); + } + @DELETE @Path("/{id}/columnTest/{columnName}/{columnTestType}") @Operation( @@ -572,6 +591,26 @@ public class TableResource extends EntityResource { return addHref(uriInfo, table); } + @DELETE + @Path("/{id}/customMetric/{columnName}/{customMetricName}") + @Operation( + summary = "delete custom metric from a column", + tags = "tables", + description = "Delete a custom metric from a column.") + public Table deleteCustomMetric( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the table", schema = @Schema(type = "string")) @PathParam("id") String id, + @Parameter(description = "column of the table", schema = @Schema(type = "string")) @PathParam("columnName") + String columnName, + @Parameter(description = "column Test Type", schema = @Schema(type = "string")) @PathParam("customMetricName") + String customMetricName) + throws IOException, ParseException { + SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); + Table table = dao.deleteCustomMetric(UUID.fromString(id), columnName, customMetricName); + return addHref(uriInfo, table); + } + @DELETE @Path("/{id}/followers/{userId}") @Operation( @@ -654,4 +693,16 @@ public class TableResource extends EntityResource { .withUpdatedBy(securityContext.getUserPrincipal().getName()) .withUpdatedAt(System.currentTimeMillis()); } + + private CustomMetric getCustomMetric(SecurityContext securityContext, CreateCustomMetric create) { + return new CustomMetric() + .withId(UUID.randomUUID()) + .withDescription(create.getDescription()) + .withName(create.getName()) + .withColumnName(create.getColumnName()) + .withOwner(create.getOwner()) + .withExpression(create.getExpression()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } } diff --git a/catalog-rest-service/src/main/resources/json/schema/api/tests/createCustomMetric.json b/catalog-rest-service/src/main/resources/json/schema/api/tests/createCustomMetric.json new file mode 100644 index 00000000000..11b04c49cbe --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/api/tests/createCustomMetric.json @@ -0,0 +1,42 @@ +{ + "$id": "https://open-metadata.org/schema/api/tests/customMetric.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateCustomMetricRequest", + "description": "Custom Metric definition that we will associate with a column.", + "type": "object", + "properties": { + "description": { + "description": "Description of the custom metric.", + "type": "string" + }, + "name": { + "description": "Name that identifies this Custom Metric.", + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "columnName": { + "description": "Name of the column in a table.", + "type": "string" + }, + "expression": { + "description": "SQL expression to compute the Metric. It should return a single numerical value.", + "type": "string" + }, + "owner": { + "description": "Owner of this Pipeline.", + "$ref": "../../type/entityReference.json", + "default": null + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + } + }, + "required": ["name", "columnName", "expression"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json index f8b74fa57d4..c8981d31644 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json @@ -198,6 +198,14 @@ "$ref": "../../tests/columnTest.json" }, "default": null + }, + "customMetrics": { + "description": "List of Custom Metrics registered for a column.", + "type": "array", + "items": { + "$ref": "../../tests/customMetric.json" + }, + "default": null } }, "required": ["name", "dataType"], @@ -275,6 +283,22 @@ }, "additionalProperties": false }, + "customMetricProfile": { + "type": "object", + "javaType": "org.openmetadata.catalog.type.CustomMetricProfile", + "description": "Profiling results of a Custom Metric.", + "properties": { + "name": { + "description": "Custom metric name.", + "type": "string" + }, + "value": { + "description": "Profiling results for the metric.", + "type": "number" + } + }, + "additionalProperties": false + }, "columnProfile": { "type": "object", "javaType": "org.openmetadata.catalog.type.ColumnProfile", @@ -377,6 +401,14 @@ } }, "additionalProperties": false + }, + "customMetricsProfile": { + "description": "Custom Metrics profile list bound to a column.", + "type": "array", + "items": { + "$ref": "#/definitions/customMetricProfile" + }, + "default": null } }, "additionalProperties": false diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json b/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json index 574dee2b685..53fdd02669b 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/columnTest.json @@ -97,6 +97,6 @@ "type": "string" } }, - "required": ["name", "column", "testCase"], + "required": ["name", "columnName", "testCase"], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/customMetric.json b/catalog-rest-service/src/main/resources/json/schema/tests/customMetric.json new file mode 100644 index 00000000000..ba469a69978 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/tests/customMetric.json @@ -0,0 +1,47 @@ +{ + "$id": "https://open-metadata.org/schema/tests/customMetric.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CustomMetric", + "description": "Custom Metric definition that we will associate with a column.", + "type": "object", + "javaType": "org.openmetadata.catalog.tests.CustomMetric", + "properties": { + "id": { + "description": "Unique identifier of this Custom Metric instance.", + "$ref": "../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name that identifies this Custom Metric.", + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "description": { + "description": "Description of the Metric.", + "type": "string" + }, + "columnName": { + "description": "Name of the column in a table.", + "type": "string" + }, + "expression": { + "description": "SQL expression to compute the Metric. It should return a single numerical value.", + "type": "string" + }, + "owner": { + "description": "Owner of this Custom Metric.", + "$ref": "../type/entityReference.json", + "default": null + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + } + }, + "required": ["name", "columnName", "expression"], + "additionalProperties": false +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index 40d48f6bbff..249c17e14ad 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -76,6 +76,7 @@ import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.data.CreateLocation; import org.openmetadata.catalog.api.data.CreateTable; import org.openmetadata.catalog.api.tests.CreateColumnTest; +import org.openmetadata.catalog.api.tests.CreateCustomMetric; import org.openmetadata.catalog.api.tests.CreateTableTest; import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.data.Location; @@ -91,6 +92,7 @@ import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest; import org.openmetadata.catalog.resources.tags.TagResourceTest; import org.openmetadata.catalog.tests.ColumnTest; import org.openmetadata.catalog.tests.ColumnTestCase; +import org.openmetadata.catalog.tests.CustomMetric; import org.openmetadata.catalog.tests.TableTest; import org.openmetadata.catalog.tests.TableTestCase; import org.openmetadata.catalog.tests.column.ColumnValueLengthsToBeBetween; @@ -1052,6 +1054,49 @@ public class TableResourceTest extends EntityResourceTest { assertEquals(expected.getGeneratedAt(), actual.getGeneratedAt()); } + @Test + void createUpdateDelete_tableCustomMetrics_200(TestInfo test) throws IOException { + Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS); + Column c1 = table.getColumns().get(0); + + CreateCustomMetric createMetric = + new CreateCustomMetric() + .withName("custom") + .withColumnName(c1.getName()) + .withExpression("SELECT SUM(xyz) FROM abc"); + Table putResponse = putCustomMetric(table.getId(), createMetric, ADMIN_AUTH_HEADERS); + verifyCustomMetrics(putResponse, c1, List.of(createMetric)); + + table = getEntity(table.getId(), "customMetrics", ADMIN_AUTH_HEADERS); + verifyCustomMetrics(table, c1, List.of(createMetric)); + + // Update Custom Metric + CreateCustomMetric updatedMetric = + new CreateCustomMetric() + .withName("custom") + .withColumnName(c1.getName()) + .withExpression("Another select statement"); + putResponse = putCustomMetric(table.getId(), updatedMetric, ADMIN_AUTH_HEADERS); + verifyCustomMetrics(putResponse, c1, List.of(updatedMetric)); + + // Add another Custom Metric + CreateCustomMetric createMetric2 = + new CreateCustomMetric() + .withName("custom2") + .withColumnName(c1.getName()) + .withExpression("Yet another statement"); + putResponse = putCustomMetric(table.getId(), createMetric2, ADMIN_AUTH_HEADERS); + verifyCustomMetrics(putResponse, c1, List.of(createMetric2)); + + table = getEntity(table.getId(), "customMetrics", ADMIN_AUTH_HEADERS); + verifyCustomMetrics(table, c1, List.of(updatedMetric, createMetric2)); + + // Delete Custom Metric + putResponse = deleteCustomMetric(table.getId(), c1.getName(), updatedMetric.getName(), ADMIN_AUTH_HEADERS); + table = getEntity(table.getId(), "customMetrics", ADMIN_AUTH_HEADERS); + verifyCustomMetrics(table, c1, List.of(createMetric2)); + } + @Test void createUpdateDelete_tableColumnTests_200(TestInfo test) throws IOException { Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS); @@ -1706,6 +1751,20 @@ public class TableResourceTest extends EntityResourceTest { return TestUtils.delete(target, Table.class, authHeaders); } + public static Table putCustomMetric(UUID tableId, CreateCustomMetric data, Map authHeaders) + throws HttpResponseException { + WebTarget target = CatalogApplicationTest.getResource("tables/" + tableId + "/customMetric"); + return TestUtils.put(target, data, Table.class, OK, authHeaders); + } + + public static Table deleteCustomMetric( + UUID tableId, String columnName, String metricName, Map authHeaders) + throws HttpResponseException { + WebTarget target = + CatalogApplicationTest.getResource("tables/" + tableId + "/customMetric/" + columnName + "/" + metricName); + return TestUtils.delete(target, Table.class, authHeaders); + } + private static int getTagUsageCount(String tagFQN, Map authHeaders) throws HttpResponseException { return TagResourceTest.getTag(tagFQN, "usageCount", authHeaders).getUsageCount(); } @@ -1807,6 +1866,29 @@ public class TableResourceTest extends EntityResourceTest { } } + private void verifyCustomMetrics(Table table, Column column, List expectedMetrics) { + List actualMetrics = new ArrayList<>(); + for (Column c : table.getColumns()) { + if (c.getName().equals(column.getName())) { + actualMetrics = c.getCustomMetrics(); + } + } + assertEquals(actualMetrics.size(), expectedMetrics.size()); + + Map columnMetricMap = new HashMap<>(); + for (CustomMetric metric : actualMetrics) { + columnMetricMap.put(metric.getName(), metric); + } + + for (CreateCustomMetric metric : expectedMetrics) { + CustomMetric storedMetric = columnMetricMap.get(metric.getName()); + assertNotNull(storedMetric); + assertEquals(metric.getDescription(), storedMetric.getDescription()); + assertEquals(metric.getOwner(), storedMetric.getOwner()); + assertEquals(metric.getExpression(), storedMetric.getExpression()); + } + } + private void verifyColumnTestCase(ColumnTestCase expected, ColumnTestCase actual) { assertEquals(expected.getColumnTestType(), actual.getColumnTestType()); if (expected.getColumnTestType() == ColumnTestCase.ColumnTestType.COLUMN_VALUES_TO_BE_UNIQUE) { diff --git a/ingestion-core/src/metadata/_version.py b/ingestion-core/src/metadata/_version.py index d239a5a4d8b..4688c49cdf7 100644 --- a/ingestion-core/src/metadata/_version.py +++ b/ingestion-core/src/metadata/_version.py @@ -7,5 +7,5 @@ Provides metadata version information. from incremental import Version -__version__ = Version("metadata", 0, 9, 0, dev=2) +__version__ = Version("metadata", 0, 9, 0, dev=3) __all__ = ["__version__"]