diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/Criterion.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/Criterion.java index 3ca7a8f260..12317b2261 100644 --- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/Criterion.java +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/Criterion.java @@ -31,7 +31,8 @@ public class Criterion { STARTS_WITH, ENDS_WITH, EXISTS, - IN + IN, + CONTAIN } /** Convert this criterion to its counterpart in RecordTemplate. */ diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityScrollResultV3.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityScrollResultV3.java index b2d096e814..7c5f0a112a 100644 --- a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityScrollResultV3.java +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityScrollResultV3.java @@ -11,4 +11,5 @@ public class GenericEntityScrollResultV3 implements GenericEntityScrollResult { private String scrollId; private List entities; + private int totalCount; } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index bc386296bb..2a5dc93265 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -120,7 +120,8 @@ public abstract class GenericEntitiesController< Set aspectNames, boolean withSystemMetadata, @Nullable String scrollId, - boolean expandEmpty) + boolean expandEmpty, + int totalCount) throws URISyntaxException; protected List buildEntityList( @@ -294,7 +295,8 @@ public abstract class GenericEntitiesController< mergedAspects, withSystemMetadata, result.getScrollId(), - true)); + true, + result.getNumEntities())); } @Tag(name = "Generic Entities") diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java index a80c05dd50..056c4807ce 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java @@ -129,7 +129,8 @@ public class EntityController Set aspectNames, boolean withSystemMetadata, @Nullable String scrollId, - boolean expandEmpty) + boolean expandEmpty, + int totalCount) throws URISyntaxException { return GenericEntityScrollResultV2.builder() .results( diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index 71fdb5413c..ae6be4e988 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -1403,7 +1403,9 @@ public class OpenAPIV3Generator { "condition", newSchema() .type(TYPE_STRING) - ._enum(Arrays.asList("EQUAL", "STARTS_WITH", "ENDS_WITH", "EXISTS", "IN")) + ._enum( + Arrays.asList( + "EQUAL", "STARTS_WITH", "ENDS_WITH", "EXISTS", "IN", "CONTAIN")) ._default("EQUAL") .description("The condition for the criterion.")) .addProperties( @@ -1559,7 +1561,12 @@ public class OpenAPIV3Generator { newSchema() .$ref( String.format( - "#/components/schemas/%s%s", ENTITIES, ENTITY_RESPONSE_SUFFIX)))); + "#/components/schemas/%s%s", ENTITIES, ENTITY_RESPONSE_SUFFIX)))) + .addProperty( + "totalCount", + newSchema() + .type(TYPE_INTEGER) + .description("Total number of entities satisfy the criteria.")); } private static Schema buildEntityScrollSchema(final EntitySpec entity) { diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java index 21d6d79687..6fd07078d5 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java @@ -266,7 +266,8 @@ public class EntityController entityAspectsBody.getAspects(), withSystemMetadata, result.getScrollId(), - entityAspectsBody.getAspects() != null)); + entityAspectsBody.getAspects() != null, + result.getNumEntities())); } @Tag(name = "EntityVersioning") @@ -596,13 +597,15 @@ public class EntityController Set aspectNames, boolean withSystemMetadata, @Nullable String scrollId, - boolean expandEmpty) + boolean expandEmpty, + int totalCount) throws URISyntaxException { return GenericEntityScrollResultV3.builder() .entities( toRecordTemplates( opContext, searchEntities, aspectNames, withSystemMetadata, expandEmpty)) .scrollId(scrollId) + .totalCount(totalCount) .build(); } diff --git a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java index 47acdfb910..e676d215f1 100644 --- a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java +++ b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/controller/EntityControllerTest.java @@ -154,6 +154,7 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests { // Mock scroll ascending/descending results ScrollResult expectedResultAscending = new ScrollResult() + .setNumEntities(3) .setEntities( new SearchEntityArray( List.of( @@ -172,6 +173,7 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests { .thenReturn(expectedResultAscending); ScrollResult expectedResultDescending = new ScrollResult() + .setNumEntities(3) .setEntities( new SearchEntityArray( List.of( @@ -603,6 +605,7 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests { ScrollResult expectedResult = new ScrollResult() + .setNumEntities(2) .setEntities( new SearchEntityArray( List.of( @@ -649,6 +652,7 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests { ScrollResult expectedResult = new ScrollResult() + .setNumEntities(1) .setEntities( new SearchEntityArray(List.of(new SearchEntity().setEntity(TEST_URNS.get(0))))) .setScrollId("test-scroll-id"); @@ -694,6 +698,7 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests { ScrollResult expectedResult = new ScrollResult() + .setNumEntities(2) .setEntities( new SearchEntityArray( List.of( @@ -768,6 +773,7 @@ public class EntityControllerTest extends AbstractTestNGSpringContextTests { ScrollResult expectedResult = new ScrollResult() + .setNumEntities(1) .setEntities( new SearchEntityArray(List.of(new SearchEntity().setEntity(TEST_URNS.get(0))))); diff --git a/smoke-test/tests/openapi/v3/entities.json b/smoke-test/tests/openapi/v3/entities.json index 7165cbe052..bec9ff35cd 100644 --- a/smoke-test/tests/openapi/v3/entities.json +++ b/smoke-test/tests/openapi/v3/entities.json @@ -113,7 +113,8 @@ }, "response": { "exclude_regex_paths": [ - "root\\['scrollId'\\]" + "root\\['scrollId'\\]", + "root\\['totalCount'\\]" ], "json": { "entities": [ @@ -188,7 +189,63 @@ } } } - ] + ], + "totalCount": 1 + } + } + }, + { + "request": { + "url": "/openapi/v3/entity/scroll", + "params": { + "count": "1" + }, + "description": "Generic scroll with filters", + "json": { + "aspects": [], + "entities": [ + "dataPlatform" + ], + "filter": { + "and": [ + { + "criteria": [ + { + "field": "name", + "values": ["vert"], + "condition": "CONTAIN" + } + ] + } + ] + } + } + }, + "response": { + "exclude_regex_paths": [ + "root\\['scrollId'\\]" + ], + "json": { + "entities": [ + { + "urn": "urn:li:dataPlatform:vertexai", + "dataPlatformKey": { + "value": { + "platformName": "vertexai" + } + }, + "dataPlatformInfo": { + "value": { + "name": "vertexai", + "datasetNameDelimiter": ".", + "type": "OTHERS", + "displayName": "vertexai", + "logoUrl": "/assets/platforms/vertexai.png" + } + } + } + ], + "totalCount": 2 } } }, @@ -212,6 +269,7 @@ "response": { "exclude_regex_paths": [ "root\\['scrollId'\\]", + "root\\['totalCount'\\]", "root\\['entities'\\]\\[0\\]\\['datasetProperties'\\]\\['systemMetadata'\\]\\['properties'\\]\\['sourceIP'\\]", "root\\['entities'\\]\\[0\\]\\['datasetProperties'\\]\\['systemMetadata'\\]\\['properties'\\]\\['telemetryTraceId'\\]", "root\\['entities'\\]\\[0\\]\\['datasetProperties'\\]\\['auditStamp'\\]", @@ -263,7 +321,8 @@ }, "response": { "exclude_regex_paths": [ - "root\\['scrollId'\\]" + "root\\['scrollId'\\]", + "root\\['totalCount'\\]" ], "json": { "entities": [