From 0ed3d7f4a4ce63db1348ece71a4731f9e920f2eb Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:07:33 -0600 Subject: [PATCH] feat(openapi): point in time parameter (elasticsearch only) (#12553) --- .../controller/GenericEntitiesController.java | 12 +- .../openapi/v3/OpenAPIV3Generator.java | 8 +- .../v3/controller/EntityController.java | 12 +- smoke-test/tests/openapi/v3/entities.json | 160 ++++++++++++++++++ 4 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 smoke-test/tests/openapi/v3/entities.json diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index d3c67355d0f..b5e6120ec08 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -57,6 +57,8 @@ import io.datahubproject.openapi.models.GenericEntity; import io.datahubproject.openapi.models.GenericEntityScrollResult; import io.datahubproject.openapi.util.RequestInputUtil; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.servlet.http.HttpServletRequest; import java.lang.reflect.InvocationTargetException; @@ -212,8 +214,12 @@ public abstract class GenericEntitiesController< Boolean skipCache, @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false") Boolean includeSoftDelete, - @RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m") - String pitKeepALive) + @Parameter( + schema = @Schema(nullable = true), + description = + "Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.") + @RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m") + String pitKeepAlive) throws URISyntaxException { EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); @@ -261,7 +267,7 @@ public abstract class GenericEntitiesController< null, sortCriteria, scrollId, - pitKeepALive, + pitKeepAlive != null && pitKeepAlive.isEmpty() ? null : pitKeepAlive, count); if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) { diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index 349672ef2dc..4a2293fd4c6 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -360,6 +360,12 @@ public class OpenAPIV3Generator { .$ref( String.format( "#/components/parameters/%s", aspectParameterName + MODEL_VERSION)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_PIT_KEEP_ALIVE) + .description( + "Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.") + .schema(new Schema().type(TYPE_STRING)._default("5m")), new Parameter().$ref("#/components/parameters/PaginationCount" + MODEL_VERSION), new Parameter().$ref("#/components/parameters/ScrollId" + MODEL_VERSION), new Parameter().$ref("#/components/parameters/SortBy" + MODEL_VERSION), @@ -534,7 +540,7 @@ public class OpenAPIV3Generator { .name(NAME_PIT_KEEP_ALIVE) .description( "Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.") - .schema(new Schema().type(TYPE_STRING)._default("5m")), + .schema(new Schema().type(TYPE_STRING)._default("5m").nullable(true)), new Parameter().$ref("#/components/parameters/PaginationCount" + MODEL_VERSION), new Parameter().$ref("#/components/parameters/ScrollId" + MODEL_VERSION), new Parameter().$ref("#/components/parameters/SortBy" + MODEL_VERSION), diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java index 9bf7851653d..6236b1eb7a7 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java @@ -59,6 +59,8 @@ import io.datahubproject.openapi.v3.models.GenericEntityScrollResultV3; import io.datahubproject.openapi.v3.models.GenericEntityV3; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.servlet.http.HttpServletRequest; import java.net.URISyntaxException; @@ -160,8 +162,12 @@ public class EntityController Boolean skipCache, @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false") Boolean includeSoftDelete, - @RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m") - String pitKeepALive, + @Parameter( + schema = @Schema(nullable = true), + description = + "Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.") + @RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m") + String pitKeepAlive, @RequestBody @Nonnull GenericEntityAspectsBodyV3 entityAspectsBody) throws URISyntaxException { @@ -218,7 +224,7 @@ public class EntityController null, sortCriteria, scrollId, - pitKeepALive, + pitKeepAlive != null && pitKeepAlive.isEmpty() ? null : pitKeepAlive, count); if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) { diff --git a/smoke-test/tests/openapi/v3/entities.json b/smoke-test/tests/openapi/v3/entities.json new file mode 100644 index 00000000000..079174fd78b --- /dev/null +++ b/smoke-test/tests/openapi/v3/entities.json @@ -0,0 +1,160 @@ +[ + { + "request": { + "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset1EntityV3%2CPROD%29", + "description": "Remove test dataset 1", + "method": "delete" + } + }, + { + "request": { + "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset2EntityV3%2CPROD%29", + "description": "Remove test dataset 2", + "method": "delete" + } + }, + { + "request": { + "url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset3EntityV3%2CPROD%29", + "description": "Remove test dataset 3", + "method": "delete" + } + }, + { + "request": { + "url": "/openapi/v3/entity/dataset", + "params": { + "createIfNotExists": "false", + "createEntityIfNotExists": "false", + "async": "false" + }, + "description": "Create 3 datasets", + "json": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset1EntityV3,PROD)", + "datasetProperties": { + "value": { + "name": "dataset1EntityV3", + "qualifiedName": "entities.dataset1EntityV3", + "customProperties": {}, + "tags": [] + } + }, + "status": { + "value": { + "removed": false + } + } + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset2EntityV3,PROD)", + "datasetProperties": { + "value": { + "name": "dataset2EntityV3", + "qualifiedName": "entities.dataset2EntityV3", + "customProperties": {}, + "tags": [] + } + }, + "status": { + "value": { + "removed": false + } + } + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset3EntityV3,PROD)", + "datasetProperties": { + "value": { + "name": "dataset3EntityV3", + "qualifiedName": "entities.dataset3EntityV3", + "customProperties": {}, + "tags": [] + } + }, + "status": { + "value": { + "removed": false + } + } + } + ] + } + }, + { + "request": { + "url": "/openapi/v3/entity/scroll", + "params": { + "pitKeepAlive": "1m", + "count": "1", + "query": "qualifiedName:entities\\.dataset*", + "sortCriteria": "urn" + }, + "description": "Generic scroll", + "json": { + "aspects": [ + "datasetProperties" + ] + } + }, + "response": { + "exclude_regex_paths": [ + "root\\['scrollId'\\]" + ], + "json": { + "entities": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset1EntityV3,PROD)", + "datasetProperties": { + "value": { + "name": "dataset1EntityV3", + "qualifiedName": "entities.dataset1EntityV3", + "customProperties": {}, + "tags": [] + } + } + } + ] + } + } + }, + { + "request": { + "url": "/openapi/v3/entity/dataset?aspects=datasetProperties&aspects=status", + "method": "get", + "params": { + "pitKeepAlive": "1m", + "count": "1", + "query": "qualifiedName:entities\\.dataset*", + "sortCriteria": "urn", + "sortOrder": "DESCENDING" + }, + "description": "Dataset Entity scroll" + }, + "response": { + "exclude_regex_paths": [ + "root\\['scrollId'\\]" + ], + "json": { + "entities": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset3EntityV3,PROD)", + "datasetProperties": { + "value": { + "name": "dataset3EntityV3", + "qualifiedName": "entities.dataset3EntityV3", + "customProperties": {}, + "tags": [] + } + }, + "status": { + "value": { + "removed": false + } + } + } + ] + } + } + } +] \ No newline at end of file