yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-07 15:05:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(openapi): point in time parameter (elasticsearch only) (#12553)

Browse Source
This commit is contained in:
david-leifker 2025-02-05 18:07:33 -06:00 committed by GitHub
parent 7f88710e02
commit 0ed3d7f4a4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 185 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java
View File

@ -57,6 +57,8 @@ import io.datahubproject.openapi.models.GenericEntity;
import io.datahubproject.openapi.models.GenericEntityScrollResult;
import io.datahubproject.openapi.util.RequestInputUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import java.lang.reflect.InvocationTargetException;
@ -212,8 +214,12 @@ public abstract class GenericEntitiesController<
Boolean skipCache,
@RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false")
Boolean includeSoftDelete,
@RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m")
String pitKeepALive)
@Parameter(
schema = @Schema(nullable = true),
description =
"Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.")
@RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m")
String pitKeepAlive)
throws URISyntaxException {
EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName);
@ -261,7 +267,7 @@ public abstract class GenericEntitiesController<
null,
sortCriteria,
scrollId,
pitKeepALive,
pitKeepAlive != null && pitKeepAlive.isEmpty() ? null : pitKeepAlive,
count);
if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) {

8
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java
View File

@ -360,6 +360,12 @@ public class OpenAPIV3Generator {
.$ref(
String.format(
"#/components/parameters/%s", aspectParameterName + MODEL_VERSION)),
new Parameter()
.in(NAME_QUERY)
.name(NAME_PIT_KEEP_ALIVE)
.description(
"Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.")
.schema(new Schema().type(TYPE_STRING)._default("5m")),
new Parameter().$ref("#/components/parameters/PaginationCount" + MODEL_VERSION),
new Parameter().$ref("#/components/parameters/ScrollId" + MODEL_VERSION),
new Parameter().$ref("#/components/parameters/SortBy" + MODEL_VERSION),
@ -534,7 +540,7 @@ public class OpenAPIV3Generator {
.name(NAME_PIT_KEEP_ALIVE)
.description(
"Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.")
.schema(new Schema().type(TYPE_STRING)._default("5m")),
.schema(new Schema().type(TYPE_STRING)._default("5m").nullable(true)),
new Parameter().$ref("#/components/parameters/PaginationCount" + MODEL_VERSION),
new Parameter().$ref("#/components/parameters/ScrollId" + MODEL_VERSION),
new Parameter().$ref("#/components/parameters/SortBy" + MODEL_VERSION),

12
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java
View File

@ -59,6 +59,8 @@ import io.datahubproject.openapi.v3.models.GenericEntityScrollResultV3;
import io.datahubproject.openapi.v3.models.GenericEntityV3;
import io.swagger.v3.oas.annotations.Hidden;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import java.net.URISyntaxException;
@ -160,8 +162,12 @@ public class EntityController
Boolean skipCache,
@RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false")
Boolean includeSoftDelete,
@RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m")
String pitKeepALive,
@Parameter(
schema = @Schema(nullable = true),
description =
"Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.")
@RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m")
String pitKeepAlive,
@RequestBody @Nonnull GenericEntityAspectsBodyV3 entityAspectsBody)
throws URISyntaxException {
@ -218,7 +224,7 @@ public class EntityController
null,
sortCriteria,
scrollId,
pitKeepALive,
pitKeepAlive != null && pitKeepAlive.isEmpty() ? null : pitKeepAlive,
count);
if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) {

160
smoke-test/tests/openapi/v3/entities.json Normal file
View File

@ -0,0 +1,160 @@
[
{
"request": {
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset1EntityV3%2CPROD%29",
"description": "Remove test dataset 1",
"method": "delete"
}
},
{
"request": {
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset2EntityV3%2CPROD%29",
"description": "Remove test dataset 2",
"method": "delete"
}
},
{
"request": {
"url": "/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atest%2Cdataset3EntityV3%2CPROD%29",
"description": "Remove test dataset 3",
"method": "delete"
}
},
{
"request": {
"url": "/openapi/v3/entity/dataset",
"params": {
"createIfNotExists": "false",
"createEntityIfNotExists": "false",
"async": "false"
},
"description": "Create 3 datasets",
"json": [
{
"urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset1EntityV3,PROD)",
"datasetProperties": {
"value": {
"name": "dataset1EntityV3",
"qualifiedName": "entities.dataset1EntityV3",
"customProperties": {},
"tags": []
}
},
"status": {
"value": {
"removed": false
}
}
},
{
"urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset2EntityV3,PROD)",
"datasetProperties": {
"value": {
"name": "dataset2EntityV3",
"qualifiedName": "entities.dataset2EntityV3",
"customProperties": {},
"tags": []
}
},
"status": {
"value": {
"removed": false
}
}
},
{
"urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset3EntityV3,PROD)",
"datasetProperties": {
"value": {
"name": "dataset3EntityV3",
"qualifiedName": "entities.dataset3EntityV3",
"customProperties": {},
"tags": []
}
},
"status": {
"value": {
"removed": false
}
}
}
]
}
},
{
"request": {
"url": "/openapi/v3/entity/scroll",
"params": {
"pitKeepAlive": "1m",
"count": "1",
"query": "qualifiedName:entities\\.dataset*",
"sortCriteria": "urn"
},
"description": "Generic scroll",
"json": {
"aspects": [
"datasetProperties"
]
}
},
"response": {
"exclude_regex_paths": [
"root\\['scrollId'\\]"
],
"json": {
"entities": [
{
"urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset1EntityV3,PROD)",
"datasetProperties": {
"value": {
"name": "dataset1EntityV3",
"qualifiedName": "entities.dataset1EntityV3",
"customProperties": {},
"tags": []
}
}
}
]
}
}
},
{
"request": {
"url": "/openapi/v3/entity/dataset?aspects=datasetProperties&aspects=status",
"method": "get",
"params": {
"pitKeepAlive": "1m",
"count": "1",
"query": "qualifiedName:entities\\.dataset*",
"sortCriteria": "urn",
"sortOrder": "DESCENDING"
},
"description": "Dataset Entity scroll"
},
"response": {
"exclude_regex_paths": [
"root\\['scrollId'\\]"
],
"json": {
"entities": [
{
"urn": "urn:li:dataset:(urn:li:dataPlatform:test,dataset3EntityV3,PROD)",
"datasetProperties": {
"value": {
"name": "dataset3EntityV3",
"qualifiedName": "entities.dataset3EntityV3",
"customProperties": {},
"tags": []
}
},
"status": {
"value": {
"removed": false
}
}
}
]
}
}
}
]