yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 00:48:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "fix(openapi): fix openapi v2 endpoints & v3 documentation update"

Browse Source 
This reverts commit 573c1cb8407c2a5d152e5abb6b7d9f012eea75cb.
This commit is contained in:
David Leifker 2024-08-09 12:40:34 -05:00
parent 573c1cb840
commit 469654ced7
4 changed files with 110 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
docs/api/tutorials/structured-properties.md
View File

@ -158,37 +158,29 @@ curl -X 'POST' -v \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"qualifiedName": "io.acryl.privacy.retentionTime",
"valueType": "urn:li:dataType:datahub.number",
"description": "Retention Time is used to figure out how long to retain records in a dataset",
"displayName": "Retention Time",
"cardinality": "MULTIPLE",
"entityTypes": [
"urn:li:entityType:datahub.dataset",
"urn:li:entityType:datahub.dataFlow"
],
"allowedValues": [
{
"value": {
"double": 30
},
"description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
},
{
"value": {
"double": 60
},
"description": "Use this for datasets that drive monthly reporting but contain pii"
},
{
"value": {
"double": 365
},
"description": "Use this for non-sensitive data that can be retained for longer"
}
]
}
"valueType": "urn:li:dataType:datahub.number",
"description": "Retention Time is used to figure out how long to retain records in a dataset",
"displayName": "Retention Time",
"cardinality": "MULTIPLE",
"entityTypes": [
"urn:li:entityType:datahub.dataset",
"urn:li:entityType:datahub.dataFlow"
],
"allowedValues": [
{
"value": {"double": 30},
"description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
},
{
"value": {"double": 60},
"description": "Use this for datasets that drive monthly reporting but contain pii"
},
{
"value": {"double": 365},
"description": "Use this for non-sensitive data that can be retained for longer"
}
]
}' | jq
```
@ -482,16 +474,14 @@ curl -X 'POST' -v \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"properties": [
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
"values": [
{"double": 60.0}
]
}
]
}
"properties": [
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
"values": [
{"double": 60.0}
]
}
]
}' | jq
```
Example Response:
@ -637,25 +627,23 @@ curl -X 'POST' -v \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"qualifiedName": "io.acryl.privacy.retentionTime02",
"displayName": "Retention Time 02",
"valueType": "urn:li:dataType:datahub.string",
"allowedValues": [
{
"value": {"string": "foo2"},
"description": "test foo2 value"
},
{
"value": {"string": "bar2"},
"description": "test bar2 value"
}
],
"cardinality": "SINGLE",
"entityTypes": [
"urn:li:entityType:datahub.dataset"
]
}
"qualifiedName": "io.acryl.privacy.retentionTime02",
"displayName": "Retention Time 02",
"valueType": "urn:li:dataType:datahub.string",
"allowedValues": [
{
"value": {"string": "foo2"},
"description": "test foo2 value"
},
{
"value": {"string": "bar2"},
"description": "test bar2 value"
}
],
"cardinality": "SINGLE",
"entityTypes": [
"urn:li:entityType:datahub.dataset"
]
}' | jq
```
@ -698,26 +686,24 @@ Specically, this will set `io.acryl.privacy.retentionTime` as `60.0` and `io.acr
```shell
curl -X 'POST' -v \
'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties?createIfNotExists=false' \
'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"properties": [
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
"values": [
{"double": 60.0}
]
},
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02",
"values": [
{"string": "bar2"}
]
}
]
}
"properties": [
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
"values": [
{"double": 60.0}
]
},
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02",
"values": [
{"string": "bar2"}
]
}
]
}' | jq
```
@ -1125,9 +1111,7 @@ curl -X 'POST' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"removed": true
}
"removed": true
}' | jq
```
@ -1148,13 +1132,11 @@ If you want to **remove the soft delete**, you can do so by either hard deleting
```shell
curl -X 'POST' \
'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false&createIfNotExists=false' \
'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"removed": true
}
"removed": false
}' | jq
```
@ -1289,42 +1271,34 @@ Change the cardinality to `SINGLE` and add a `version`.
```shell
curl -X 'POST' -v \
'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition?createIfNotExists=false' \
'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"qualifiedName": "io.acryl.privacy.retentionTime",
"valueType": "urn:li:dataType:datahub.number",
"description": "Retention Time is used to figure out how long to retain records in a dataset",
"displayName": "Retention Time",
"cardinality": "SINGLE",
"version": "20240614080000",
"entityTypes": [
"urn:li:entityType:datahub.dataset",
"urn:li:entityType:datahub.dataFlow"
],
"allowedValues": [
{
"value": {
"double": 30
},
"description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
},
{
"value": {
"double": 60
},
"description": "Use this for datasets that drive monthly reporting but contain pii"
},
{
"value": {
"double": 365
},
"description": "Use this for non-sensitive data that can be retained for longer"
}
]
}
"valueType": "urn:li:dataType:datahub.number",
"description": "Retention Time is used to figure out how long to retain records in a dataset",
"displayName": "Retention Time",
"cardinality": "SINGLE",
"version": "20240614080000",
"entityTypes": [
"urn:li:entityType:datahub.dataset",
"urn:li:entityType:datahub.dataFlow"
],
"allowedValues": [
{
"value": {"double": 30},
"description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
},
{
"value": {"double": 60},
"description": "Use this for datasets that drive monthly reporting but contain pii"
},
{
"value": {"double": 365},
"description": "Use this for non-sensitive data that can be retained for longer"
}
]
}' | jq
```

23
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java
View File

@ -13,11 +13,14 @@ import com.datahub.authorization.AuthUtil;
import com.datahub.authorization.AuthorizerChain;
import com.datahub.util.RecordUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableSet;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.ByteString;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.entity.EnvelopedAspect;
import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.aspect.AspectRetriever;
import com.linkedin.metadata.aspect.batch.AspectsBatch;
import com.linkedin.metadata.aspect.batch.ChangeMCP;
@ -38,6 +41,7 @@ import com.linkedin.metadata.search.ScrollResult;
import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchService;
import com.linkedin.metadata.utils.AuditStampUtils;
import com.linkedin.metadata.utils.GenericRecordUtils;
import com.linkedin.metadata.utils.SearchUtil;
import com.linkedin.mxe.SystemMetadata;
import com.linkedin.util.Pair;
@ -53,6 +57,7 @@ import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import java.lang.reflect.InvocationTargetException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
@ -721,14 +726,28 @@ public abstract class GenericEntitiesController<
aspectSpec.getDataTemplateClass(), envelopedAspect.getValue().data());
}
protected abstract ChangeMCP toUpsertItem(
protected ChangeMCP toUpsertItem(
@Nonnull AspectRetriever aspectRetriever,
Urn entityUrn,
AspectSpec aspectSpec,
Boolean createIfNotExists,
String jsonAspect,
Actor actor)
throws URISyntaxException, JsonProcessingException;
throws JsonProcessingException {
JsonNode jsonNode = objectMapper.readTree(jsonAspect);
String aspectJson = jsonNode.get("value").toString();
return ChangeItemImpl.builder()
.urn(entityUrn)
.aspectName(aspectSpec.getName())
.changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT)
.auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
.recordTemplate(
GenericRecordUtils.deserializeAspect(
ByteString.copyString(aspectJson, StandardCharsets.UTF_8),
GenericRecordUtils.JSON,
aspectSpec))
.build(aspectRetriever);
}
protected ChangeMCP toUpsertItem(
@Nonnull AspectRetriever aspectRetriever,

25
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java
View File

@ -13,11 +13,8 @@ import com.linkedin.common.urn.UrnUtils;
import com.linkedin.data.ByteString;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.entity.EnvelopedAspect;
import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.aspect.AspectRetriever;
import com.linkedin.metadata.aspect.batch.AspectsBatch;
import com.linkedin.metadata.aspect.batch.BatchItem;
import com.linkedin.metadata.aspect.batch.ChangeMCP;
import com.linkedin.metadata.entity.EntityApiUtils;
import com.linkedin.metadata.entity.IngestResult;
import com.linkedin.metadata.entity.UpdateAspectResult;
@ -263,26 +260,4 @@ public class EntityController
}
return responseList;
}
@Override
protected ChangeMCP toUpsertItem(
@Nonnull AspectRetriever aspectRetriever,
Urn entityUrn,
AspectSpec aspectSpec,
Boolean createIfNotExists,
String jsonAspect,
Actor actor)
throws URISyntaxException {
return ChangeItemImpl.builder()
.urn(entityUrn)
.aspectName(aspectSpec.getName())
.changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT)
.auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
.recordTemplate(
GenericRecordUtils.deserializeAspect(
ByteString.copyString(jsonAspect, StandardCharsets.UTF_8),
GenericRecordUtils.JSON,
aspectSpec))
.build(aspectRetriever);
}
}

27
metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java
View File

@ -14,11 +14,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.ByteString;
import com.linkedin.entity.EnvelopedAspect;
import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.aspect.AspectRetriever;
import com.linkedin.metadata.aspect.batch.AspectsBatch;
import com.linkedin.metadata.aspect.batch.BatchItem;
import com.linkedin.metadata.aspect.batch.ChangeMCP;
import com.linkedin.metadata.entity.EntityApiUtils;
import com.linkedin.metadata.entity.IngestResult;
import com.linkedin.metadata.entity.UpdateAspectResult;
@ -351,28 +348,4 @@ public class EntityController
.retrieverContext(opContext.getRetrieverContext().get())
.build();
}
@Override
protected ChangeMCP toUpsertItem(
@Nonnull AspectRetriever aspectRetriever,
Urn entityUrn,
AspectSpec aspectSpec,
Boolean createIfNotExists,
String jsonAspect,
Actor actor)
throws JsonProcessingException {
JsonNode jsonNode = objectMapper.readTree(jsonAspect);
String aspectJson = jsonNode.get("value").toString();
return ChangeItemImpl.builder()
.urn(entityUrn)
.aspectName(aspectSpec.getName())
.changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT)
.auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
.recordTemplate(
GenericRecordUtils.deserializeAspect(
ByteString.copyString(aspectJson, StandardCharsets.UTF_8),
GenericRecordUtils.JSON,
aspectSpec))
.build(aspectRetriever);
}
}