mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-28 18:38:17 +00:00
e8f6c4cabd
* Changes rollback behaviour to apply soft deletes by default Summary: Addresses feature request: Flag in delete command to only delete aspects touched by an ingestion run; add flag to nuke everything by modifying the default behaviour of a rollback operation which will not by default delete an entity if a keyAspect is being rolled-back. Instead the key aspect is kept and a StatusAspect is upserted with removed=true, effectively making a soft delete. Another PR will follow to perform garbage collection on these soft deleted entities. To keep old behaviour, a new parameter to the cli ingest rollback endpoint: --hard-delete was added. * Adds restli specs * Fixes deleteAspect endpoint & adds support for nested transactions * Enable regression test & fix docker-compose for local development * Add generated quickstart * Fix quickstart generation script * Adds missing var env to docker-compose-without-neo4j * Sets status removed=true when ingesting resources * Adds soft deletes for ElasticSearch + soft delete flags across ingestion sub-commands * Makes elastic search consistent * Update tests with new behaviour * apply review comments * apply review comment * Forces Elastic search to add documents with status removed false when ingesting * Reset gradle properties to default * Fix tests
143 lines
4.8 KiB
JSON
143 lines
4.8 KiB
JSON
[
|
|
{
|
|
"auditHeader": null,
|
|
"proposedSnapshot": {
|
|
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
|
"urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,test-rollback,PROD)",
|
|
"aspects": [
|
|
{
|
|
"com.linkedin.pegasus2avro.common.BrowsePaths": {
|
|
"paths": ["/prod/kafka/SampleKafkaDataset"]
|
|
}
|
|
},
|
|
{
|
|
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
|
"description": null,
|
|
"uri": null,
|
|
"tags": [],
|
|
"customProperties": {
|
|
"prop1": "fakeprop",
|
|
"prop2": "pikachu"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"com.linkedin.pegasus2avro.common.Ownership": {
|
|
"owners": [
|
|
{
|
|
"owner": "urn:li:corpuser:jdoe",
|
|
"type": "DATAOWNER",
|
|
"source": null
|
|
},
|
|
{
|
|
"owner": "urn:li:corpuser:datahub",
|
|
"type": "DATAOWNER",
|
|
"source": null
|
|
}
|
|
],
|
|
"lastModified": {
|
|
"time": 1581407189000,
|
|
"actor": "urn:li:corpuser:jdoe",
|
|
"impersonator": null
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"com.linkedin.pegasus2avro.common.InstitutionalMemory": {
|
|
"elements": [
|
|
{
|
|
"url": "https://www.linkedin.com",
|
|
"description": "Sample doc",
|
|
"createStamp": {
|
|
"time": 1581407189000,
|
|
"actor": "urn:li:corpuser:jdoe",
|
|
"impersonator": null
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
|
"schemaName": "SampleKafkaSchema",
|
|
"platform": "urn:li:dataPlatform:kafka",
|
|
"version": 0,
|
|
"created": {
|
|
"time": 1581407189000,
|
|
"actor": "urn:li:corpuser:jdoe",
|
|
"impersonator": null
|
|
},
|
|
"lastModified": {
|
|
"time": 1581407189000,
|
|
"actor": "urn:li:corpuser:jdoe",
|
|
"impersonator": null
|
|
},
|
|
"deleted": null,
|
|
"dataset": null,
|
|
"cluster": null,
|
|
"hash": "",
|
|
"platformSchema": {
|
|
"com.linkedin.pegasus2avro.schema.KafkaSchema": {
|
|
"documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"
|
|
}
|
|
},
|
|
"fields": [
|
|
{
|
|
"fieldPath": "[version=2.0].[type=boolean].field_foo_2",
|
|
"jsonPath": null,
|
|
"nullable": false,
|
|
"description": {
|
|
"string": "Foo field description"
|
|
},
|
|
"type": {
|
|
"type": {
|
|
"com.linkedin.pegasus2avro.schema.BooleanType": {}
|
|
}
|
|
},
|
|
"nativeDataType": "varchar(100)",
|
|
"globalTags": {
|
|
"tags": [{ "tag": "urn:li:tag:NeedsDocumentation" }]
|
|
},
|
|
"recursive": false
|
|
},
|
|
{
|
|
"fieldPath": "[version=2.0].[type=boolean].field_bar",
|
|
"jsonPath": null,
|
|
"nullable": false,
|
|
"description": {
|
|
"string": "Bar field description"
|
|
},
|
|
"type": {
|
|
"type": {
|
|
"com.linkedin.pegasus2avro.schema.BooleanType": {}
|
|
}
|
|
},
|
|
"nativeDataType": "boolean",
|
|
"recursive": false
|
|
},
|
|
{
|
|
"fieldPath": "[version=2.0].[key=True].[type=int].id",
|
|
"jsonPath": null,
|
|
"nullable": false,
|
|
"description": {
|
|
"string": "Id specifying which partition the message should go to"
|
|
},
|
|
"type": {
|
|
"type": {
|
|
"com.linkedin.pegasus2avro.schema.BooleanType": {}
|
|
}
|
|
},
|
|
"nativeDataType": "boolean",
|
|
"recursive": false
|
|
}
|
|
],
|
|
"primaryKeys": null,
|
|
"foreignKeysSpecs": null
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"proposedDelta": null
|
|
}
|
|
] |