diff --git a/ingestion/src/metadata/ingestion/models/table_metadata.py b/ingestion/src/metadata/ingestion/models/table_metadata.py index 19c7cf7ed6a..1138e2dfb6c 100644 --- a/ingestion/src/metadata/ingestion/models/table_metadata.py +++ b/ingestion/src/metadata/ingestion/models/table_metadata.py @@ -11,6 +11,8 @@ """ Table related pydantic definitions """ +from typing import Optional + from pydantic import BaseModel from metadata.generated.schema.entity.data.table import Table @@ -20,3 +22,4 @@ class DeleteTable(BaseModel): """Entity Reference of a table to be deleted""" table: Table + mark_deleted_tables: Optional[bool] = False diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index c3c9fa4e2a4..1844098fa6e 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -339,7 +339,12 @@ class MetadataRestSink(Sink[Entity]): def delete_table(self, record: DeleteTable): try: - self.metadata.delete(entity=Table, entity_id=record.table.id) + + self.metadata.delete( + entity=Table, + entity_id=record.table.id, + recursive=record.mark_deleted_tables, + ) logger.debug( f"{record.table.name} doesn't exist in source state, marking it as deleted" ) diff --git a/ingestion/src/metadata/ingestion/source/database/database_service.py b/ingestion/src/metadata/ingestion/source/database/database_service.py index d0f8a07a531..9c7c7c2cd9c 100644 --- a/ingestion/src/metadata/ingestion/source/database/database_service.py +++ b/ingestion/src/metadata/ingestion/source/database/database_service.py @@ -429,7 +429,10 @@ class DatabaseServiceSource( ) for table in database_state: if str(table.fullyQualifiedName.__root__) not in self.database_source_state: - yield DeleteTable(table=table) + yield DeleteTable( + table=table, + mark_deleted_tables=self.source_config.markDeletedTables, + ) def fetch_all_schema_and_delete_tables(self): """ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index efee0b83fb5..5707691969f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -381,9 +381,13 @@ public class TableResource extends EntityResource { @QueryParam("hardDelete") @DefaultValue("false") boolean hardDelete, + @Parameter(description = "Recursively delete this entity and it's children. (Default `false`)") + @QueryParam("recursive") + @DefaultValue("false") + boolean recursive, @Parameter(description = "Id of the table", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) throws IOException { - return delete(uriInfo, securityContext, id, false, hardDelete); + return delete(uriInfo, securityContext, id, recursive, hardDelete); } @DELETE diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json index 090c292d65c..41822f07022 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json @@ -19,7 +19,7 @@ "default": "DatabaseMetadata" }, "markDeletedTables": { - "description": "Optional configuration to soft delete tables in OpenMetadata if the source tables are deleted.", + "description": "Optional configuration to soft delete tables in OpenMetadata if the source tables are deleted. Also, if the table is deleted, all the associated entities like testSuite, lineage, etc., with that table will be deleted", "type": "boolean", "default": true },