diff --git a/ingestion/src/metadata/ingestion/models/table_metadata.py b/ingestion/src/metadata/ingestion/models/table_metadata.py index 19c7cf7ed6a..1138e2dfb6c 100644 --- a/ingestion/src/metadata/ingestion/models/table_metadata.py +++ b/ingestion/src/metadata/ingestion/models/table_metadata.py @@ -11,6 +11,8 @@ """ Table related pydantic definitions """ +from typing import Optional + from pydantic import BaseModel from metadata.generated.schema.entity.data.table import Table @@ -20,3 +22,4 @@ class DeleteTable(BaseModel): """Entity Reference of a table to be deleted""" table: Table + mark_deleted_tables: Optional[bool] = False diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index c3c9fa4e2a4..1844098fa6e 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -339,7 +339,12 @@ class MetadataRestSink(Sink[Entity]): def delete_table(self, record: DeleteTable): try: - self.metadata.delete(entity=Table, entity_id=record.table.id) + + self.metadata.delete( + entity=Table, + entity_id=record.table.id, + recursive=record.mark_deleted_tables, + ) logger.debug( f"{record.table.name} doesn't exist in source state, marking it as deleted" ) diff --git a/ingestion/src/metadata/ingestion/source/database/database_service.py b/ingestion/src/metadata/ingestion/source/database/database_service.py index d0f8a07a531..9c7c7c2cd9c 100644 --- a/ingestion/src/metadata/ingestion/source/database/database_service.py +++ b/ingestion/src/metadata/ingestion/source/database/database_service.py @@ -429,7 +429,10 @@ class DatabaseServiceSource( ) for table in database_state: if str(table.fullyQualifiedName.__root__) not in self.database_source_state: - yield DeleteTable(table=table) + yield DeleteTable( + table=table, + mark_deleted_tables=self.source_config.markDeletedTables, + ) def fetch_all_schema_and_delete_tables(self): """ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index efee0b83fb5..5707691969f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -381,9 +381,13 @@ public class TableResource extends EntityResource