Fix: Safer default Mark Deleted Tables (#9065)

* Fix: Safer default Mark Deleted Tables

* Fix: Change Based On Comments

* Fix: Change Based On Comments

* Add: Migration Files

Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>

bootstrap/sql/com.mysql.cj.jdbc.Driver/v007__create_db_connection_info.sql

@@ -0,0 +1,3 @@
+-- Remove markDeletedTablesFromFilterOnly 
+UPDATE ingestion_pipeline_entity
+SET json = JSON_REMOVE(json ,'$.sourceConfig.config.markDeletedTablesFromFilterOnly');

bootstrap/sql/org.postgresql.Driver/v007__create_db_connection_info.sql

@@ -0,0 +1,3 @@
+-- Remove markDeletedTablesFromFilterOnly 
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{sourceConfig,config,markDeletedTablesFromFilterOnly}';

ingestion/src/metadata/ingestion/source/database/database_service.py

@@ -533,7 +533,12 @@ class DatabaseServiceSource(
             logger.info(
                 f"Mark Deleted Tables set to True. Processing database [{self.context.database.name.__root__}]"
             )
-            if self.source_config.markDeletedTablesFromFilterOnly:
+            # If markAllDeletedTables is True, all tables Which are not in FilterPattern will be deleted
+            if self.source_config.markAllDeletedTables:
+                yield from self.fetch_all_schema_and_delete_tables()
+
+            # If markAllDeletedTables is False (Default), Only delete tables which are deleted from the datasource
+            else:
                 schema_names_list = self.get_database_schema_names()
                 for schema_name in schema_names_list:
                     schema_fqn = fqn.build(
@@ -545,5 +550,3 @@ class DatabaseServiceSource(
                     )
 
                     yield from self.delete_schema_tables(schema_fqn)
-            else:
-                yield from self.fetch_all_schema_and_delete_tables()

openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json

@@ -23,8 +23,8 @@
       "type": "boolean",
       "default": true
     },
-    "markDeletedTablesFromFilterOnly": {
+    "markAllDeletedTables": {
-      "description": "Optional configuration to mark deleted tables only to the filtered schema",
+      "description": "Optional configuration to mark all deleted tables to the filtered schema",
       "type": "boolean",
       "default": false
     },

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx

@@ -170,15 +170,14 @@ const AddIngestion = ({
         )
       : undefined
   );
-  const [markDeletedTablesFromFilterOnly, setMarkDeletedTablesFromFilterOnly] =
+  const [markAllDeletedTables, setMarkAllDeletedTables] = useState(
-    useState(
+    isDatabaseService
-      isDatabaseService
+      ? Boolean(
-        ? Boolean(
+          (data?.sourceConfig.config as ConfigClass)?.markAllDeletedTables ??
-            (data?.sourceConfig.config as ConfigClass)
+            false
-              ?.markDeletedTablesFromFilterOnly ?? false
+        )
-          )
+      : undefined
-        : undefined
+  );
-    );
   const [includeView, setIncludeView] = useState(
     Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews)
   );
@@ -466,7 +465,7 @@ const AddIngestion = ({
             showTableFilter
           ),
           markDeletedTables,
-          markDeletedTablesFromFilterOnly,
+          markAllDeletedTables,
           ...DatabaseConfigData,
           type: ConfigType.DatabaseMetadata,
         };
@@ -734,10 +733,10 @@ const AddIngestion = ({
             handleIncludeView={() => setIncludeView((pre) => !pre)}
             handleIngestSampleData={() => setIngestSampleData((pre) => !pre)}
             handleIngestionName={(val) => setIngestionName(val)}
-            handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)}
+            handleMarkAllDeletedTables={() =>
-            handleMarkDeletedTablesFromFilterOnly={() =>
+              setMarkAllDeletedTables((pre) => !pre)
-              setMarkDeletedTablesFromFilterOnly((pre) => !pre)
             }
+            handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)}
             handleProfileSample={(val) => setProfileSample(val)}
             handleQueryLogDuration={(val) => setQueryLogDuration(val)}
             handleResultLimit={setResultLimit}
@@ -749,8 +748,8 @@ const AddIngestion = ({
             includeView={includeView}
             ingestSampleData={ingestSampleData}
             ingestionName={ingestionName}
+            markAllDeletedTables={markAllDeletedTables}
             markDeletedTables={markDeletedTables}
-            markDeletedTablesFromFilterOnly={markDeletedTablesFromFilterOnly}
             mlModelFilterPattern={mlModelFilterPattern}
             pipelineFilterPattern={pipelineFilterPattern}
             pipelineType={pipelineType}

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx

@@ -42,7 +42,7 @@ const ConfigureIngestion = ({
   includeView,
   includeTags,
   markDeletedTables,
-  markDeletedTablesFromFilterOnly,
+  markAllDeletedTables,
   serviceCategory,
   pipelineType,
   showDatabaseFilter,
@@ -70,7 +70,7 @@ const ConfigureIngestion = ({
   handleIncludeView,
   handleIncludeTags,
   handleMarkDeletedTables,
-  handleMarkDeletedTablesFromFilterOnly,
+  handleMarkAllDeletedTables,
   handleIngestSampleData,
   handleDatasetServiceName,
   handleQueryLogDuration,
@@ -234,15 +234,15 @@ const ConfigureIngestion = ({
               {getSeparator('')}
             </Field>
           )}
-          {!isNil(markDeletedTablesFromFilterOnly) && (
+          {!isNil(markAllDeletedTables) && (
             <Field>
               <div className="tw-flex tw-gap-1">
-                <label>Mark Deleted Tables from Filter Only</label>
+                <label>Mark All Deleted Tables</label>
                 <ToggleSwitchV1
-                  checked={markDeletedTablesFromFilterOnly}
+                  checked={markAllDeletedTables}
                   handleCheck={() => {
-                    if (handleMarkDeletedTablesFromFilterOnly) {
+                    if (handleMarkAllDeletedTables) {
-                      handleMarkDeletedTablesFromFilterOnly();
+                      handleMarkAllDeletedTables();
                     }
                   }}
                   testId="mark-deleted-filter-only"

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts

@@ -72,7 +72,7 @@ export interface ConfigureIngestionProps {
   includeView: boolean;
   includeTags: boolean;
   markDeletedTables?: boolean;
-  markDeletedTablesFromFilterOnly?: boolean;
+  markAllDeletedTables?: boolean;
   enableDebugLog: boolean;
   profileSample?: number;
   ingestSampleData: boolean;
@@ -98,7 +98,7 @@ export interface ConfigureIngestionProps {
   handleIncludeView: () => void;
   handleIncludeTags: () => void;
   handleMarkDeletedTables?: () => void;
-  handleMarkDeletedTablesFromFilterOnly?: () => void;
+  handleMarkAllDeletedTables?: () => void;
   handleEnableDebugLog: () => void;
   handleIngestSampleData: () => void;
   getIncludeValue: (value: string[], type: FilterPatternEnum) => void;

openmetadata-ui/src/main/resources/ui/src/pages/LogsViewer/mocks/LogsViewer.mock.ts

@@ -23,7 +23,7 @@ export const mockIngestionPipeline = {
     config: {
       type: 'DatabaseMetadata',
       markDeletedTables: true,
-      markDeletedTablesFromFilterOnly: false,
+      markAllDeletedTables: false,
       includeTables: true,
       includeViews: true,
       includeTags: true,