UI: Fix filter patterns in Add Ingestion form (#4490)

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx

@@ -109,6 +109,9 @@ const AddIngestion = ({
     (data?.source.sourceConfig.config as ConfigClass)?.generateSampleData ??
       true
   );
+  const [markDeletedTables, setMarkDeletedTables] = useState(
+    (data?.source.sourceConfig.config as ConfigClass)?.markDeletedTables ?? true
+  );
   const [dashboardFilterPattern, setDashboardFilterPattern] =
     useState<FilterPattern>(
       (data?.source.sourceConfig.config as ConfigClass)
@@ -299,6 +302,7 @@ const AddIngestion = ({
           enableDataProfiler: enableDataProfiler,
           generateSampleData: ingestSampleData,
           includeViews: includeView,
+          markDeletedTables: markDeletedTables,
           schemaFilterPattern: getFilterPatternData(schemaFilterPattern),
           tableFilterPattern: getFilterPatternData(tableFilterPattern),
           chartFilterPattern: getFilterPatternData(chartFilterPattern),
@@ -344,7 +348,12 @@ const AddIngestion = ({
             onSuccessSave?.();
           }
         })
-        .finally(() => setTimeout(() => setSaveState('initial'), 500));
+        .catch(() => {
+          // ignore since error is displayed in toast in the parent promise
+        })
+        .finally(() => {
+          setTimeout(() => setSaveState('initial'), 500);
+        });
     }
   };
 
@@ -421,6 +430,7 @@ const AddIngestion = ({
             handleIncludeView={() => setIncludeView((pre) => !pre)}
             handleIngestSampleData={() => setIngestSampleData((pre) => !pre)}
             handleIngestionName={(val) => setIngestionName(val)}
+            handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)}
             handleQueryLogDuration={(val) => setQueryLogDuration(val)}
             handleResultLimit={(val) => setResultLimit(val)}
             handleShowFilter={handleShowFilter}
@@ -428,6 +438,7 @@ const AddIngestion = ({
             includeView={includeView}
             ingestSampleData={ingestSampleData}
             ingestionName={ingestionName}
+            markDeletedTables={markDeletedTables}
             pipelineType={pipelineType}
             queryLogDuration={queryLogDuration}
             resultLimit={resultLimit}

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx

@@ -61,6 +61,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
   stageFileLocation: '',
   enableDataProfiler: false,
   ingestSampleData: false,
+  markDeletedTables: false,
   showDashboardFilter: false,
   showSchemaFilter: false,
   showTableFilter: false,
@@ -74,6 +75,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
   handleQueryLogDuration: jest.fn(),
   handleResultLimit: jest.fn(),
   handleStageFileLocation: jest.fn(),
+  handleMarkDeletedTables: jest.fn(),
   getIncludeValue: jest.fn(),
   getExcludeValue: jest.fn(),
   handleShowFilter: jest.fn(),
@@ -108,6 +110,6 @@ describe('Test ConfigureIngestion component', () => {
     expect(backButton).toBeInTheDocument();
     expect(nextButton).toBeInTheDocument();
     expect(filterPatternComponents.length).toBe(2);
-    expect(toggleSwitchs.length).toBe(3);
+    expect(toggleSwitchs.length).toBe(4);
   });
 });

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx

@@ -38,6 +38,7 @@ const ConfigureIngestion = ({
   enableDataProfiler,
   ingestSampleData,
   pipelineType,
+  markDeletedTables,
   showDashboardFilter,
   showSchemaFilter,
   showTableFilter,
@@ -50,6 +51,7 @@ const ConfigureIngestion = ({
   getExcludeValue,
   getIncludeValue,
   handleIngestionName,
+  handleMarkDeletedTables,
   handleDescription,
   handleShowFilter,
   handleEnableDataProfiler,
@@ -187,8 +189,8 @@ const ConfigureIngestion = ({
               />
             </div>
             <p className="tw-text-grey-muted tw-mt-3">
-              Slowdown metadata extraction by calculate the metrics and
+              Enable Data Profiler to collect metrics and distribution of data
-              distribution of data in the table
+              in the table. This will however slowdown the metadata extraction.
             </p>
             {getSeparator('')}
           </Field>
@@ -205,6 +207,20 @@ const ConfigureIngestion = ({
             </p>
             {getSeparator('')}
           </Field>
+          <Field>
+            <div className="tw-flex tw-gap-1">
+              <label>Mark Deleted Tables</label>
+              <ToggleSwitchV1
+                checked={markDeletedTables}
+                handleCheck={handleMarkDeletedTables}
+              />
+            </div>
+            <p className="tw-text-grey-muted tw-mt-3">
+              Any deleted tables in the data source will be soft deleted in
+              OpenMetadata
+            </p>
+            {getSeparator('')}
+          </Field>
         </div>
       </>
     );

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts

@@ -59,6 +59,7 @@ export interface ConfigureIngestionProps {
   includeView: boolean;
   enableDataProfiler: boolean;
   ingestSampleData: boolean;
+  markDeletedTables: boolean;
   pipelineType: PipelineType;
   showDashboardFilter: boolean;
   showSchemaFilter: boolean;
@@ -74,6 +75,7 @@ export interface ConfigureIngestionProps {
   handleIncludeView: () => void;
   handleEnableDataProfiler: () => void;
   handleIngestSampleData: () => void;
+  handleMarkDeletedTables: () => void;
   getIncludeValue: (value: string[], type: FilterPatternEnum) => void;
   getExcludeValue: (value: string[], type: FilterPatternEnum) => void;
   handleShowFilter: (value: boolean, type: FilterPatternEnum) => void;

openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx

@@ -810,7 +810,7 @@ export const AddServiceModal: FunctionComponent<Props> = ({
           />
         </Field>
 
-        {/* optional filed for snowflik */}
+        {/* optional field for snowflake */}
         {selectService === DatabaseServiceType.Snowflake && (
           <div className="tw-mt-4 tw-grid tw-grid-cols-2 tw-gap-2">
             <div>
@@ -1526,13 +1526,13 @@ export const AddServiceModal: FunctionComponent<Props> = ({
   };
 
   const previousStepHandler = () => {
-    let increamentCount = 1;
+    let incrementCount = 1;
 
     if (activeStepperStep === 5 && !isIngestionEnable) {
-      increamentCount = 2;
+      incrementCount = 2;
     }
 
-    setActiveStepperStep((pre) => (pre > 1 ? pre - increamentCount : pre));
+    setActiveStepperStep((pre) => (pre > 1 ? pre - incrementCount : pre));
   };
 
   const forwardStepHandler = (activeStep: number) => {
@@ -1598,13 +1598,13 @@ export const AddServiceModal: FunctionComponent<Props> = ({
     }
 
     setActiveStepperStep((pre) => {
-      let increamentCount = 1;
+      let incrementCount = 1;
 
       if (activeStepperStep === 3 && !isIngestionEnable) {
-        increamentCount = 2;
+        incrementCount = 2;
       }
 
-      return pre < steps.length && isValid ? pre + increamentCount : pre;
+      return pre < steps.length && isValid ? pre + incrementCount : pre;
     });
   };
 

openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx

@@ -11,7 +11,7 @@
  *  limitations under the License.
  */
 
-import { findAllByText, findByTestId, render } from '@testing-library/react';
+import { findByTestId, render } from '@testing-library/react';
 import React from 'react';
 import { FilterPatternEnum } from '../../../enums/filterPattern.enum';
 import FilterPattern from './FilterPattern';
@@ -46,14 +46,19 @@ describe('Test FilterPattern component', () => {
       container,
       `${mockFilterPatternProps.type}-filter-pattern-checkbox`
     );
-    const reactSelectMultiInputs = await findAllByText(
+    const includeFilterInput = await findByTestId(
       container,
-      'ReactSelectMultiInput.component'
+      'filter-pattern-includes-table'
+    );
+    const excludeFilterInput = await findByTestId(
+      container,
+      'filter-pattern-excludes-table'
     );
 
     expect(filterPatternContainer).toBeInTheDocument();
     expect(checkbox).toBeInTheDocument();
     expect(fieldContainer).toBeInTheDocument();
-    expect(reactSelectMultiInputs.length).toBe(2);
+    expect(includeFilterInput).toBeInTheDocument();
+    expect(excludeFilterInput).toBeInTheDocument();
   });
 });

openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx

@@ -15,7 +15,6 @@ import { capitalize } from 'lodash';
 import React from 'react';
 import { getSeparator } from '../../../utils/CommonUtils';
 import { Field } from '../../Field/Field';
-import ReactSelectMultiInput from '../react-select-component/ReactSelectMultiInput';
 import { FilterPatternProps } from './filterPattern.interface';
 
 const FilterPattern = ({
@@ -28,6 +27,20 @@ const FilterPattern = ({
   getExcludeValue,
   type,
 }: FilterPatternProps) => {
+  const includeFilterChangeHandler = (
+    event: React.ChangeEvent<HTMLInputElement | HTMLSelectElement>
+  ) => {
+    const value = event.target.value ? event.target.value.split(',') : [];
+    getIncludeValue(value, type);
+  };
+
+  const excludeFilterChangeHandler = (
+    event: React.ChangeEvent<HTMLInputElement | HTMLSelectElement>
+  ) => {
+    const value = event.target.value ? event.target.value.split(',') : [];
+    getExcludeValue(value, type);
+  };
+
   return (
     <div className="tw-mt-4" data-testid="filter-pattern-container">
       <div className="tw-flex tw-items-center">
@@ -49,18 +62,24 @@ const FilterPattern = ({
         <div data-testid="field-container">
           <Field>
             <label className="tw-block tw-form-label">Include:</label>
-            <ReactSelectMultiInput
+            <input
-              getTagValue={(data) => getIncludeValue(data, type)}
+              className="tw-form-inputs tw-relative tw-px-2 tw-py-2"
-              initialData={includePattern}
+              data-testid={`filter-pattern-includes-${type}`}
-              placeholder="Type include filter pattern and hit enter"
+              placeholder="Enter a list of strings/regex patterns as a comma separated value"
+              type="text"
+              value={includePattern}
+              onChange={includeFilterChangeHandler}
             />
           </Field>
           <Field>
             <label className="tw-block tw-form-label">Exclude:</label>
-            <ReactSelectMultiInput
+            <input
-              getTagValue={(data) => getExcludeValue(data, type)}
+              className="tw-form-inputs tw-relative tw-px-2 tw-py-2"
-              initialData={excludePattern}
+              data-testid={`filter-pattern-excludes-${type}`}
-              placeholder="Type exclude filter pattern and hit enter"
+              placeholder="Enter a list of strings/regex patterns as a comma separated value"
+              type="text"
+              value={excludePattern}
+              onChange={excludeFilterChangeHandler}
             />
           </Field>
           {showSeparator && getSeparator('')}

openmetadata-ui/src/main/resources/ui/src/pages/AddIngestionPage/AddIngestionPage.component.tsx

@@ -107,9 +107,9 @@ const AddIngestionPage = () => {
                   err,
                   jsonData['api-error-messages']['create-ingestion-error']
                 );
-                reject();
               });
           }
+          reject();
         });
     });
   };