diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
index 1dc4d2d1e51..75e445b6659 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
@@ -109,6 +109,9 @@ const AddIngestion = ({
(data?.source.sourceConfig.config as ConfigClass)?.generateSampleData ??
true
);
+ const [markDeletedTables, setMarkDeletedTables] = useState(
+ (data?.source.sourceConfig.config as ConfigClass)?.markDeletedTables ?? true
+ );
const [dashboardFilterPattern, setDashboardFilterPattern] =
useState(
(data?.source.sourceConfig.config as ConfigClass)
@@ -299,6 +302,7 @@ const AddIngestion = ({
enableDataProfiler: enableDataProfiler,
generateSampleData: ingestSampleData,
includeViews: includeView,
+ markDeletedTables: markDeletedTables,
schemaFilterPattern: getFilterPatternData(schemaFilterPattern),
tableFilterPattern: getFilterPatternData(tableFilterPattern),
chartFilterPattern: getFilterPatternData(chartFilterPattern),
@@ -344,7 +348,12 @@ const AddIngestion = ({
onSuccessSave?.();
}
})
- .finally(() => setTimeout(() => setSaveState('initial'), 500));
+ .catch(() => {
+ // ignore since error is displayed in toast in the parent promise
+ })
+ .finally(() => {
+ setTimeout(() => setSaveState('initial'), 500);
+ });
}
};
@@ -421,6 +430,7 @@ const AddIngestion = ({
handleIncludeView={() => setIncludeView((pre) => !pre)}
handleIngestSampleData={() => setIngestSampleData((pre) => !pre)}
handleIngestionName={(val) => setIngestionName(val)}
+ handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)}
handleQueryLogDuration={(val) => setQueryLogDuration(val)}
handleResultLimit={(val) => setResultLimit(val)}
handleShowFilter={handleShowFilter}
@@ -428,6 +438,7 @@ const AddIngestion = ({
includeView={includeView}
ingestSampleData={ingestSampleData}
ingestionName={ingestionName}
+ markDeletedTables={markDeletedTables}
pipelineType={pipelineType}
queryLogDuration={queryLogDuration}
resultLimit={resultLimit}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
index 96a11d8feeb..1419398e619 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
@@ -61,6 +61,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
stageFileLocation: '',
enableDataProfiler: false,
ingestSampleData: false,
+ markDeletedTables: false,
showDashboardFilter: false,
showSchemaFilter: false,
showTableFilter: false,
@@ -74,6 +75,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
handleQueryLogDuration: jest.fn(),
handleResultLimit: jest.fn(),
handleStageFileLocation: jest.fn(),
+ handleMarkDeletedTables: jest.fn(),
getIncludeValue: jest.fn(),
getExcludeValue: jest.fn(),
handleShowFilter: jest.fn(),
@@ -108,6 +110,6 @@ describe('Test ConfigureIngestion component', () => {
expect(backButton).toBeInTheDocument();
expect(nextButton).toBeInTheDocument();
expect(filterPatternComponents.length).toBe(2);
- expect(toggleSwitchs.length).toBe(3);
+ expect(toggleSwitchs.length).toBe(4);
});
});
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
index 3851424c0d4..fe24f42a0bb 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
@@ -38,6 +38,7 @@ const ConfigureIngestion = ({
enableDataProfiler,
ingestSampleData,
pipelineType,
+ markDeletedTables,
showDashboardFilter,
showSchemaFilter,
showTableFilter,
@@ -50,6 +51,7 @@ const ConfigureIngestion = ({
getExcludeValue,
getIncludeValue,
handleIngestionName,
+ handleMarkDeletedTables,
handleDescription,
handleShowFilter,
handleEnableDataProfiler,
@@ -187,8 +189,8 @@ const ConfigureIngestion = ({
/>
- Slowdown metadata extraction by calculate the metrics and
- distribution of data in the table
+ Enable Data Profiler to collect metrics and distribution of data
+ in the table. This will however slowdown the metadata extraction.
{getSeparator('')}
@@ -205,6 +207,20 @@ const ConfigureIngestion = ({
{getSeparator('')}
+
+
+ Mark Deleted Tables
+
+
+
+ Any deleted tables in the data source will be soft deleted in
+ OpenMetadata
+
+ {getSeparator('')}
+
>
);
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
index ffc5cbaf790..ad29bb021da 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
@@ -59,6 +59,7 @@ export interface ConfigureIngestionProps {
includeView: boolean;
enableDataProfiler: boolean;
ingestSampleData: boolean;
+ markDeletedTables: boolean;
pipelineType: PipelineType;
showDashboardFilter: boolean;
showSchemaFilter: boolean;
@@ -74,6 +75,7 @@ export interface ConfigureIngestionProps {
handleIncludeView: () => void;
handleEnableDataProfiler: () => void;
handleIngestSampleData: () => void;
+ handleMarkDeletedTables: () => void;
getIncludeValue: (value: string[], type: FilterPatternEnum) => void;
getExcludeValue: (value: string[], type: FilterPatternEnum) => void;
handleShowFilter: (value: boolean, type: FilterPatternEnum) => void;
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx
index 34ca028b5c6..fc3090ee693 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx
@@ -810,7 +810,7 @@ export const AddServiceModal: FunctionComponent = ({
/>
- {/* optional filed for snowflik */}
+ {/* optional field for snowflake */}
{selectService === DatabaseServiceType.Snowflake && (
@@ -1526,13 +1526,13 @@ export const AddServiceModal: FunctionComponent
= ({
};
const previousStepHandler = () => {
- let increamentCount = 1;
+ let incrementCount = 1;
if (activeStepperStep === 5 && !isIngestionEnable) {
- increamentCount = 2;
+ incrementCount = 2;
}
- setActiveStepperStep((pre) => (pre > 1 ? pre - increamentCount : pre));
+ setActiveStepperStep((pre) => (pre > 1 ? pre - incrementCount : pre));
};
const forwardStepHandler = (activeStep: number) => {
@@ -1598,13 +1598,13 @@ export const AddServiceModal: FunctionComponent = ({
}
setActiveStepperStep((pre) => {
- let increamentCount = 1;
+ let incrementCount = 1;
if (activeStepperStep === 3 && !isIngestionEnable) {
- increamentCount = 2;
+ incrementCount = 2;
}
- return pre < steps.length && isValid ? pre + increamentCount : pre;
+ return pre < steps.length && isValid ? pre + incrementCount : pre;
});
};
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx
index 25d2a5aba64..d10e59393a1 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx
@@ -11,7 +11,7 @@
* limitations under the License.
*/
-import { findAllByText, findByTestId, render } from '@testing-library/react';
+import { findByTestId, render } from '@testing-library/react';
import React from 'react';
import { FilterPatternEnum } from '../../../enums/filterPattern.enum';
import FilterPattern from './FilterPattern';
@@ -46,14 +46,19 @@ describe('Test FilterPattern component', () => {
container,
`${mockFilterPatternProps.type}-filter-pattern-checkbox`
);
- const reactSelectMultiInputs = await findAllByText(
+ const includeFilterInput = await findByTestId(
container,
- 'ReactSelectMultiInput.component'
+ 'filter-pattern-includes-table'
+ );
+ const excludeFilterInput = await findByTestId(
+ container,
+ 'filter-pattern-excludes-table'
);
expect(filterPatternContainer).toBeInTheDocument();
expect(checkbox).toBeInTheDocument();
expect(fieldContainer).toBeInTheDocument();
- expect(reactSelectMultiInputs.length).toBe(2);
+ expect(includeFilterInput).toBeInTheDocument();
+ expect(excludeFilterInput).toBeInTheDocument();
});
});
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx
index 02c3f4bf45f..2ee561c3af2 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx
@@ -15,7 +15,6 @@ import { capitalize } from 'lodash';
import React from 'react';
import { getSeparator } from '../../../utils/CommonUtils';
import { Field } from '../../Field/Field';
-import ReactSelectMultiInput from '../react-select-component/ReactSelectMultiInput';
import { FilterPatternProps } from './filterPattern.interface';
const FilterPattern = ({
@@ -28,6 +27,20 @@ const FilterPattern = ({
getExcludeValue,
type,
}: FilterPatternProps) => {
+ const includeFilterChangeHandler = (
+ event: React.ChangeEvent
+ ) => {
+ const value = event.target.value ? event.target.value.split(',') : [];
+ getIncludeValue(value, type);
+ };
+
+ const excludeFilterChangeHandler = (
+ event: React.ChangeEvent
+ ) => {
+ const value = event.target.value ? event.target.value.split(',') : [];
+ getExcludeValue(value, type);
+ };
+
return (