diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 1dc4d2d1e51..75e445b6659 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -109,6 +109,9 @@ const AddIngestion = ({ (data?.source.sourceConfig.config as ConfigClass)?.generateSampleData ?? true ); + const [markDeletedTables, setMarkDeletedTables] = useState( + (data?.source.sourceConfig.config as ConfigClass)?.markDeletedTables ?? true + ); const [dashboardFilterPattern, setDashboardFilterPattern] = useState( (data?.source.sourceConfig.config as ConfigClass) @@ -299,6 +302,7 @@ const AddIngestion = ({ enableDataProfiler: enableDataProfiler, generateSampleData: ingestSampleData, includeViews: includeView, + markDeletedTables: markDeletedTables, schemaFilterPattern: getFilterPatternData(schemaFilterPattern), tableFilterPattern: getFilterPatternData(tableFilterPattern), chartFilterPattern: getFilterPatternData(chartFilterPattern), @@ -344,7 +348,12 @@ const AddIngestion = ({ onSuccessSave?.(); } }) - .finally(() => setTimeout(() => setSaveState('initial'), 500)); + .catch(() => { + // ignore since error is displayed in toast in the parent promise + }) + .finally(() => { + setTimeout(() => setSaveState('initial'), 500); + }); } }; @@ -421,6 +430,7 @@ const AddIngestion = ({ handleIncludeView={() => setIncludeView((pre) => !pre)} handleIngestSampleData={() => setIngestSampleData((pre) => !pre)} handleIngestionName={(val) => setIngestionName(val)} + handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)} handleQueryLogDuration={(val) => setQueryLogDuration(val)} handleResultLimit={(val) => setResultLimit(val)} handleShowFilter={handleShowFilter} @@ -428,6 +438,7 @@ const AddIngestion = ({ includeView={includeView} ingestSampleData={ingestSampleData} ingestionName={ingestionName} + markDeletedTables={markDeletedTables} pipelineType={pipelineType} queryLogDuration={queryLogDuration} resultLimit={resultLimit} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx index 96a11d8feeb..1419398e619 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx @@ -61,6 +61,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = { stageFileLocation: '', enableDataProfiler: false, ingestSampleData: false, + markDeletedTables: false, showDashboardFilter: false, showSchemaFilter: false, showTableFilter: false, @@ -74,6 +75,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = { handleQueryLogDuration: jest.fn(), handleResultLimit: jest.fn(), handleStageFileLocation: jest.fn(), + handleMarkDeletedTables: jest.fn(), getIncludeValue: jest.fn(), getExcludeValue: jest.fn(), handleShowFilter: jest.fn(), @@ -108,6 +110,6 @@ describe('Test ConfigureIngestion component', () => { expect(backButton).toBeInTheDocument(); expect(nextButton).toBeInTheDocument(); expect(filterPatternComponents.length).toBe(2); - expect(toggleSwitchs.length).toBe(3); + expect(toggleSwitchs.length).toBe(4); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index 3851424c0d4..fe24f42a0bb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -38,6 +38,7 @@ const ConfigureIngestion = ({ enableDataProfiler, ingestSampleData, pipelineType, + markDeletedTables, showDashboardFilter, showSchemaFilter, showTableFilter, @@ -50,6 +51,7 @@ const ConfigureIngestion = ({ getExcludeValue, getIncludeValue, handleIngestionName, + handleMarkDeletedTables, handleDescription, handleShowFilter, handleEnableDataProfiler, @@ -187,8 +189,8 @@ const ConfigureIngestion = ({ />

- Slowdown metadata extraction by calculate the metrics and - distribution of data in the table + Enable Data Profiler to collect metrics and distribution of data + in the table. This will however slowdown the metadata extraction.

{getSeparator('')} @@ -205,6 +207,20 @@ const ConfigureIngestion = ({

{getSeparator('')} + +
+ + +
+

+ Any deleted tables in the data source will be soft deleted in + OpenMetadata +

+ {getSeparator('')} +
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts index ffc5cbaf790..ad29bb021da 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts @@ -59,6 +59,7 @@ export interface ConfigureIngestionProps { includeView: boolean; enableDataProfiler: boolean; ingestSampleData: boolean; + markDeletedTables: boolean; pipelineType: PipelineType; showDashboardFilter: boolean; showSchemaFilter: boolean; @@ -74,6 +75,7 @@ export interface ConfigureIngestionProps { handleIncludeView: () => void; handleEnableDataProfiler: () => void; handleIngestSampleData: () => void; + handleMarkDeletedTables: () => void; getIncludeValue: (value: string[], type: FilterPatternEnum) => void; getExcludeValue: (value: string[], type: FilterPatternEnum) => void; handleShowFilter: (value: boolean, type: FilterPatternEnum) => void; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx index 34ca028b5c6..fc3090ee693 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx @@ -810,7 +810,7 @@ export const AddServiceModal: FunctionComponent = ({ /> - {/* optional filed for snowflik */} + {/* optional field for snowflake */} {selectService === DatabaseServiceType.Snowflake && (
@@ -1526,13 +1526,13 @@ export const AddServiceModal: FunctionComponent = ({ }; const previousStepHandler = () => { - let increamentCount = 1; + let incrementCount = 1; if (activeStepperStep === 5 && !isIngestionEnable) { - increamentCount = 2; + incrementCount = 2; } - setActiveStepperStep((pre) => (pre > 1 ? pre - increamentCount : pre)); + setActiveStepperStep((pre) => (pre > 1 ? pre - incrementCount : pre)); }; const forwardStepHandler = (activeStep: number) => { @@ -1598,13 +1598,13 @@ export const AddServiceModal: FunctionComponent = ({ } setActiveStepperStep((pre) => { - let increamentCount = 1; + let incrementCount = 1; if (activeStepperStep === 3 && !isIngestionEnable) { - increamentCount = 2; + incrementCount = 2; } - return pre < steps.length && isValid ? pre + increamentCount : pre; + return pre < steps.length && isValid ? pre + incrementCount : pre; }); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx index 25d2a5aba64..d10e59393a1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.test.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { findAllByText, findByTestId, render } from '@testing-library/react'; +import { findByTestId, render } from '@testing-library/react'; import React from 'react'; import { FilterPatternEnum } from '../../../enums/filterPattern.enum'; import FilterPattern from './FilterPattern'; @@ -46,14 +46,19 @@ describe('Test FilterPattern component', () => { container, `${mockFilterPatternProps.type}-filter-pattern-checkbox` ); - const reactSelectMultiInputs = await findAllByText( + const includeFilterInput = await findByTestId( container, - 'ReactSelectMultiInput.component' + 'filter-pattern-includes-table' + ); + const excludeFilterInput = await findByTestId( + container, + 'filter-pattern-excludes-table' ); expect(filterPatternContainer).toBeInTheDocument(); expect(checkbox).toBeInTheDocument(); expect(fieldContainer).toBeInTheDocument(); - expect(reactSelectMultiInputs.length).toBe(2); + expect(includeFilterInput).toBeInTheDocument(); + expect(excludeFilterInput).toBeInTheDocument(); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx index 02c3f4bf45f..2ee561c3af2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx @@ -15,7 +15,6 @@ import { capitalize } from 'lodash'; import React from 'react'; import { getSeparator } from '../../../utils/CommonUtils'; import { Field } from '../../Field/Field'; -import ReactSelectMultiInput from '../react-select-component/ReactSelectMultiInput'; import { FilterPatternProps } from './filterPattern.interface'; const FilterPattern = ({ @@ -28,6 +27,20 @@ const FilterPattern = ({ getExcludeValue, type, }: FilterPatternProps) => { + const includeFilterChangeHandler = ( + event: React.ChangeEvent + ) => { + const value = event.target.value ? event.target.value.split(',') : []; + getIncludeValue(value, type); + }; + + const excludeFilterChangeHandler = ( + event: React.ChangeEvent + ) => { + const value = event.target.value ? event.target.value.split(',') : []; + getExcludeValue(value, type); + }; + return (
@@ -49,18 +62,24 @@ const FilterPattern = ({
- getIncludeValue(data, type)} - initialData={includePattern} - placeholder="Type include filter pattern and hit enter" + - getExcludeValue(data, type)} - initialData={excludePattern} - placeholder="Type exclude filter pattern and hit enter" + {showSeparator && getSeparator('')} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/AddIngestionPage/AddIngestionPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/AddIngestionPage/AddIngestionPage.component.tsx index ba96dd900f9..6478cc25910 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/AddIngestionPage/AddIngestionPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/AddIngestionPage/AddIngestionPage.component.tsx @@ -107,9 +107,9 @@ const AddIngestionPage = () => { err, jsonData['api-error-messages']['create-ingestion-error'] ); - reject(); }); } + reject(); }); }); };