From 7575a3f55c1c05e94db7681b9cce6bf15c6a4d6c Mon Sep 17 00:00:00 2001 From: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> Date: Tue, 10 Jan 2023 18:12:12 +0530 Subject: [PATCH] fix(ui): sonar-cloud code-smells and code optimizations (#9634) * fix(ui): sonar-cloud code-smells and code optimizations * fix errors * fix other pending code-smells --- .../AddIngestion/AddIngestion.component.tsx | 601 +++++++----------- .../Steps/ConfigureIngestion.test.tsx | 143 ++--- .../AddIngestion/Steps/ConfigureIngestion.tsx | 289 +++++---- .../Steps/ScheduleInterval.test.tsx | 2 +- .../AddIngestion/Steps/ScheduleInterval.tsx | 16 +- .../AddIngestion/addIngestion.interface.ts | 134 ++-- .../SliderWithInput/SliderWithInput.tsx | 6 +- .../Component/ProfilerSettingsModal.tsx | 200 ++++-- .../TableProfiler/TableProfiler.interface.ts | 20 +- .../DBTConfigForm.interface.ts | 15 +- .../DBTConfigFormBuilder.test.tsx | 102 ++- .../DBTConfigFormBuilder.tsx | 121 ++-- .../resources/ui/src/utils/CommonUtils.tsx | 29 + 13 files changed, 912 insertions(+), 766 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 482d4955a1f..5fdb54da019 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -13,7 +13,13 @@ import { isEmpty, isUndefined, omit, trim } from 'lodash'; import { LoadingState } from 'Models'; -import React, { useMemo, useState } from 'react'; +import React, { + Reducer, + useCallback, + useMemo, + useReducer, + useState, +} from 'react'; import { useTranslation } from 'react-i18next'; import { INITIAL_FILTER_PATTERN, @@ -41,355 +47,200 @@ import { } from '../../generated/metadataIngestion/dbtPipeline'; import { getCurrentUserId, + getFilterTypes, getIngestionFrequency, + reducerWithoutAction, } from '../../utils/CommonUtils'; import { getSourceTypeFromConfig } from '../../utils/DBTConfigFormUtil'; import { escapeBackwardSlashChar } from '../../utils/JSONSchemaFormUtils'; import { getIngestionName } from '../../utils/ServiceUtils'; import DBTConfigFormBuilder from '../common/DBTConfigFormBuilder/DBTConfigFormBuilder'; -import { - DBT_SOURCES, - GCS_CONFIG, -} from '../common/DBTConfigFormBuilder/DBTFormEnum'; +import { DBT_SOURCES } from '../common/DBTConfigFormBuilder/DBTFormEnum'; import SuccessScreen from '../common/success-screen/SuccessScreen'; import IngestionStepper from '../IngestionStepper/IngestionStepper.component'; import DeployIngestionLoaderModal from '../Modals/DeployIngestionLoaderModal/DeployIngestionLoaderModal'; -import { AddIngestionProps, ModifiedDbtConfig } from './addIngestion.interface'; +import { + AddIngestionProps, + AddIngestionState, + ModifiedDbtConfig, +} from './addIngestion.interface'; import ConfigureIngestion from './Steps/ConfigureIngestion'; import MetadataToESConfigForm from './Steps/MetadataToESConfigForm/MetadataToESConfigForm'; import ScheduleInterval from './Steps/ScheduleInterval'; const AddIngestion = ({ activeIngestionStep, - heading, - status, - pipelineType, data, - serviceData, - serviceCategory, - showSuccessScreen = true, + handleCancelClick, + handleViewServiceClick, + heading, + ingestionAction = '', ingestionProgress = 0, isIngestionCreated = false, isIngestionDeployed = false, - ingestionAction = '', - showDeployButton, - setActiveIngestionStep, - onIngestionDeploy, - onUpdateIngestion, - onSuccessSave, onAddIngestionSave, - handleCancelClick, - handleViewServiceClick, + onIngestionDeploy, + onSuccessSave, + onUpdateIngestion, + pipelineType, + serviceCategory, + serviceData, + setActiveIngestionStep, + showDeployButton, + showSuccessScreen = true, + status, }: AddIngestionProps) => { const { t } = useTranslation(); - const isDatabaseService = useMemo(() => { - return serviceCategory === ServiceCategory.DATABASE_SERVICES; - }, [serviceCategory]); - const isServiceTypeOpenMetadata = useMemo(() => { - return serviceData.serviceType === MetadataServiceType.OpenMetadata; + console.log('data:', data); + const { sourceConfig, sourceConfigType } = useMemo( + () => ({ + sourceConfig: data?.sourceConfig.config as ConfigClass, + sourceConfigType: (data?.sourceConfig.config as ConfigClass)?.type, + }), + [] + ); + + const { + configData, + usageIngestionType, + lineageIngestionType, + profilerIngestionType, + } = useMemo(() => { + return { + configData: (data?.sourceConfig.config as DbtPipelineClass) + ?.dbtConfigSource, + usageIngestionType: sourceConfigType ?? ConfigType.DatabaseUsage, + lineageIngestionType: sourceConfigType ?? ConfigType.DatabaseLineage, + profilerIngestionType: sourceConfigType ?? ConfigType.Profiler, + }; + }, [data]); + + const { isDatabaseService, isServiceTypeOpenMetadata } = useMemo(() => { + return { + isDatabaseService: serviceCategory === ServiceCategory.DATABASE_SERVICES, + isServiceTypeOpenMetadata: + serviceData.serviceType === MetadataServiceType.OpenMetadata, + }; }, [serviceCategory]); + const showDBTConfig = useMemo(() => { return isDatabaseService && pipelineType === PipelineType.Dbt; }, [isDatabaseService, pipelineType]); - const [saveState, setSaveState] = useState('initial'); - const [showDeployModal, setShowDeployModal] = useState(false); - const [ingestionName, setIngestionName] = useState( - data?.name ?? getIngestionName(serviceData.name, pipelineType) - ); - const [ingestSampleData, setIngestSampleData] = useState( - (data?.sourceConfig.config as ConfigClass)?.generateSampleData ?? true - ); - const [useFqnFilter, setUseFqnFilter] = useState( - (data?.sourceConfig.config as ConfigClass)?.useFqnForFiltering ?? false - ); - const [databaseServiceNames, setDatabaseServiceNames] = useState( - (data?.sourceConfig.config as ConfigClass)?.dbServiceNames ?? [] - ); - const [description, setDescription] = useState(data?.description ?? ''); - const [repeatFrequency, setRepeatFrequency] = useState( - data?.airflowConfig.scheduleInterval ?? getIngestionFrequency(pipelineType) - ); - const [showDashboardFilter, setShowDashboardFilter] = useState( - !isUndefined( - (data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern - ) - ); - const [showDatabaseFilter, setShowDatabaseFilter] = useState( - !isUndefined( - (data?.sourceConfig.config as ConfigClass)?.databaseFilterPattern - ) - ); - const [showSchemaFilter, setShowSchemaFilter] = useState( - !isUndefined( - (data?.sourceConfig.config as ConfigClass)?.schemaFilterPattern - ) - ); - const [showTableFilter, setShowTableFilter] = useState( - !isUndefined((data?.sourceConfig.config as ConfigClass)?.tableFilterPattern) - ); - const [showTopicFilter, setShowTopicFilter] = useState( - !isUndefined((data?.sourceConfig.config as ConfigClass)?.topicFilterPattern) - ); - const [showChartFilter, setShowChartFilter] = useState( - !isUndefined((data?.sourceConfig.config as ConfigClass)?.chartFilterPattern) - ); - const [showPipelineFilter, setShowPipelineFilter] = useState( - !isUndefined( - (data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern - ) - ); - const [showMlModelFilter, setShowMlModelFilter] = useState( - !isUndefined( - (data?.sourceConfig.config as ConfigClass)?.mlModelFilterPattern - ) - ); - const configData = useMemo( - () => (data?.sourceConfig.config as DbtPipelineClass)?.dbtConfigSource, - [data] - ); - const [dbtConfigSource, setDbtConfigSource] = useState< - ModifiedDbtConfig | undefined - >(configData as DbtConfig); - const sourceTypeData = useMemo( () => getSourceTypeFromConfig(configData as DbtConfig | undefined), [configData] ); - const [dbtConfigSourceType, setDbtConfigSourceType] = useState( - sourceTypeData.sourceType + + const initialState: AddIngestionState = useMemo( + () => ({ + saveState: 'initial', + showDeployModal: false, + ingestionName: + data?.name ?? getIngestionName(serviceData.name, pipelineType), + ingestSampleData: sourceConfig?.generateSampleData ?? true, + useFqnFilter: sourceConfig?.useFqnForFiltering ?? false, + databaseServiceNames: sourceConfig?.dbServiceNames ?? [], + description: data?.description ?? '', + repeatFrequency: + data?.airflowConfig.scheduleInterval ?? + getIngestionFrequency(pipelineType), + showDashboardFilter: !isUndefined(sourceConfig?.dashboardFilterPattern), + showDatabaseFilter: !isUndefined(sourceConfig?.databaseFilterPattern), + showSchemaFilter: !isUndefined(sourceConfig?.schemaFilterPattern), + showTableFilter: !isUndefined(sourceConfig?.tableFilterPattern), + showTopicFilter: !isUndefined(sourceConfig?.topicFilterPattern), + showChartFilter: !isUndefined(sourceConfig?.chartFilterPattern), + showPipelineFilter: !isUndefined(sourceConfig?.pipelineFilterPattern), + showMlModelFilter: !isUndefined(sourceConfig?.mlModelFilterPattern), + dbtConfigSource: configData as ModifiedDbtConfig, + gcsConfigType: showDBTConfig ? sourceTypeData.gcsType : undefined, + chartFilterPattern: + sourceConfig?.chartFilterPattern ?? INITIAL_FILTER_PATTERN, + dbtConfigSourceType: sourceTypeData.sourceType || DBT_SOURCES.local, + markDeletedTables: isDatabaseService + ? Boolean(sourceConfig?.markDeletedTables ?? true) + : undefined, + dashboardFilterPattern: + sourceConfig?.dashboardFilterPattern ?? INITIAL_FILTER_PATTERN, + databaseFilterPattern: + sourceConfig?.databaseFilterPattern ?? INITIAL_FILTER_PATTERN, + markAllDeletedTables: isDatabaseService + ? Boolean(sourceConfig?.markAllDeletedTables ?? false) + : undefined, + includeView: Boolean(sourceConfig?.includeViews), + includeTags: Boolean(sourceConfig?.includeTags), + includeLineage: Boolean(sourceConfig?.includeLineage ?? true), + enableDebugLog: data?.loggerLevel === LogLevels.Debug, + profileSample: sourceConfig?.profileSample, + profileSampleType: + sourceConfig?.profileSampleType || ProfileSampleType.Percentage, + threadCount: sourceConfig?.threadCount ?? 5, + timeoutSeconds: sourceConfig?.timeoutSeconds ?? 43200, + schemaFilterPattern: + sourceConfig?.schemaFilterPattern ?? INITIAL_FILTER_PATTERN, + tableFilterPattern: + sourceConfig?.tableFilterPattern ?? INITIAL_FILTER_PATTERN, + topicFilterPattern: + sourceConfig?.topicFilterPattern ?? INITIAL_FILTER_PATTERN, + pipelineFilterPattern: + sourceConfig?.pipelineFilterPattern ?? INITIAL_FILTER_PATTERN, + mlModelFilterPattern: + sourceConfig?.mlModelFilterPattern ?? INITIAL_FILTER_PATTERN, + queryLogDuration: sourceConfig?.queryLogDuration ?? 1, + stageFileLocation: sourceConfig?.stageFileLocation ?? '/tmp/query_log', + resultLimit: sourceConfig?.resultLimit ?? 1000, + metadataToESConfig: undefined, + }), + [] ); - const [gcsConfigType, setGcsConfigType] = useState( - showDBTConfig ? sourceTypeData.gcsType : undefined - ); - const [markDeletedTables, setMarkDeletedTables] = useState( - isDatabaseService - ? Boolean( - (data?.sourceConfig.config as ConfigClass)?.markDeletedTables ?? true - ) - : undefined - ); - const [markAllDeletedTables, setMarkAllDeletedTables] = useState( - isDatabaseService - ? Boolean( - (data?.sourceConfig.config as ConfigClass)?.markAllDeletedTables ?? - false - ) - : undefined - ); - const [includeView, setIncludeView] = useState( - Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews) - ); - const [includeTag, setIncludeTags] = useState( - Boolean((data?.sourceConfig.config as ConfigClass)?.includeTags) - ); - const [includeLineage, setIncludeLineage] = useState( - Boolean((data?.sourceConfig.config as ConfigClass)?.includeLineage ?? true) - ); - const [enableDebugLog, setEnableDebugLog] = useState( - data?.loggerLevel === LogLevels.Debug - ); - const [profileSample, setProfileSample] = useState( - (data?.sourceConfig.config as ConfigClass)?.profileSample - ); - const [profileSampleType, setProfileSampleType] = useState( - (data?.sourceConfig.config as ConfigClass)?.profileSampleType || - ProfileSampleType.Percentage - ); - const [threadCount, setThreadCount] = useState( - (data?.sourceConfig.config as ConfigClass)?.threadCount ?? 5 - ); - const [timeoutSeconds, setTimeoutSeconds] = useState( - (data?.sourceConfig.config as ConfigClass)?.timeoutSeconds ?? 43200 - ); - const [dashboardFilterPattern, setDashboardFilterPattern] = - useState( - (data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern ?? - INITIAL_FILTER_PATTERN - ); - const [databaseFilterPattern, setDatabaseFilterPattern] = - useState( - (data?.sourceConfig.config as ConfigClass)?.databaseFilterPattern ?? - INITIAL_FILTER_PATTERN - ); - const [schemaFilterPattern, setSchemaFilterPattern] = useState( - (data?.sourceConfig.config as ConfigClass)?.schemaFilterPattern ?? - INITIAL_FILTER_PATTERN - ); - const [tableFilterPattern, setTableFilterPattern] = useState( - (data?.sourceConfig.config as ConfigClass)?.tableFilterPattern ?? - INITIAL_FILTER_PATTERN - ); - const [topicFilterPattern, setTopicFilterPattern] = useState( - (data?.sourceConfig.config as ConfigClass)?.topicFilterPattern ?? - INITIAL_FILTER_PATTERN - ); - const [chartFilterPattern, setChartFilterPattern] = useState( - (data?.sourceConfig.config as ConfigClass)?.chartFilterPattern ?? - INITIAL_FILTER_PATTERN - ); - const [pipelineFilterPattern, setPipelineFilterPattern] = - useState( - (data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern ?? - INITIAL_FILTER_PATTERN - ); + const [state, dispatch] = useReducer< + Reducer> + >(reducerWithoutAction, initialState); - const [mlModelFilterPattern, setMlModelFilterPattern] = - useState( - (data?.sourceConfig.config as ConfigClass)?.mlModelFilterPattern ?? - INITIAL_FILTER_PATTERN - ); + const [saveState, setSaveState] = useState('initial'); + const [showDeployModal, setShowDeployModal] = useState(false); - const [queryLogDuration, setQueryLogDuration] = useState( - (data?.sourceConfig.config as ConfigClass)?.queryLogDuration ?? 1 + const handleStateChange = useCallback( + (newState: Partial) => { + dispatch(newState); + }, + [] ); - const [stageFileLocation, setStageFileLocation] = useState( - (data?.sourceConfig.config as ConfigClass)?.stageFileLocation ?? - '/tmp/query_log' - ); - const [resultLimit, setResultLimit] = useState( - (data?.sourceConfig.config as ConfigClass)?.resultLimit ?? 1000 - ); - const [metadataToESConfig, SetMetadataToESConfig] = useState(); - - const usageIngestionType = useMemo(() => { - return ( - (data?.sourceConfig.config as ConfigClass)?.type ?? - ConfigType.DatabaseUsage - ); - }, [data]); - const lineageIngestionType = useMemo(() => { - return ( - (data?.sourceConfig.config as ConfigClass)?.type ?? - ConfigType.DatabaseLineage - ); - }, [data]); - const profilerIngestionType = useMemo(() => { - return ( - (data?.sourceConfig.config as ConfigClass)?.type ?? ConfigType.Profiler - ); - }, [data]); const handleMetadataToESConfig = (data: ConfigClass) => { - SetMetadataToESConfig(data); + handleStateChange({ + metadataToESConfig: data, + }); }; const getIncludeValue = (value: Array, type: FilterPatternEnum) => { - switch (type) { - case FilterPatternEnum.DASHBOARD: - setDashboardFilterPattern({ - ...dashboardFilterPattern, - includes: value, - }); + const pattern = getFilterTypes(type); - break; - case FilterPatternEnum.DATABASE: - setDatabaseFilterPattern({ ...databaseFilterPattern, includes: value }); - - break; - case FilterPatternEnum.SCHEMA: - setSchemaFilterPattern({ ...schemaFilterPattern, includes: value }); - - break; - case FilterPatternEnum.TABLE: - setTableFilterPattern({ ...tableFilterPattern, includes: value }); - - break; - case FilterPatternEnum.TOPIC: - setTopicFilterPattern({ ...topicFilterPattern, includes: value }); - - break; - case FilterPatternEnum.CHART: - setChartFilterPattern({ ...chartFilterPattern, includes: value }); - - break; - case FilterPatternEnum.PIPELINE: - setPipelineFilterPattern({ ...pipelineFilterPattern, includes: value }); - - break; - case FilterPatternEnum.MLMODEL: - setMlModelFilterPattern({ ...mlModelFilterPattern, includes: value }); - - break; - } + return handleStateChange({ + [pattern]: { + ...(state[pattern] as AddIngestionState), + includes: value, + }, + }); }; const getExcludeValue = (value: Array, type: FilterPatternEnum) => { - switch (type) { - case FilterPatternEnum.DASHBOARD: - setDashboardFilterPattern({ - ...dashboardFilterPattern, - excludes: value, - }); + const pattern = getFilterTypes(type); - break; - case FilterPatternEnum.DATABASE: - setDatabaseFilterPattern({ ...databaseFilterPattern, excludes: value }); - - break; - case FilterPatternEnum.SCHEMA: - setSchemaFilterPattern({ ...schemaFilterPattern, excludes: value }); - - break; - case FilterPatternEnum.TABLE: - setTableFilterPattern({ ...tableFilterPattern, excludes: value }); - - break; - case FilterPatternEnum.TOPIC: - setTopicFilterPattern({ ...topicFilterPattern, excludes: value }); - - break; - case FilterPatternEnum.CHART: - setChartFilterPattern({ ...chartFilterPattern, excludes: value }); - - break; - case FilterPatternEnum.PIPELINE: - setPipelineFilterPattern({ ...pipelineFilterPattern, excludes: value }); - - break; - case FilterPatternEnum.MLMODEL: - setMlModelFilterPattern({ ...mlModelFilterPattern, excludes: value }); - - break; - } + return handleStateChange({ + [pattern]: { + ...(state[pattern] as AddIngestionState), + excludes: value, + }, + }); }; - const handleShowFilter = (value: boolean, type: FilterPatternEnum) => { - switch (type) { - case FilterPatternEnum.DASHBOARD: - setShowDashboardFilter(value); - - break; - case FilterPatternEnum.DATABASE: - setShowDatabaseFilter(value); - - break; - case FilterPatternEnum.SCHEMA: - setShowSchemaFilter(value); - - break; - case FilterPatternEnum.TABLE: - setShowTableFilter(value); - - break; - case FilterPatternEnum.TOPIC: - setShowTopicFilter(value); - - break; - case FilterPatternEnum.CHART: - setShowChartFilter(value); - - break; - case FilterPatternEnum.PIPELINE: - setShowPipelineFilter(value); - - break; - case FilterPatternEnum.MLMODEL: - setShowMlModelFilter(value); - - break; - } - }; + // It takes a boolean and a string, and returns a function that takes an object and returns a new + const handleShowFilter = (value: boolean, showFilter: string) => + handleStateChange({ + [showFilter]: value, + }); const handleNext = () => { let nextStep; @@ -447,12 +298,39 @@ const AddIngestion = ({ }; const getMetadataIngestionFields = () => { + const { + chartFilterPattern, + dashboardFilterPattern, + databaseFilterPattern, + databaseServiceNames, + includeLineage, + includeTags, + includeView, + ingestSampleData, + markAllDeletedTables, + markDeletedTables, + mlModelFilterPattern, + pipelineFilterPattern, + schemaFilterPattern, + showChartFilter, + showDashboardFilter, + showDatabaseFilter, + showMlModelFilter, + showPipelineFilter, + showSchemaFilter, + showTableFilter, + showTopicFilter, + tableFilterPattern, + topicFilterPattern, + useFqnFilter, + } = state; + switch (serviceCategory) { case ServiceCategory.DATABASE_SERVICES: { return { useFqnForFiltering: useFqnFilter, includeViews: includeView, - includeTags: includeTag, + includeTags: includeTags, databaseFilterPattern: getFilterPatternData( databaseFilterPattern, showDatabaseFilter @@ -465,8 +343,8 @@ const AddIngestion = ({ tableFilterPattern, showTableFilter ), - markDeletedTables, - markAllDeletedTables, + markDeletedTables: markDeletedTables, + markAllDeletedTables: markAllDeletedTables, type: ConfigType.DatabaseMetadata, }; } @@ -520,19 +398,37 @@ const AddIngestion = ({ }; const getConfigData = (type: PipelineType): ConfigClass => { + const { + databaseFilterPattern, + dbtConfigSource, + ingestSampleData, + metadataToESConfig, + profileSample, + profileSampleType, + queryLogDuration, + resultLimit, + schemaFilterPattern, + showDatabaseFilter, + showSchemaFilter, + showTableFilter, + stageFileLocation, + tableFilterPattern, + threadCount, + timeoutSeconds, + } = state; switch (type) { case PipelineType.Usage: { return { - queryLogDuration, - resultLimit, - stageFileLocation, + queryLogDuration: queryLogDuration, + resultLimit: resultLimit, + stageFileLocation: stageFileLocation, type: usageIngestionType, }; } case PipelineType.Lineage: { return { - queryLogDuration, - resultLimit, + queryLogDuration: queryLogDuration, + resultLimit: resultLimit, type: lineageIngestionType, }; } @@ -587,6 +483,7 @@ const AddIngestion = ({ }; const createNewIngestion = () => { + const { repeatFrequency, enableDebugLog, ingestionName } = state; const ingestionDetails: CreateIngestionPipeline = { airflowConfig: { scheduleInterval: isEmpty(repeatFrequency) @@ -632,6 +529,7 @@ const AddIngestion = ({ }; const updateIngestion = () => { + const { repeatFrequency, enableDebugLog } = state; if (data) { const updatedData: IngestionPipeline = { ...data, @@ -704,7 +602,7 @@ const AddIngestion = ({ return ( - "{ingestionName}" + "{state.ingestionName}" {status === FormSubmitType.ADD ? createMessage : updateMessage} @@ -739,86 +637,31 @@ const AddIngestion = ({
{activeIngestionStep === 1 && ( setDatabaseServiceNames(val)} - handleDescription={(val) => setDescription(val)} - handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)} - handleIncludeLineage={() => setIncludeLineage((pre) => !pre)} - handleIncludeTags={() => setIncludeTags((pre) => !pre)} - handleIncludeView={() => setIncludeView((pre) => !pre)} - handleIngestSampleData={() => setIngestSampleData((pre) => !pre)} - handleIngestionName={(val) => setIngestionName(val)} - handleMarkAllDeletedTables={() => - setMarkAllDeletedTables((pre) => !pre) - } - handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)} - handleProfileSample={(val) => setProfileSample(val)} - handleProfileSampleType={(val) => setProfileSampleType(val)} - handleQueryLogDuration={(val) => setQueryLogDuration(val)} - handleResultLimit={setResultLimit} handleShowFilter={handleShowFilter} - handleStageFileLocation={(val) => setStageFileLocation(val)} - handleThreadCount={setThreadCount} - handleTimeoutSeconds={setTimeoutSeconds} - includeLineage={includeLineage} - includeTags={includeTag} - includeView={includeView} - ingestSampleData={ingestSampleData} - ingestionName={ingestionName} - markAllDeletedTables={markAllDeletedTables} - markDeletedTables={markDeletedTables} - mlModelFilterPattern={mlModelFilterPattern} - pipelineFilterPattern={pipelineFilterPattern} pipelineType={pipelineType} - profileSample={profileSample} - profileSampleType={profileSampleType} - queryLogDuration={queryLogDuration} - resultLimit={resultLimit} - schemaFilterPattern={schemaFilterPattern} serviceCategory={serviceCategory} - showChartFilter={showChartFilter} - showDashboardFilter={showDashboardFilter} - showDatabaseFilter={showDatabaseFilter} - showMlModelFilter={showMlModelFilter} - showPipelineFilter={showPipelineFilter} - showSchemaFilter={showSchemaFilter} - showTableFilter={showTableFilter} - showTopicFilter={showTopicFilter} - stageFileLocation={stageFileLocation} - tableFilterPattern={tableFilterPattern} - threadCount={threadCount} - timeoutSeconds={timeoutSeconds} - topicFilterPattern={topicFilterPattern} - useFqnFilter={useFqnFilter} onCancel={handleCancelClick} + onChange={handleStateChange} onNext={handleNext} - onUseFqnFilterClick={() => setUseFqnFilter((pre) => !pre)} /> )} {activeIngestionStep === 2 && ( setGcsConfigType(type)} - handleIngestionName={(val) => setIngestionName(val)} - handleSourceChange={(src) => setDbtConfigSourceType(src)} - ingestionName={ingestionName} okText={t('label.next')} - source={dbtConfigSourceType} onCancel={handleCancelClick} + onChange={handleStateChange} onSubmit={(dbtConfigData) => { - setDbtConfigSource(dbtConfigData); + handleStateChange({ + dbtConfigSource: dbtConfigData, + }); handleNext(); }} /> @@ -834,18 +677,16 @@ const AddIngestion = ({ {activeIngestionStep === 4 && ( - setRepeatFrequency(value) - } includePeriodOptions={ pipelineType === PipelineType.DataInsight ? ['day'] : undefined } - repeatFrequency={repeatFrequency} + repeatFrequency={state.repeatFrequency} status={saveState} submitButtonLabel={ isUndefined(data) ? t('label.add-deploy') : t('label.submit') } onBack={handlePrev} + onChange={handleStateChange} onDeploy={handleScheduleIntervalDeployClick} /> )} @@ -854,7 +695,7 @@ const AddIngestion = ({ { @@ -31,86 +36,74 @@ jest.mock('../../common/toggle-switch/ToggleSwitchV1', () => { }); const mockConfigureIngestion: ConfigureIngestionProps = { - ingestionName: '', - databaseFilterPattern: { - includes: [], - excludes: [], - }, - dashboardFilterPattern: { - includes: [], - excludes: [], - }, - chartFilterPattern: { - includes: [], - excludes: [], - }, - schemaFilterPattern: { - includes: [], - excludes: [], - }, - tableFilterPattern: { - includes: [], - excludes: [], - }, - topicFilterPattern: { - includes: [], - excludes: [], - }, - pipelineFilterPattern: { - includes: [], - excludes: [], - }, - mlModelFilterPattern: { - includes: [], - excludes: [], - }, - includeLineage: false, - includeView: false, - includeTags: false, pipelineType: PipelineType.Metadata, formType: FormSubmitType.EDIT, - queryLogDuration: 1, - resultLimit: 100, - stageFileLocation: '', - markDeletedTables: false, - showDashboardFilter: false, - showDatabaseFilter: false, - showSchemaFilter: false, - showTableFilter: false, - showTopicFilter: false, - showChartFilter: false, - showPipelineFilter: false, - showMlModelFilter: false, - handleIncludeLineage: jest.fn(), - handleIncludeView: jest.fn(), - handleIncludeTags: jest.fn(), - handleIngestionName: jest.fn(), - handleMarkDeletedTables: jest.fn(), - handleProfileSample: jest.fn(), - handleQueryLogDuration: jest.fn(), - handleResultLimit: jest.fn(), - handleStageFileLocation: jest.fn(), getIncludeValue: jest.fn(), getExcludeValue: jest.fn(), handleShowFilter: jest.fn(), onCancel: jest.fn(), onNext: jest.fn(), - handleProfileSampleType: jest.fn(), - profileSample: 1, - profileSampleType: ProfileSampleType.Percentage, serviceCategory: ServiceCategory.DATABASE_SERVICES, - enableDebugLog: false, - handleEnableDebugLog: jest.fn(), - ingestSampleData: false, - handleIngestSampleData: jest.fn(), - databaseServiceNames: [''], - handleDatasetServiceName: jest.fn(), - threadCount: 5, - handleThreadCount: jest.fn(), - timeoutSeconds: 43200, - handleTimeoutSeconds: jest.fn(), - useFqnFilter: false, - onUseFqnFilterClick: jest.fn(), + onChange: jest.fn(), + data: { + ingestionName: '', + databaseFilterPattern: { + includes: [], + excludes: [], + }, + dashboardFilterPattern: { + includes: [], + excludes: [], + }, + chartFilterPattern: { + includes: [], + excludes: [], + }, + schemaFilterPattern: { + includes: [], + excludes: [], + }, + tableFilterPattern: { + includes: [], + excludes: [], + }, + topicFilterPattern: { + includes: [], + excludes: [], + }, + pipelineFilterPattern: { + includes: [], + excludes: [], + }, + mlModelFilterPattern: { + includes: [], + excludes: [], + }, + includeLineage: false, + includeView: false, + includeTags: false, + + queryLogDuration: 1, + resultLimit: 100, + stageFileLocation: '', + markDeletedTables: false, + showDashboardFilter: false, + showDatabaseFilter: false, + showSchemaFilter: false, + showTableFilter: false, + showTopicFilter: false, + showChartFilter: false, + showPipelineFilter: false, + showMlModelFilter: false, + profileSample: 1, + profileSampleType: ProfileSampleType.Percentage, + enableDebugLog: false, + ingestSampleData: false, + databaseServiceNames: [''], + threadCount: 5, + timeoutSeconds: 43200, + useFqnFilter: false, + } as unknown as AddIngestionState, }; describe('Test ConfigureIngestion component', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index 4078f3df1e0..09e3c536d94 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -13,7 +13,7 @@ import { Form, InputNumber, Select, Typography } from 'antd'; import { isNil } from 'lodash'; -import React, { Fragment, useRef } from 'react'; +import React, { Fragment, useMemo, useRef } from 'react'; import { useTranslation } from 'react-i18next'; import { PROFILE_SAMPLE_OPTIONS } from '../../../constants/profiler.constant'; import { FilterPatternEnum } from '../../../enums/filterPattern.enum'; @@ -29,78 +29,174 @@ import { EditorContentRef } from '../../common/rich-text-editor/RichTextEditor.i import ToggleSwitchV1 from '../../common/toggle-switch/ToggleSwitchV1'; import { Field } from '../../Field/Field'; import SliderWithInput from '../../SliderWithInput/SliderWithInput'; -import { ConfigureIngestionProps } from '../addIngestion.interface'; +import { + AddIngestionState, + ConfigureIngestionProps, + ShowFilter, +} from '../addIngestion.interface'; const ConfigureIngestion = ({ - ingestionName, - description = '', - databaseServiceNames, - databaseFilterPattern, - dashboardFilterPattern, - schemaFilterPattern, - tableFilterPattern, - topicFilterPattern, - chartFilterPattern, - pipelineFilterPattern, - mlModelFilterPattern, - includeLineage, - includeView, - includeTags, - markDeletedTables, - markAllDeletedTables, - serviceCategory, - pipelineType, - showDatabaseFilter, - ingestSampleData, - showDashboardFilter, - showSchemaFilter, - showTableFilter, - showTopicFilter, - showChartFilter, - showPipelineFilter, - showMlModelFilter, - queryLogDuration, - stageFileLocation, - threadCount, - timeoutSeconds, - resultLimit, - enableDebugLog, - profileSample, - handleEnableDebugLog, + data, + formType, getExcludeValue, getIncludeValue, - handleIngestionName, - handleDescription, handleShowFilter, - handleIncludeLineage, - handleIncludeView, - handleIncludeTags, - handleMarkDeletedTables, - handleMarkAllDeletedTables, - handleIngestSampleData, - handleDatasetServiceName, - handleQueryLogDuration, - handleProfileSample, - handleStageFileLocation, - handleResultLimit, - handleThreadCount, - handleTimeoutSeconds, - useFqnFilter, - onUseFqnFilterClick, onCancel, + onChange, onNext, - formType, - profileSampleType, - handleProfileSampleType, + pipelineType, + serviceCategory, }: ConfigureIngestionProps) => { const { t } = useTranslation(); const markdownRef = useRef(); + const { + chartFilterPattern, + dashboardFilterPattern, + databaseFilterPattern, + databaseServiceNames, + description, + enableDebugLog, + includeLineage, + includeTags, + includeView, + ingestionName, + ingestSampleData, + markAllDeletedTables, + markDeletedTables, + mlModelFilterPattern, + pipelineFilterPattern, + profileSample, + profileSampleType, + queryLogDuration, + resultLimit, + schemaFilterPattern, + showChartFilter, + showDashboardFilter, + showDatabaseFilter, + showMlModelFilter, + showPipelineFilter, + showSchemaFilter, + showTableFilter, + showTopicFilter, + stageFileLocation, + tableFilterPattern, + threadCount, + timeoutSeconds, + topicFilterPattern, + useFqnFilter, + } = useMemo( + () => ({ + chartFilterPattern: data.chartFilterPattern, + dashboardFilterPattern: data.dashboardFilterPattern, + databaseFilterPattern: data.databaseFilterPattern, + databaseServiceNames: data.databaseServiceNames, + description: data.description, + enableDebugLog: data.enableDebugLog, + includeLineage: data.includeLineage, + includeTags: data.includeTags, + includeView: data.includeView, + ingestionName: data.ingestionName, + ingestSampleData: data.ingestSampleData, + markAllDeletedTables: data.markAllDeletedTables, + markDeletedTables: data.markDeletedTables, + mlModelFilterPattern: data.mlModelFilterPattern, + pipelineFilterPattern: data.pipelineFilterPattern, + profileSample: data.profileSample, + profileSampleType: data.profileSampleType, + queryLogDuration: data.queryLogDuration, + resultLimit: data.resultLimit, + schemaFilterPattern: data.schemaFilterPattern, + showChartFilter: data.showChartFilter, + showDashboardFilter: data.showDashboardFilter, + showDatabaseFilter: data.showDatabaseFilter, + showMlModelFilter: data.showMlModelFilter, + showPipelineFilter: data.showPipelineFilter, + showSchemaFilter: data.showSchemaFilter, + showTableFilter: data.showTableFilter, + showTopicFilter: data.showTopicFilter, + stageFileLocation: data.stageFileLocation, + tableFilterPattern: data.tableFilterPattern, + threadCount: data.threadCount, + timeoutSeconds: data.timeoutSeconds, + topicFilterPattern: data.topicFilterPattern, + useFqnFilter: data.useFqnFilter, + }), + [data] + ); + + const toggleField = (field: keyof AddIngestionState) => + onChange({ [field]: !data[field] }); + + const handleValueParseInt = + (property: keyof AddIngestionState) => + (event: React.ChangeEvent) => + onChange({ + [property]: parseInt(event.target.value), + }); + + const handleValueChange = + (property: keyof AddIngestionState) => + (event: React.ChangeEvent) => + onChange({ + [property]: event.target.value, + }); + + const handleProfileSample = (profileSample: number | undefined | null) => + onChange({ + profileSample: profileSample ?? undefined, + }); + const handleProfileSampleTypeChange = (value: ProfileSampleType) => { - handleProfileSampleType(value); + onChange({ + profileSampleType: value, + }); + handleProfileSample(undefined); }; + const handleDashBoardServiceNames = (inputValue: string) => { + const separator = ','; + + const databaseNames = inputValue.includes(separator) + ? inputValue.split(separator) + : Array(inputValue); + + if (databaseNames) { + onChange({ + databaseServiceNames: databaseNames, + }); + } + }; + + const handleEnableDebugLogCheck = () => toggleField('enableDebugLog'); + + const handleIncludeLineage = () => toggleField('includeLineage'); + + const handleIncludeTags = () => toggleField('includeTags'); + + const handleIncludeViewToggle = () => toggleField('includeView'); + + const handleIngestSampleToggle = () => toggleField('ingestSampleData'); + + const handleMarkAllDeletedTables = () => toggleField('markAllDeletedTables'); + + const handleMarkDeletedTables = () => toggleField('markDeletedTables'); + + const handleFqnFilter = () => toggleField('useFqnFilter'); + + const handleQueryLogDuration = handleValueParseInt('queryLogDuration'); + + const handleResultLimit = handleValueParseInt('resultLimit'); + + const handleStageFileLocation = handleValueChange('stageFileLocation'); + + const handleThreadCount = handleValueParseInt('threadCount'); + + const handleTimeoutSeconds = handleValueParseInt('timeoutSeconds'); + + const handleIngestionName = handleValueChange('ingestionName'); + const getIngestSampleToggle = (label: string, desc: string) => { return ( <> @@ -109,7 +205,7 @@ const ConfigureIngestion = ({
@@ -127,7 +223,7 @@ const ConfigureIngestion = ({ @@ -169,9 +265,7 @@ const ConfigureIngestion = ({ - handleProfileSample(value ?? undefined) - } + onChange={handleProfileSample} /> )} @@ -188,7 +282,7 @@ const ConfigureIngestion = ({ name: t('label.row-count-lowercase'), })} value={profileSample} - onChange={(value) => handleProfileSample(value ?? undefined)} + onChange={handleProfileSample} /> )} @@ -212,7 +306,7 @@ const ConfigureIngestion = ({ placeholder="5" type="number" value={threadCount} - onChange={(e) => handleThreadCount(parseInt(e.target.value))} + onChange={handleThreadCount} /> ); @@ -233,7 +327,7 @@ const ConfigureIngestion = ({ placeholder="43200" type="number" value={timeoutSeconds} - onChange={(e) => handleTimeoutSeconds(parseInt(e.target.value))} + onChange={handleTimeoutSeconds} /> ); @@ -250,7 +344,7 @@ const ConfigureIngestion = ({ @@ -286,11 +380,7 @@ const ConfigureIngestion = ({ { - if (handleMarkDeletedTables) { - handleMarkDeletedTables(); - } - }} + handleCheck={handleMarkDeletedTables} testId="mark-deleted" /> @@ -306,11 +396,7 @@ const ConfigureIngestion = ({ { - if (handleMarkAllDeletedTables) { - handleMarkAllDeletedTables(); - } - }} + handleCheck={handleMarkAllDeletedTables} testId="mark-deleted-filter-only" /> @@ -357,7 +443,7 @@ const ConfigureIngestion = ({ @@ -369,18 +455,6 @@ const ConfigureIngestion = ({ ); }; - const handleDashBoardServiceNames = (inputValue: string) => { - const separator = ','; - - const databaseNames = inputValue.includes(separator) - ? inputValue.split(separator) - : Array(inputValue); - - if (databaseNames) { - handleDatasetServiceName(databaseNames); - } - }; - const getDashboardDBServiceName = () => { return ( @@ -413,7 +487,7 @@ const ConfigureIngestion = ({ getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleChecked={(value) => - handleShowFilter(value, FilterPatternEnum.DATABASE) + handleShowFilter(value, ShowFilter.showDatabaseFilter) } includePattern={databaseFilterPattern?.includes ?? []} type={FilterPatternEnum.DATABASE} @@ -424,7 +498,7 @@ const ConfigureIngestion = ({ getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleChecked={(value) => - handleShowFilter(value, FilterPatternEnum.SCHEMA) + handleShowFilter(value, ShowFilter.showSchemaFilter) } includePattern={schemaFilterPattern?.includes ?? []} type={FilterPatternEnum.SCHEMA} @@ -435,7 +509,7 @@ const ConfigureIngestion = ({ getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleChecked={(value) => - handleShowFilter(value, FilterPatternEnum.TABLE) + handleShowFilter(value, ShowFilter.showTableFilter) } includePattern={tableFilterPattern?.includes ?? []} showSeparator={false} @@ -465,7 +539,7 @@ const ConfigureIngestion = ({ getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleChecked={(value) => - handleShowFilter(value, FilterPatternEnum.DASHBOARD) + handleShowFilter(value, ShowFilter.showDashboardFilter) } includePattern={dashboardFilterPattern.includes ?? []} type={FilterPatternEnum.DASHBOARD} @@ -476,7 +550,7 @@ const ConfigureIngestion = ({ getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleChecked={(value) => - handleShowFilter(value, FilterPatternEnum.CHART) + handleShowFilter(value, ShowFilter.showChartFilter) } includePattern={chartFilterPattern.includes ?? []} showSeparator={false} @@ -497,7 +571,7 @@ const ConfigureIngestion = ({ getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleChecked={(value) => - handleShowFilter(value, FilterPatternEnum.TOPIC) + handleShowFilter(value, ShowFilter.showTopicFilter) } includePattern={topicFilterPattern.includes ?? []} showSeparator={false} @@ -522,7 +596,7 @@ const ConfigureIngestion = ({ getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleChecked={(value) => - handleShowFilter(value, FilterPatternEnum.PIPELINE) + handleShowFilter(value, ShowFilter.showPipelineFilter) } includePattern={pipelineFilterPattern.includes ?? []} showSeparator={false} @@ -542,7 +616,7 @@ const ConfigureIngestion = ({ getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleChecked={(value) => - handleShowFilter(value, FilterPatternEnum.MLMODEL) + handleShowFilter(value, ShowFilter.showMlModelFilter) } includePattern={mlModelFilterPattern.includes ?? []} showSeparator={false} @@ -574,7 +648,7 @@ const ConfigureIngestion = ({ name="name" type="text" value={ingestionName} - onChange={(e) => handleIngestionName(e.target.value)} + onChange={handleIngestionName} /> {getSeparator('')} @@ -602,7 +676,7 @@ const ConfigureIngestion = ({ name="query-log-duration" type="number" value={queryLogDuration} - onChange={(e) => handleQueryLogDuration(parseInt(e.target.value))} + onChange={handleQueryLogDuration} /> {getSeparator('')} @@ -622,7 +696,7 @@ const ConfigureIngestion = ({ name="stage-file-location" type="text" value={stageFileLocation} - onChange={(e) => handleStageFileLocation(e.target.value)} + onChange={handleStageFileLocation} /> {getSeparator('')} @@ -642,7 +716,7 @@ const ConfigureIngestion = ({ name="result-limit" type="number" value={resultLimit} - onChange={(e) => handleResultLimit(parseInt(e.target.value))} + onChange={handleResultLimit} /> {getSeparator('')} @@ -670,7 +744,7 @@ const ConfigureIngestion = ({ name="query-log-duration" type="number" value={queryLogDuration} - onChange={(e) => handleQueryLogDuration(parseInt(e.target.value))} + onChange={handleQueryLogDuration} /> {getSeparator('')} @@ -690,7 +764,7 @@ const ConfigureIngestion = ({ name="result-limit" type="number" value={resultLimit} - onChange={(e) => handleResultLimit(parseInt(e.target.value))} + onChange={handleResultLimit} /> {getSeparator('')} @@ -717,7 +791,7 @@ const ConfigureIngestion = ({ name="name" type="text" value={ingestionName} - onChange={(e) => handleIngestionName(e.target.value)} + onChange={handleIngestionName} /> {getSeparator('')} @@ -776,8 +850,9 @@ const ConfigureIngestion = ({ }; const handleNext = () => { - handleDescription && - handleDescription(markdownRef.current?.getEditorContent() || ''); + onChange({ + description: markdownRef.current?.getEditorContent() || '', + }); onNext(); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.test.tsx index bcf6d2769d7..3a27e6abdcf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.test.tsx @@ -29,10 +29,10 @@ jest.mock('../../common/toggle-switch/ToggleSwitchV1', () => { const mockScheduleIntervalProps: ScheduleIntervalProps = { status: 'initial', repeatFrequency: '', - handleRepeatFrequencyChange: jest.fn(), onBack: jest.fn(), onDeploy: jest.fn(), submitButtonLabel: 'Add', + onChange: jest.fn(), }; describe('Test ScheduleInterval component', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx index 6dcccedd36e..035bbdee470 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx @@ -21,14 +21,18 @@ import Loader from '../../Loader/Loader'; import { ScheduleIntervalProps } from '../addIngestion.interface'; const ScheduleInterval = ({ - status, - repeatFrequency, - handleRepeatFrequencyChange, - submitButtonLabel, - onBack, - onDeploy, includePeriodOptions, + onBack, + onChange, + onDeploy, + repeatFrequency, + status, + submitButtonLabel, }: ScheduleIntervalProps) => { + const handleRepeatFrequencyChange = (repeatFrequency: string) => + onChange({ + repeatFrequency: repeatFrequency, + }); const { t } = useTranslation(); return ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts index f99ea8d6f0a..f0172b986a4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts @@ -15,18 +15,23 @@ import { LoadingState } from 'Models'; import { FilterPatternEnum } from '../../enums/filterPattern.enum'; import { FormSubmitType } from '../../enums/form.enum'; import { ServiceCategory } from '../../enums/service.enum'; -import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; +import { + ConfigClass, + CreateIngestionPipeline, + DbtConfig, +} from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; import { ProfileSampleType } from '../../generated/entity/data/table'; import { FilterPattern, IngestionPipeline, PipelineType, } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; -import { - DbtConfig, - DbtPipelineClass, -} from '../../generated/metadataIngestion/dbtPipeline'; +import { DbtPipelineClass } from '../../generated/metadataIngestion/dbtPipeline'; import { DataObj } from '../../interface/service.interface'; +import { + DBT_SOURCES, + GCS_CONFIG, +} from '../common/DBTConfigFormBuilder/DBTFormEnum'; export interface AddIngestionProps { activeIngestionStep: number; @@ -58,72 +63,22 @@ export interface AddIngestionProps { } export interface ConfigureIngestionProps { + data: AddIngestionState; formType: FormSubmitType; - ingestionName: string; - description?: string; - databaseServiceNames: string[]; - serviceCategory: ServiceCategory; - databaseFilterPattern: FilterPattern; - dashboardFilterPattern: FilterPattern; - schemaFilterPattern: FilterPattern; - tableFilterPattern: FilterPattern; - topicFilterPattern: FilterPattern; - chartFilterPattern: FilterPattern; - pipelineFilterPattern: FilterPattern; - mlModelFilterPattern: FilterPattern; - includeLineage: boolean; - includeView: boolean; - includeTags: boolean; - markDeletedTables?: boolean; - markAllDeletedTables?: boolean; - enableDebugLog: boolean; - profileSample?: number; - profileSampleType?: ProfileSampleType; - ingestSampleData: boolean; - useFqnFilter: boolean; - pipelineType: PipelineType; - showDatabaseFilter: boolean; - showDashboardFilter: boolean; - showSchemaFilter: boolean; - showTableFilter: boolean; - showTopicFilter: boolean; - showChartFilter: boolean; - showPipelineFilter: boolean; - showMlModelFilter: boolean; - threadCount: number; - queryLogDuration: number; - stageFileLocation: string; - resultLimit: number; - timeoutSeconds: number; - handleIngestionName: (value: string) => void; - handleDatasetServiceName: (value: string[]) => void; - handleDescription?: (value: string) => void; - handleIncludeLineage: () => void; - onUseFqnFilterClick: () => void; - handleIncludeView: () => void; - handleIncludeTags: () => void; - handleMarkDeletedTables?: () => void; - handleMarkAllDeletedTables?: () => void; - handleEnableDebugLog: () => void; - handleIngestSampleData: () => void; - getIncludeValue: (value: string[], type: FilterPatternEnum) => void; getExcludeValue: (value: string[], type: FilterPatternEnum) => void; - handleShowFilter: (value: boolean, type: FilterPatternEnum) => void; - handleProfileSample: (value?: number) => void; - handleQueryLogDuration: (value: number) => void; - handleProfileSampleType: (value: ProfileSampleType) => void; - handleStageFileLocation: (value: string) => void; - handleResultLimit: (value: number) => void; - handleThreadCount: (value: number) => void; - handleTimeoutSeconds: (value: number) => void; + getIncludeValue: (value: string[], type: FilterPatternEnum) => void; + handleShowFilter: (value: boolean, type: string) => void; onCancel: () => void; + onChange: (newState: Partial) => void; onNext: () => void; + pipelineType: PipelineType; + serviceCategory: ServiceCategory; } export type ScheduleIntervalProps = { + onChange: (newState: Partial) => void; status: LoadingState; repeatFrequency: string; - handleRepeatFrequencyChange: (value: string) => void; includePeriodOptions?: string[]; submitButtonLabel: string; onBack: () => void; @@ -133,3 +88,58 @@ export type ScheduleIntervalProps = { // Todo: Need to refactor below type, as per schema change #9575 export type ModifiedDbtConfig = DbtConfig & Pick; + +export interface AddIngestionState { + chartFilterPattern: FilterPattern; + dashboardFilterPattern: FilterPattern; + databaseFilterPattern: FilterPattern; + databaseServiceNames: string[]; + dbtConfigSource: ModifiedDbtConfig; + dbtConfigSourceType: DBT_SOURCES; + description: string; + enableDebugLog: boolean; + gcsConfigType: GCS_CONFIG | undefined; + includeLineage: boolean; + includeTags: boolean; + includeView: boolean; + ingestionName: string; + ingestSampleData: boolean; + markAllDeletedTables: boolean | undefined; + markDeletedTables: boolean | undefined; + metadataToESConfig: ConfigClass | undefined; + mlModelFilterPattern: FilterPattern; + pipelineFilterPattern: FilterPattern; + profileSample: number | undefined; + profileSampleType: ProfileSampleType; + queryLogDuration: number; + repeatFrequency: string; + resultLimit: number; + saveState: LoadingState; + schemaFilterPattern: FilterPattern; + showChartFilter: boolean; + showDashboardFilter: boolean; + showDatabaseFilter: boolean; + showDeployModal: boolean; + showMlModelFilter: boolean; + showPipelineFilter: boolean; + showSchemaFilter: boolean; + showTableFilter: boolean; + showTopicFilter: boolean; + stageFileLocation: string; + tableFilterPattern: FilterPattern; + threadCount: number; + timeoutSeconds: number; + topicFilterPattern: FilterPattern; + useFqnFilter: boolean; +} + +export enum ShowFilter { + showChartFilter = 'showChartFilter', + showDashboardFilter = 'showDashboardFilter', + showDatabaseFilter = 'showDatabaseFilter', + showMlModelFilter = 'showMlModelFilter', + showPipelineFilter = 'showPipelineFilter', + showSchemaFilter = 'showSchemaFilter', + showTableFilter = 'showTableFilter', + showTopicFilter = 'showTopicFilter', +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SliderWithInput/SliderWithInput.tsx b/openmetadata-ui/src/main/resources/ui/src/components/SliderWithInput/SliderWithInput.tsx index b267e03acb6..3d7e52d198f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SliderWithInput/SliderWithInput.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/SliderWithInput/SliderWithInput.tsx @@ -12,7 +12,7 @@ */ import { Col, InputNumber, Row, Slider } from 'antd'; -import React from 'react'; +import React, { useCallback } from 'react'; import { SliderWithInputProps } from './SliderWithInput.interface'; const SliderWithInput = ({ @@ -20,6 +20,8 @@ const SliderWithInput = ({ onChange, className, }: SliderWithInputProps) => { + const formatter = useCallback((value) => `${value}%`, [value]); + return ( @@ -38,7 +40,7 @@ const SliderWithInput = ({ `${value}%`} + formatter={formatter} max={100} min={0} step={1} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ProfilerSettingsModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ProfilerSettingsModal.tsx index bfa5e9673cb..f691344ee1a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ProfilerSettingsModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ProfilerSettingsModal.tsx @@ -29,7 +29,14 @@ import { AxiosError } from 'axios'; import classNames from 'classnames'; import 'codemirror/addon/fold/foldgutter.css'; import { isEmpty, isEqual, isUndefined, omit, startCase } from 'lodash'; -import React, { useEffect, useMemo, useState } from 'react'; +import React, { + Reducer, + useCallback, + useEffect, + useMemo, + useReducer, + useState, +} from 'react'; import { Controlled as CodeMirror } from 'react-codemirror2'; import { useTranslation } from 'react-i18next'; import { @@ -42,16 +49,18 @@ import { SUPPORTED_PARTITION_TYPE, } from '../../../constants/profiler.constant'; import { - ColumnProfilerConfig, - PartitionProfilerConfig, ProfileSampleType, TableProfilerConfig, } from '../../../generated/entity/data/table'; import jsonData from '../../../jsons/en'; +import { reducerWithoutAction } from '../../../utils/CommonUtils'; import SVGIcons, { Icons } from '../../../utils/SvgUtils'; import { showErrorToast, showSuccessToast } from '../../../utils/ToastUtils'; import SliderWithInput from '../../SliderWithInput/SliderWithInput'; -import { ProfilerSettingsModalProps } from '../TableProfiler.interface'; +import { + ProfilerSettingModalState, + ProfilerSettingsModalProps, +} from '../TableProfiler.interface'; import '../tableProfiler.less'; const ProfilerSettingsModal: React.FC = ({ @@ -62,20 +71,32 @@ const ProfilerSettingsModal: React.FC = ({ }) => { const { t } = useTranslation(); const [form] = Form.useForm(); - const [data, setData] = useState(); - const [sqlQuery, setSqlQuery] = useState(''); - const [profileSample, setProfileSample] = useState(100); - const [excludeCol, setExcludeCol] = useState([]); - const [includeCol, setIncludeCol] = useState( - DEFAULT_INCLUDE_PROFILE - ); + const [isLoading, setIsLoading] = useState(false); - const [enablePartition, setEnablePartition] = useState(false); - const [partitionData, setPartitionData] = useState(); - const [selectedProfileSampleType, setSelectedProfileSampleType] = useState< - ProfileSampleType | undefined - >(ProfileSampleType.Percentage); + const initialState: ProfilerSettingModalState = useMemo( + () => ({ + data: undefined, + sqlQuery: '', + profileSample: 100, + excludeCol: [], + includeCol: DEFAULT_INCLUDE_PROFILE, + enablePartition: false, + partitionData: undefined, + selectedProfileSampleType: ProfileSampleType.Percentage, + }), + [] + ); + const [state, dispatch] = useReducer< + Reducer> + >(reducerWithoutAction, initialState); + + const handleStateChange = useCallback( + (newState: Partial) => { + dispatch(newState); + }, + [] + ); const selectOptions = useMemo(() => { return columns.map(({ name }) => ({ @@ -131,12 +152,13 @@ const ProfilerSettingsModal: React.FC = ({ profileSampleType, excludeColumns, } = tableProfilerConfig; - setSqlQuery(profileQuery || ''); - setProfileSample(profileSample); - setExcludeCol(excludeColumns || []); - setSelectedProfileSampleType( - profileSampleType || ProfileSampleType.Percentage - ); + handleStateChange({ + sqlQuery: profileQuery || '', + profileSample: profileSample, + excludeCol: excludeColumns || [], + selectedProfileSampleType: + profileSampleType || ProfileSampleType.Percentage, + }); const profileSampleTypeCheck = profileSampleType === ProfileSampleType.Percentage; @@ -162,10 +184,15 @@ const ProfilerSettingsModal: React.FC = ({ return col; }); form.setFieldsValue({ includeColumns: includeColValue }); - setIncludeCol(includeColValue); + handleStateChange({ + includeCol: includeColValue, + }); } if (partitioning) { - setEnablePartition(partitioning.enablePartitioning || false); + handleStateChange({ + enablePartition: partitioning.enablePartitioning || false, + }); + form.setFieldsValue({ ...partitioning }); } }; @@ -176,7 +203,10 @@ const ProfilerSettingsModal: React.FC = ({ if (response) { const { tableProfilerConfig } = response; if (tableProfilerConfig) { - setData(tableProfilerConfig); + handleStateChange({ + data: tableProfilerConfig, + }); + updateInitialConfig(tableProfilerConfig); } } else { @@ -193,10 +223,13 @@ const ProfilerSettingsModal: React.FC = ({ }; const getIncludesColumns = () => { - const includeCols = includeCol.filter( + const includeCols = state.includeCol.filter( ({ columnName }) => !isUndefined(columnName) ); - setIncludeCol(includeCols); + + handleStateChange({ + includeCol: includeCols, + }); return includeCols.map((col) => { if (col.metrics && col.metrics[0] === 'all') { @@ -209,7 +242,9 @@ const ProfilerSettingsModal: React.FC = ({ }); }; - const handleSave: FormProps['onFinish'] = async (data) => { + const handleSave: FormProps['onFinish'] = useCallback(async (data) => { + const { excludeCol, sqlQuery, includeCol, enablePartition, partitionData } = + state; setIsLoading(true); const { profileSamplePercentage, profileSampleRows, profileSampleType } = data; @@ -252,11 +287,55 @@ const ProfilerSettingsModal: React.FC = ({ } finally { setIsLoading(false); } - }; - const handleCancel = () => { + }, []); + const handleCancel = useCallback(() => { + const { data } = state; data && updateInitialConfig(data); onVisibilityChange(false); - }; + }, []); + + const handleProfileSampleType = useCallback( + (selectedProfileSampleType) => + handleStateChange({ + selectedProfileSampleType, + }), + [] + ); + + const handleProfileSample = useCallback( + (value) => + handleStateChange({ + profileSample: Number(value), + }), + [] + ); + + const handleCodeMirrorChange = useCallback( + (_Editor, _EditorChange, value) => { + handleStateChange({ + sqlQuery: value, + }); + }, + [] + ); + + const handleIncludeColumnsProfiler = useCallback((_, data) => { + handleStateChange({ + includeCol: data.includeColumns, + partitionData: omit(data, 'includeColumns'), + }); + }, []); + + const handleChange = + (field: keyof ProfilerSettingModalState) => + (value: ProfilerSettingModalState[keyof ProfilerSettingModalState]) => + handleStateChange({ + [field]: value, + }); + + const handleExcludeCol = handleChange('excludeCol'); + + const handleEnablePartition = handleChange('enablePartition'); useEffect(() => { fetchProfileConfig(); @@ -292,8 +371,8 @@ const ProfilerSettingsModal: React.FC = ({ data-testid="configure-ingestion-container" form={form} initialValues={{ - profileSampleType: selectedProfileSampleType, - profileSamplePercentage: profileSample || 100, + profileSampleType: state?.selectedProfileSampleType, + profileSamplePercentage: state?.profileSample || 100, }} layout="vertical"> = ({ className="w-full" data-testid="profile-sample" options={PROFILE_SAMPLE_OPTIONS} - onChange={setSelectedProfileSampleType} + onChange={handleProfileSampleType} /> - {selectedProfileSampleType === ProfileSampleType.Percentage ? ( + {state?.selectedProfileSampleType === + ProfileSampleType.Percentage ? ( = ({ name="profileSamplePercentage"> setProfileSample(Number(value))} + value={state?.profileSample || 0} + onChange={handleProfileSample} /> ) : ( @@ -351,13 +431,9 @@ const ProfilerSettingsModal: React.FC = ({ className="profiler-setting-sql-editor" data-testid="profiler-setting-sql-editor" options={codeMirrorOption} - value={sqlQuery} - onBeforeChange={(_Editor, _EditorChange, value) => { - setSqlQuery(value); - }} - onChange={(_Editor, _EditorChange, value) => { - setSqlQuery(value); - }} + value={state?.sqlQuery} + onBeforeChange={handleCodeMirrorChange} + onChange={handleCodeMirrorChange} /> @@ -371,8 +447,8 @@ const ProfilerSettingsModal: React.FC = ({ options={selectOptions} placeholder={t('label.select-column-plural-to-exclude')} size="middle" - value={excludeCol} - onChange={(value) => setExcludeCol(value)} + value={state?.excludeCol} + onChange={handleExcludeCol} /> @@ -382,16 +458,13 @@ const ProfilerSettingsModal: React.FC = ({ form={form} id="profiler-setting-form" initialValues={{ - includeColumns: includeCol, - ...data?.partitioning, + includeColumns: state?.includeCol, + ...state?.data?.partitioning, }} layout="vertical" name="includeColumnsProfiler" onFinish={handleSave} - onValuesChange={(_, data) => { - setIncludeCol(data.includeColumns); - setPartitionData(omit(data, 'includeColumns')); - }}> + onValuesChange={handleIncludeColumnsProfiler}> {(fields, { add, remove }) => ( <> @@ -409,7 +482,8 @@ const ProfilerSettingsModal: React.FC = ({
1, + 'tw-max-h-40 tw-overflow-y-auto': + state?.includeCol.length > 1, })} data-testid="include-column-container"> {fields.map(({ key, name, ...restField }) => ( @@ -466,10 +540,10 @@ const ProfilerSettingsModal: React.FC = ({

{t('label.enable-partition')}

setEnablePartition(value)} + onChange={handleEnablePartition} /> @@ -492,7 +566,7 @@ const ProfilerSettingsModal: React.FC = ({ name="partitionColumnName" rules={[ { - required: enablePartition, + required: state?.enablePartition, message: t('message.field-text-is-required', { fieldText: t('label.column-entity', { entity: t('label.name'), @@ -504,7 +578,7 @@ const ProfilerSettingsModal: React.FC = ({ allowClear className="w-full" data-testid="column-name" - disabled={!enablePartition} + disabled={!state?.enablePartition} options={partitionColumnOptions} placeholder={t('message.select-column-name')} size="middle" @@ -525,7 +599,7 @@ const ProfilerSettingsModal: React.FC = ({ name="partitionIntervalType" rules={[ { - required: enablePartition, + required: state?.enablePartition, message: t('message.field-text-is-required', { fieldText: t('label.interval-type'), }), @@ -535,7 +609,7 @@ const ProfilerSettingsModal: React.FC = ({ allowClear className="w-full" data-testid="interval-type" - disabled={!enablePartition} + disabled={!state?.enablePartition} options={INTERVAL_TYPE_OPTIONS} placeholder={t('message.select-interval-type')} size="middle" @@ -554,7 +628,7 @@ const ProfilerSettingsModal: React.FC = ({ name="partitionInterval" rules={[ { - required: enablePartition, + required: state?.enablePartition, message: t('message.field-text-is-required', { fieldText: t('label.interval'), }), @@ -563,7 +637,7 @@ const ProfilerSettingsModal: React.FC = ({ @@ -583,7 +657,7 @@ const ProfilerSettingsModal: React.FC = ({ name="partitionIntervalUnit" rules={[ { - required: enablePartition, + required: state?.enablePartition, message: t('message.field-text-is-required', { fieldText: t('label.interval-unit'), }), @@ -593,7 +667,7 @@ const ProfilerSettingsModal: React.FC = ({ allowClear className="w-full" data-testid="select-interval-unit" - disabled={!enablePartition} + disabled={!state?.enablePartition} options={INTERVAL_UNIT_OPTIONS} placeholder={t('message.select-interval-unit')} size="middle" diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts index 774237d723f..89c51b86400 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts @@ -13,7 +13,14 @@ import { PROFILER_FILTER_RANGE } from '../../constants/profiler.constant'; import { SystemProfile } from '../../generated/api/data/createTableProfile'; -import { Column, TableProfile } from '../../generated/entity/data/table'; +import { + Column, + ColumnProfilerConfig, + PartitionProfilerConfig, + ProfileSampleType, + TableProfile, + TableProfilerConfig, +} from '../../generated/entity/data/table'; import { TestCase } from '../../generated/tests/testCase'; import { OperationPermission } from '../PermissionProvider/PermissionProvider.interface'; @@ -76,3 +83,14 @@ export type TableProfilerData = { export type TableProfilerChartProps = { selectedTimeRange: keyof typeof PROFILER_FILTER_RANGE; }; + +export interface ProfilerSettingModalState { + data: TableProfilerConfig | undefined; + sqlQuery: string; + profileSample: number | undefined; + excludeCol: string[]; + includeCol: ColumnProfilerConfig[]; + enablePartition: boolean; + partitionData: PartitionProfilerConfig | undefined; + selectedProfileSampleType: ProfileSampleType | undefined; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigForm.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigForm.interface.ts index bc9d6d8d754..c54712cb196 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigForm.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigForm.interface.ts @@ -17,7 +17,10 @@ import { GCSCredentialsValues, SCredentials, } from '../../../generated/metadataIngestion/dbtPipeline'; -import { ModifiedDbtConfig } from '../../AddIngestion/addIngestion.interface'; +import { + AddIngestionState, + ModifiedDbtConfig, +} from '../../AddIngestion/addIngestion.interface'; import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum'; export interface DBTFormCommonProps { @@ -29,13 +32,9 @@ export interface DBTFormCommonProps { export interface DBTConfigFormProps extends DBTFormCommonProps { formType: FormSubmitType; - data: DbtConfig; - gcsType?: GCS_CONFIG; - source?: DBT_SOURCES; - handleGcsTypeChange?: (type: GCS_CONFIG) => void; - handleSourceChange?: (src: DBT_SOURCES) => void; - ingestionName: string; - handleIngestionName: (value: string) => void; + data: AddIngestionState; + + onChange: (newState: Partial) => void; } export type DbtConfigCloud = Pick< diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.test.tsx index d1f8ab7d70f..02ee7a69a2f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.test.tsx @@ -15,6 +15,8 @@ import { fireEvent, getByTestId, render } from '@testing-library/react'; import i18n from 'i18next'; import React from 'react'; import { FormSubmitType } from '../../../enums/form.enum'; +import { AddIngestionState } from '../../AddIngestion/addIngestion.interface'; +import { DBTConfigFormProps } from './DBTConfigForm.interface'; import DBTConfigFormBuilder from './DBTConfigFormBuilder'; import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum'; @@ -148,23 +150,26 @@ jest.mock('./DBTGCSConfig', () => ({ const mockCancel = jest.fn(); const mockSubmit = jest.fn(); -const mockCatalogChange = jest.fn(); -const mockManifestChange = jest.fn(); const mockIngestionName = jest.fn(); +const mockOnChange = jest.fn(); + +const mockState = { + dbtConfigSource: '', + dbtConfigSourceType: DBT_SOURCES.local, + ingestionName: i18n.t('label.dbt-uppercase'), + gcsConfigType: GCS_CONFIG.GCSValues, +} as unknown as AddIngestionState; const mockProps = { - data: mockData, + data: mockState, okText: i18n.t('label.next'), cancelText: i18n.t('label.cancel'), - gcsType: GCS_CONFIG.GCSValues, - handleGcsTypeChange: mockCatalogChange, - handleSourceChange: mockManifestChange, onCancel: mockCancel, onSubmit: mockSubmit, formType: FormSubmitType.ADD, handleIngestionName: mockIngestionName, - ingestionName: i18n.t('label.dbt-uppercase'), -}; + onChange: mockOnChange, +} as DBTConfigFormProps; describe('Test DBT Config Form Builder', () => { it('Form should render with default dbt source', async () => { @@ -178,7 +183,14 @@ describe('Test DBT Config Form Builder', () => { it('Form should render with local dbt source', async () => { const { container } = render( - + ); const selectSource = getByTestId(container, 'dbt-source'); const dbtLocal = getByTestId(container, 'dbt-local'); @@ -189,7 +201,14 @@ describe('Test DBT Config Form Builder', () => { it('Form should render with http dbt source', async () => { const { container } = render( - + ); const selectSource = getByTestId(container, 'dbt-source'); const dbtHttp = getByTestId(container, 'dbt-http'); @@ -200,7 +219,14 @@ describe('Test DBT Config Form Builder', () => { it('Form should render with s3 dbt source', async () => { const { container } = render( - + ); const selectSource = getByTestId(container, 'dbt-source'); const dbtS3 = getByTestId(container, 'dbt-s3'); @@ -211,7 +237,14 @@ describe('Test DBT Config Form Builder', () => { it('Form should render with gcs dbt source', async () => { const { container } = render( - + ); const selectSource = getByTestId(container, 'dbt-source'); const dbtGCS = getByTestId(container, 'dbt-gcs'); @@ -222,7 +255,14 @@ describe('Test DBT Config Form Builder', () => { it('Form should render with no dbt source', async () => { const { container } = render( - + ); const selectSource = getByTestId(container, 'dbt-source'); const dbtNone = getByTestId(container, 'dbt-source-none'); @@ -233,7 +273,14 @@ describe('Test DBT Config Form Builder', () => { it('should change dbt local fields', async () => { const { container } = render( - + ); const dbtLocal = getByTestId(container, 'dbt-local'); @@ -244,7 +291,14 @@ describe('Test DBT Config Form Builder', () => { it('should change dbt http fields', async () => { const { container } = render( - + ); const dbtHttp = getByTestId(container, 'dbt-http'); @@ -255,7 +309,14 @@ describe('Test DBT Config Form Builder', () => { it('should change dbt s3 fields', async () => { const { container } = render( - + ); const dbtS3 = getByTestId(container, 'dbt-s3'); @@ -266,7 +327,14 @@ describe('Test DBT Config Form Builder', () => { it('should change dbt gcs fields', async () => { const { container } = render( - + ); const dbtGCS = getByTestId(container, 'dbt-gcs'); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx index 9fb75ce7a40..cfd96db0c90 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx @@ -12,7 +12,13 @@ */ import { Button } from 'antd'; -import React, { Fragment, FunctionComponent, useEffect, useState } from 'react'; +import React, { + Fragment, + FunctionComponent, + useEffect, + useMemo, + useState, +} from 'react'; import { useTranslation } from 'react-i18next'; import { FormSubmitType } from '../../../enums/form.enum'; import { @@ -25,28 +31,42 @@ import { Field } from '../../Field/Field'; import { DBTCloudConfig } from './DBTCloudConfig'; import { DBTConfigFormProps } from './DBTConfigForm.interface'; import { DBTSources } from './DBTFormConstants'; -import { DBT_SOURCES } from './DBTFormEnum'; +import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum'; import { DBTGCSConfig } from './DBTGCSConfig'; import { DBTHttpConfig } from './DBTHttpConfig'; import { DBTLocalConfig } from './DBTLocalConfig'; import { DBTS3Config } from './DBTS3Config'; const DBTConfigFormBuilder: FunctionComponent = ({ - data, - okText, cancelText, - gcsType, - source = DBT_SOURCES.local, - handleGcsTypeChange, - handleSourceChange, - onCancel, - onSubmit, + data, formType, - ingestionName, - handleIngestionName, + okText, + onCancel, + onChange, + onSubmit, }: DBTConfigFormProps) => { const { t } = useTranslation(); - const [dbtConfig, setDbtConfig] = useState(data); + // const [dbtConfig, setDbtConfig] = useState(data); + + const { dbtConfigSource, gcsConfigType, ingestionName, dbtConfigSourceType } = + useMemo( + () => ({ + ingestionName: data.ingestionName, + gcsConfigType: data.gcsConfigType, + dbtConfigSourceType: data.dbtConfigSourceType, + dbtConfigSource: data.dbtConfigSource, + }), + [ + data.ingestionName, + data.gcsConfigType, + data.dbtConfigSourceType, + data.dbtConfigSource, + ] + ); + + const [dbtConfig, setDbtConfig] = + useState(dbtConfigSource); const updateDbtConfig = ( key: keyof ModifiedDbtConfig, @@ -61,10 +81,10 @@ const DBTConfigFormBuilder: FunctionComponent = ({ return ( { updateDbtConfig('dbtCloudAccountId', val); }} @@ -88,10 +108,10 @@ const DBTConfigFormBuilder: FunctionComponent = ({ return ( { updateDbtConfig('dbtCatalogFilePath', val); }} @@ -115,10 +135,10 @@ const DBTConfigFormBuilder: FunctionComponent = ({ return ( { updateDbtConfig('dbtCatalogHttpPath', val); }} @@ -142,9 +162,9 @@ const DBTConfigFormBuilder: FunctionComponent = ({ return ( { updateDbtConfig('dbtPrefixConfig', val); }} @@ -161,17 +181,33 @@ const DBTConfigFormBuilder: FunctionComponent = ({ ); }; + const handleGcsTypeChange = (type: GCS_CONFIG) => + onChange({ + gcsConfigType: type, + }); + + const handleOnchange = (event: React.ChangeEvent) => + onChange({ + ingestionName: event.target.value, + }); + + const handleDbtConfigSourceType = ( + event: React.ChangeEvent + ) => { + onChange({ + dbtConfigSourceType: event.target.value as DBT_SOURCES, + }); + }; + const getGCSConfigFields = () => { return ( { - handleGcsTypeChange && handleGcsTypeChange(type); - }} + dbtPrefixConfig={dbtConfig?.dbtPrefixConfig} + dbtSecurityConfig={dbtConfig?.dbtSecurityConfig} + dbtUpdateDescriptions={dbtConfig?.dbtUpdateDescriptions} + gcsType={gcsConfigType} + handleGcsTypeChange={handleGcsTypeChange} handlePrefixConfigChange={(val) => { updateDbtConfig('dbtPrefixConfig', val); }} @@ -189,7 +225,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ }; const getFields = () => { - switch (source) { + switch (dbtConfigSourceType) { case DBT_SOURCES.cloud: { return getCloudConfigFields(); } @@ -236,8 +272,8 @@ const DBTConfigFormBuilder: FunctionComponent = ({ }; useEffect(() => { - setDbtConfig(data); - }, [data, source, gcsType]); + setDbtConfig(dbtConfigSource); + }, [data, dbtConfigSourceType, gcsConfigType]); return ( @@ -256,7 +292,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ name="name" type="text" value={ingestionName} - onChange={(e) => handleIngestionName(e.target.value)} + onChange={handleOnchange} /> {getSeparator('')} @@ -275,11 +311,8 @@ const DBTConfigFormBuilder: FunctionComponent = ({ placeholder={t('label.select-field', { field: t('label.dbt-source'), })} - value={source} - onChange={(e) => { - handleSourceChange && - handleSourceChange(e.target.value as DBT_SOURCES); - }}> + value={dbtConfigSourceType} + onChange={handleDbtConfigSourceType}> {DBTSources.map((option, i) => (