yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-26 01:46:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ui): sonar-cloud code-smells and code optimizations (#9634)

Browse Source 
* fix(ui): sonar-cloud code-smells and code optimizations

* fix errors

* fix other pending code-smells
This commit is contained in:
Chirag Madlani 2023-01-10 18:12:12 +05:30 committed by GitHub
parent 8ada2c8e04
commit 7575a3f55c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 912 additions and 766 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

601
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
View File

@ -13,7 +13,13 @@
import { isEmpty, isUndefined, omit, trim } from 'lodash';
import { LoadingState } from 'Models';
import React, { useMemo, useState } from 'react';
import React, {
Reducer,
useCallback,
useMemo,
useReducer,
useState,
} from 'react';
import { useTranslation } from 'react-i18next';
import {
INITIAL_FILTER_PATTERN,
@ -41,355 +47,200 @@ import {
} from '../../generated/metadataIngestion/dbtPipeline';
import {
getCurrentUserId,
getFilterTypes,
getIngestionFrequency,
reducerWithoutAction,
} from '../../utils/CommonUtils';
import { getSourceTypeFromConfig } from '../../utils/DBTConfigFormUtil';
import { escapeBackwardSlashChar } from '../../utils/JSONSchemaFormUtils';
import { getIngestionName } from '../../utils/ServiceUtils';
import DBTConfigFormBuilder from '../common/DBTConfigFormBuilder/DBTConfigFormBuilder';
import {
DBT_SOURCES,
GCS_CONFIG,
} from '../common/DBTConfigFormBuilder/DBTFormEnum';
import { DBT_SOURCES } from '../common/DBTConfigFormBuilder/DBTFormEnum';
import SuccessScreen from '../common/success-screen/SuccessScreen';
import IngestionStepper from '../IngestionStepper/IngestionStepper.component';
import DeployIngestionLoaderModal from '../Modals/DeployIngestionLoaderModal/DeployIngestionLoaderModal';
import { AddIngestionProps, ModifiedDbtConfig } from './addIngestion.interface';
import {
AddIngestionProps,
AddIngestionState,
ModifiedDbtConfig,
} from './addIngestion.interface';
import ConfigureIngestion from './Steps/ConfigureIngestion';
import MetadataToESConfigForm from './Steps/MetadataToESConfigForm/MetadataToESConfigForm';
import ScheduleInterval from './Steps/ScheduleInterval';
const AddIngestion = ({
activeIngestionStep,
heading,
status,
pipelineType,
data,
serviceData,
serviceCategory,
showSuccessScreen = true,
handleCancelClick,
handleViewServiceClick,
heading,
ingestionAction = '',
ingestionProgress = 0,
isIngestionCreated = false,
isIngestionDeployed = false,
ingestionAction = '',
showDeployButton,
setActiveIngestionStep,
onIngestionDeploy,
onUpdateIngestion,
onSuccessSave,
onAddIngestionSave,
handleCancelClick,
handleViewServiceClick,
onIngestionDeploy,
onSuccessSave,
onUpdateIngestion,
pipelineType,
serviceCategory,
serviceData,
setActiveIngestionStep,
showDeployButton,
showSuccessScreen = true,
status,
}: AddIngestionProps) => {
const { t } = useTranslation();
const isDatabaseService = useMemo(() => {
return serviceCategory === ServiceCategory.DATABASE_SERVICES;
}, [serviceCategory]);
const isServiceTypeOpenMetadata = useMemo(() => {
return serviceData.serviceType === MetadataServiceType.OpenMetadata;
console.log('data:', data);
const { sourceConfig, sourceConfigType } = useMemo(
() => ({
sourceConfig: data?.sourceConfig.config as ConfigClass,
sourceConfigType: (data?.sourceConfig.config as ConfigClass)?.type,
}),
[]
);
const {
configData,
usageIngestionType,
lineageIngestionType,
profilerIngestionType,
} = useMemo(() => {
return {
configData: (data?.sourceConfig.config as DbtPipelineClass)
?.dbtConfigSource,
usageIngestionType: sourceConfigType ?? ConfigType.DatabaseUsage,
lineageIngestionType: sourceConfigType ?? ConfigType.DatabaseLineage,
profilerIngestionType: sourceConfigType ?? ConfigType.Profiler,
};
}, [data]);
const { isDatabaseService, isServiceTypeOpenMetadata } = useMemo(() => {
return {
isDatabaseService: serviceCategory === ServiceCategory.DATABASE_SERVICES,
isServiceTypeOpenMetadata:
serviceData.serviceType === MetadataServiceType.OpenMetadata,
};
}, [serviceCategory]);
const showDBTConfig = useMemo(() => {
return isDatabaseService && pipelineType === PipelineType.Dbt;
}, [isDatabaseService, pipelineType]);
const [saveState, setSaveState] = useState<LoadingState>('initial');
const [showDeployModal, setShowDeployModal] = useState(false);
const [ingestionName, setIngestionName] = useState(
data?.name ?? getIngestionName(serviceData.name, pipelineType)
);
const [ingestSampleData, setIngestSampleData] = useState(
(data?.sourceConfig.config as ConfigClass)?.generateSampleData ?? true
);
const [useFqnFilter, setUseFqnFilter] = useState(
(data?.sourceConfig.config as ConfigClass)?.useFqnForFiltering ?? false
);
const [databaseServiceNames, setDatabaseServiceNames] = useState(
(data?.sourceConfig.config as ConfigClass)?.dbServiceNames ?? []
);
const [description, setDescription] = useState(data?.description ?? '');
const [repeatFrequency, setRepeatFrequency] = useState(
data?.airflowConfig.scheduleInterval ?? getIngestionFrequency(pipelineType)
);
const [showDashboardFilter, setShowDashboardFilter] = useState(
!isUndefined(
(data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern
)
);
const [showDatabaseFilter, setShowDatabaseFilter] = useState(
!isUndefined(
(data?.sourceConfig.config as ConfigClass)?.databaseFilterPattern
)
);
const [showSchemaFilter, setShowSchemaFilter] = useState(
!isUndefined(
(data?.sourceConfig.config as ConfigClass)?.schemaFilterPattern
)
);
const [showTableFilter, setShowTableFilter] = useState(
!isUndefined((data?.sourceConfig.config as ConfigClass)?.tableFilterPattern)
);
const [showTopicFilter, setShowTopicFilter] = useState(
!isUndefined((data?.sourceConfig.config as ConfigClass)?.topicFilterPattern)
);
const [showChartFilter, setShowChartFilter] = useState(
!isUndefined((data?.sourceConfig.config as ConfigClass)?.chartFilterPattern)
);
const [showPipelineFilter, setShowPipelineFilter] = useState(
!isUndefined(
(data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern
)
);
const [showMlModelFilter, setShowMlModelFilter] = useState(
!isUndefined(
(data?.sourceConfig.config as ConfigClass)?.mlModelFilterPattern
)
);
const configData = useMemo(
() => (data?.sourceConfig.config as DbtPipelineClass)?.dbtConfigSource,
[data]
);
const [dbtConfigSource, setDbtConfigSource] = useState<
ModifiedDbtConfig | undefined
>(configData as DbtConfig);
const sourceTypeData = useMemo(
() => getSourceTypeFromConfig(configData as DbtConfig | undefined),
[configData]
);
const [dbtConfigSourceType, setDbtConfigSourceType] = useState<DBT_SOURCES>(
sourceTypeData.sourceType
const initialState: AddIngestionState = useMemo(
() => ({
saveState: 'initial',
showDeployModal: false,
ingestionName:
data?.name ?? getIngestionName(serviceData.name, pipelineType),
ingestSampleData: sourceConfig?.generateSampleData ?? true,
useFqnFilter: sourceConfig?.useFqnForFiltering ?? false,
databaseServiceNames: sourceConfig?.dbServiceNames ?? [],
description: data?.description ?? '',
repeatFrequency:
data?.airflowConfig.scheduleInterval ??
getIngestionFrequency(pipelineType),
showDashboardFilter: !isUndefined(sourceConfig?.dashboardFilterPattern),
showDatabaseFilter: !isUndefined(sourceConfig?.databaseFilterPattern),
showSchemaFilter: !isUndefined(sourceConfig?.schemaFilterPattern),
showTableFilter: !isUndefined(sourceConfig?.tableFilterPattern),
showTopicFilter: !isUndefined(sourceConfig?.topicFilterPattern),
showChartFilter: !isUndefined(sourceConfig?.chartFilterPattern),
showPipelineFilter: !isUndefined(sourceConfig?.pipelineFilterPattern),
showMlModelFilter: !isUndefined(sourceConfig?.mlModelFilterPattern),
dbtConfigSource: configData as ModifiedDbtConfig,
gcsConfigType: showDBTConfig ? sourceTypeData.gcsType : undefined,
chartFilterPattern:
sourceConfig?.chartFilterPattern ?? INITIAL_FILTER_PATTERN,
dbtConfigSourceType: sourceTypeData.sourceType || DBT_SOURCES.local,
markDeletedTables: isDatabaseService
? Boolean(sourceConfig?.markDeletedTables ?? true)
: undefined,
dashboardFilterPattern:
sourceConfig?.dashboardFilterPattern ?? INITIAL_FILTER_PATTERN,
databaseFilterPattern:
sourceConfig?.databaseFilterPattern ?? INITIAL_FILTER_PATTERN,
markAllDeletedTables: isDatabaseService
? Boolean(sourceConfig?.markAllDeletedTables ?? false)
: undefined,
includeView: Boolean(sourceConfig?.includeViews),
includeTags: Boolean(sourceConfig?.includeTags),
includeLineage: Boolean(sourceConfig?.includeLineage ?? true),
enableDebugLog: data?.loggerLevel === LogLevels.Debug,
profileSample: sourceConfig?.profileSample,
profileSampleType:
sourceConfig?.profileSampleType || ProfileSampleType.Percentage,
threadCount: sourceConfig?.threadCount ?? 5,
timeoutSeconds: sourceConfig?.timeoutSeconds ?? 43200,
schemaFilterPattern:
sourceConfig?.schemaFilterPattern ?? INITIAL_FILTER_PATTERN,
tableFilterPattern:
sourceConfig?.tableFilterPattern ?? INITIAL_FILTER_PATTERN,
topicFilterPattern:
sourceConfig?.topicFilterPattern ?? INITIAL_FILTER_PATTERN,
pipelineFilterPattern:
sourceConfig?.pipelineFilterPattern ?? INITIAL_FILTER_PATTERN,
mlModelFilterPattern:
sourceConfig?.mlModelFilterPattern ?? INITIAL_FILTER_PATTERN,
queryLogDuration: sourceConfig?.queryLogDuration ?? 1,
stageFileLocation: sourceConfig?.stageFileLocation ?? '/tmp/query_log',
resultLimit: sourceConfig?.resultLimit ?? 1000,
metadataToESConfig: undefined,
}),
[]
);
const [gcsConfigType, setGcsConfigType] = useState<GCS_CONFIG | undefined>(
showDBTConfig ? sourceTypeData.gcsType : undefined
);
const [markDeletedTables, setMarkDeletedTables] = useState(
isDatabaseService
? Boolean(
(data?.sourceConfig.config as ConfigClass)?.markDeletedTables ?? true
)
: undefined
);
const [markAllDeletedTables, setMarkAllDeletedTables] = useState(
isDatabaseService
? Boolean(
(data?.sourceConfig.config as ConfigClass)?.markAllDeletedTables ??
false
)
: undefined
);
const [includeView, setIncludeView] = useState(
Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews)
);
const [includeTag, setIncludeTags] = useState(
Boolean((data?.sourceConfig.config as ConfigClass)?.includeTags)
);
const [includeLineage, setIncludeLineage] = useState(
Boolean((data?.sourceConfig.config as ConfigClass)?.includeLineage ?? true)
);
const [enableDebugLog, setEnableDebugLog] = useState(
data?.loggerLevel === LogLevels.Debug
);
const [profileSample, setProfileSample] = useState(
(data?.sourceConfig.config as ConfigClass)?.profileSample
);
const [profileSampleType, setProfileSampleType] = useState(
(data?.sourceConfig.config as ConfigClass)?.profileSampleType ||
ProfileSampleType.Percentage
);
const [threadCount, setThreadCount] = useState(
(data?.sourceConfig.config as ConfigClass)?.threadCount ?? 5
);
const [timeoutSeconds, setTimeoutSeconds] = useState(
(data?.sourceConfig.config as ConfigClass)?.timeoutSeconds ?? 43200
);
const [dashboardFilterPattern, setDashboardFilterPattern] =
useState<FilterPattern>(
(data?.sourceConfig.config as ConfigClass)?.dashboardFilterPattern ??
INITIAL_FILTER_PATTERN
);
const [databaseFilterPattern, setDatabaseFilterPattern] =
useState<FilterPattern>(
(data?.sourceConfig.config as ConfigClass)?.databaseFilterPattern ??
INITIAL_FILTER_PATTERN
);
const [schemaFilterPattern, setSchemaFilterPattern] = useState<FilterPattern>(
(data?.sourceConfig.config as ConfigClass)?.schemaFilterPattern ??
INITIAL_FILTER_PATTERN
);
const [tableFilterPattern, setTableFilterPattern] = useState<FilterPattern>(
(data?.sourceConfig.config as ConfigClass)?.tableFilterPattern ??
INITIAL_FILTER_PATTERN
);
const [topicFilterPattern, setTopicFilterPattern] = useState<FilterPattern>(
(data?.sourceConfig.config as ConfigClass)?.topicFilterPattern ??
INITIAL_FILTER_PATTERN
);
const [chartFilterPattern, setChartFilterPattern] = useState<FilterPattern>(
(data?.sourceConfig.config as ConfigClass)?.chartFilterPattern ??
INITIAL_FILTER_PATTERN
);
const [pipelineFilterPattern, setPipelineFilterPattern] =
useState<FilterPattern>(
(data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern ??
INITIAL_FILTER_PATTERN
);
const [state, dispatch] = useReducer<
Reducer<AddIngestionState, Partial<AddIngestionState>>
>(reducerWithoutAction, initialState);
const [mlModelFilterPattern, setMlModelFilterPattern] =
useState<FilterPattern>(
(data?.sourceConfig.config as ConfigClass)?.mlModelFilterPattern ??
INITIAL_FILTER_PATTERN
);
const [saveState, setSaveState] = useState<LoadingState>('initial');
const [showDeployModal, setShowDeployModal] = useState(false);
const [queryLogDuration, setQueryLogDuration] = useState<number>(
(data?.sourceConfig.config as ConfigClass)?.queryLogDuration ?? 1
const handleStateChange = useCallback(
(newState: Partial<AddIngestionState>) => {
dispatch(newState);
},
[]
);
const [stageFileLocation, setStageFileLocation] = useState<string>(
(data?.sourceConfig.config as ConfigClass)?.stageFileLocation ??
'/tmp/query_log'
);
const [resultLimit, setResultLimit] = useState<number>(
(data?.sourceConfig.config as ConfigClass)?.resultLimit ?? 1000
);
const [metadataToESConfig, SetMetadataToESConfig] = useState<ConfigClass>();
const usageIngestionType = useMemo(() => {
return (
(data?.sourceConfig.config as ConfigClass)?.type ??
ConfigType.DatabaseUsage
);
}, [data]);
const lineageIngestionType = useMemo(() => {
return (
(data?.sourceConfig.config as ConfigClass)?.type ??
ConfigType.DatabaseLineage
);
}, [data]);
const profilerIngestionType = useMemo(() => {
return (
(data?.sourceConfig.config as ConfigClass)?.type ?? ConfigType.Profiler
);
}, [data]);
const handleMetadataToESConfig = (data: ConfigClass) => {
SetMetadataToESConfig(data);
handleStateChange({
metadataToESConfig: data,
});
};
const getIncludeValue = (value: Array<string>, type: FilterPatternEnum) => {
switch (type) {
case FilterPatternEnum.DASHBOARD:
setDashboardFilterPattern({
...dashboardFilterPattern,
includes: value,
});
const pattern = getFilterTypes(type);
break;
case FilterPatternEnum.DATABASE:
setDatabaseFilterPattern({ ...databaseFilterPattern, includes: value });
break;
case FilterPatternEnum.SCHEMA:
setSchemaFilterPattern({ ...schemaFilterPattern, includes: value });
break;
case FilterPatternEnum.TABLE:
setTableFilterPattern({ ...tableFilterPattern, includes: value });
break;
case FilterPatternEnum.TOPIC:
setTopicFilterPattern({ ...topicFilterPattern, includes: value });
break;
case FilterPatternEnum.CHART:
setChartFilterPattern({ ...chartFilterPattern, includes: value });
break;
case FilterPatternEnum.PIPELINE:
setPipelineFilterPattern({ ...pipelineFilterPattern, includes: value });
break;
case FilterPatternEnum.MLMODEL:
setMlModelFilterPattern({ ...mlModelFilterPattern, includes: value });
break;
}
return handleStateChange({
[pattern]: {
...(state[pattern] as AddIngestionState),
includes: value,
},
});
};
const getExcludeValue = (value: Array<string>, type: FilterPatternEnum) => {
switch (type) {
case FilterPatternEnum.DASHBOARD:
setDashboardFilterPattern({
...dashboardFilterPattern,
excludes: value,
});
const pattern = getFilterTypes(type);
break;
case FilterPatternEnum.DATABASE:
setDatabaseFilterPattern({ ...databaseFilterPattern, excludes: value });
break;
case FilterPatternEnum.SCHEMA:
setSchemaFilterPattern({ ...schemaFilterPattern, excludes: value });
break;
case FilterPatternEnum.TABLE:
setTableFilterPattern({ ...tableFilterPattern, excludes: value });
break;
case FilterPatternEnum.TOPIC:
setTopicFilterPattern({ ...topicFilterPattern, excludes: value });
break;
case FilterPatternEnum.CHART:
setChartFilterPattern({ ...chartFilterPattern, excludes: value });
break;
case FilterPatternEnum.PIPELINE:
setPipelineFilterPattern({ ...pipelineFilterPattern, excludes: value });
break;
case FilterPatternEnum.MLMODEL:
setMlModelFilterPattern({ ...mlModelFilterPattern, excludes: value });
break;
}
return handleStateChange({
[pattern]: {
...(state[pattern] as AddIngestionState),
excludes: value,
},
});
};
const handleShowFilter = (value: boolean, type: FilterPatternEnum) => {
switch (type) {
case FilterPatternEnum.DASHBOARD:
setShowDashboardFilter(value);
break;
case FilterPatternEnum.DATABASE:
setShowDatabaseFilter(value);
break;
case FilterPatternEnum.SCHEMA:
setShowSchemaFilter(value);
break;
case FilterPatternEnum.TABLE:
setShowTableFilter(value);
break;
case FilterPatternEnum.TOPIC:
setShowTopicFilter(value);
break;
case FilterPatternEnum.CHART:
setShowChartFilter(value);
break;
case FilterPatternEnum.PIPELINE:
setShowPipelineFilter(value);
break;
case FilterPatternEnum.MLMODEL:
setShowMlModelFilter(value);
break;
}
};
// It takes a boolean and a string, and returns a function that takes an object and returns a new
const handleShowFilter = (value: boolean, showFilter: string) =>
handleStateChange({
[showFilter]: value,
});
const handleNext = () => {
let nextStep;
@ -447,12 +298,39 @@ const AddIngestion = ({
};
const getMetadataIngestionFields = () => {
const {
chartFilterPattern,
dashboardFilterPattern,
databaseFilterPattern,
databaseServiceNames,
includeLineage,
includeTags,
includeView,
ingestSampleData,
markAllDeletedTables,
markDeletedTables,
mlModelFilterPattern,
pipelineFilterPattern,
schemaFilterPattern,
showChartFilter,
showDashboardFilter,
showDatabaseFilter,
showMlModelFilter,
showPipelineFilter,
showSchemaFilter,
showTableFilter,
showTopicFilter,
tableFilterPattern,
topicFilterPattern,
useFqnFilter,
} = state;
switch (serviceCategory) {
case ServiceCategory.DATABASE_SERVICES: {
return {
useFqnForFiltering: useFqnFilter,
includeViews: includeView,
includeTags: includeTag,
includeTags: includeTags,
databaseFilterPattern: getFilterPatternData(
databaseFilterPattern,
showDatabaseFilter
@ -465,8 +343,8 @@ const AddIngestion = ({
tableFilterPattern,
showTableFilter
),
markDeletedTables,
markAllDeletedTables,
markDeletedTables: markDeletedTables,
markAllDeletedTables: markAllDeletedTables,
type: ConfigType.DatabaseMetadata,
};
}
@ -520,19 +398,37 @@ const AddIngestion = ({
};
const getConfigData = (type: PipelineType): ConfigClass => {
const {
databaseFilterPattern,
dbtConfigSource,
ingestSampleData,
metadataToESConfig,
profileSample,
profileSampleType,
queryLogDuration,
resultLimit,
schemaFilterPattern,
showDatabaseFilter,
showSchemaFilter,
showTableFilter,
stageFileLocation,
tableFilterPattern,
threadCount,
timeoutSeconds,
} = state;
switch (type) {
case PipelineType.Usage: {
return {
queryLogDuration,
resultLimit,
stageFileLocation,
queryLogDuration: queryLogDuration,
resultLimit: resultLimit,
stageFileLocation: stageFileLocation,
type: usageIngestionType,
};
}
case PipelineType.Lineage: {
return {
queryLogDuration,
resultLimit,
queryLogDuration: queryLogDuration,
resultLimit: resultLimit,
type: lineageIngestionType,
};
}
@ -587,6 +483,7 @@ const AddIngestion = ({
};
const createNewIngestion = () => {
const { repeatFrequency, enableDebugLog, ingestionName } = state;
const ingestionDetails: CreateIngestionPipeline = {
airflowConfig: {
scheduleInterval: isEmpty(repeatFrequency)
@ -632,6 +529,7 @@ const AddIngestion = ({
};
const updateIngestion = () => {
const { repeatFrequency, enableDebugLog } = state;
if (data) {
const updatedData: IngestionPipeline = {
...data,
@ -704,7 +602,7 @@ const AddIngestion = ({
return (
<span>
<span className="tw-mr-1 tw-font-semibold">
&quot;{ingestionName}&quot;
&quot;{state.ingestionName}&quot;
</span>
<span>
{status === FormSubmitType.ADD ? createMessage : updateMessage}
@ -739,86 +637,31 @@ const AddIngestion = ({
<div className="tw-pt-7">
{activeIngestionStep === 1 && (
<ConfigureIngestion
chartFilterPattern={chartFilterPattern}
dashboardFilterPattern={dashboardFilterPattern}
databaseFilterPattern={databaseFilterPattern}
databaseServiceNames={databaseServiceNames}
description={description}
enableDebugLog={enableDebugLog}
data={state}
formType={status}
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleDatasetServiceName={(val) => setDatabaseServiceNames(val)}
handleDescription={(val) => setDescription(val)}
handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)}
handleIncludeLineage={() => setIncludeLineage((pre) => !pre)}
handleIncludeTags={() => setIncludeTags((pre) => !pre)}
handleIncludeView={() => setIncludeView((pre) => !pre)}
handleIngestSampleData={() => setIngestSampleData((pre) => !pre)}
handleIngestionName={(val) => setIngestionName(val)}
handleMarkAllDeletedTables={() =>
setMarkAllDeletedTables((pre) => !pre)
}
handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)}
handleProfileSample={(val) => setProfileSample(val)}
handleProfileSampleType={(val) => setProfileSampleType(val)}
handleQueryLogDuration={(val) => setQueryLogDuration(val)}
handleResultLimit={setResultLimit}
handleShowFilter={handleShowFilter}
handleStageFileLocation={(val) => setStageFileLocation(val)}
handleThreadCount={setThreadCount}
handleTimeoutSeconds={setTimeoutSeconds}
includeLineage={includeLineage}
includeTags={includeTag}
includeView={includeView}
ingestSampleData={ingestSampleData}
ingestionName={ingestionName}
markAllDeletedTables={markAllDeletedTables}
markDeletedTables={markDeletedTables}
mlModelFilterPattern={mlModelFilterPattern}
pipelineFilterPattern={pipelineFilterPattern}
pipelineType={pipelineType}
profileSample={profileSample}
profileSampleType={profileSampleType}
queryLogDuration={queryLogDuration}
resultLimit={resultLimit}
schemaFilterPattern={schemaFilterPattern}
serviceCategory={serviceCategory}
showChartFilter={showChartFilter}
showDashboardFilter={showDashboardFilter}
showDatabaseFilter={showDatabaseFilter}
showMlModelFilter={showMlModelFilter}
showPipelineFilter={showPipelineFilter}
showSchemaFilter={showSchemaFilter}
showTableFilter={showTableFilter}
showTopicFilter={showTopicFilter}
stageFileLocation={stageFileLocation}
tableFilterPattern={tableFilterPattern}
threadCount={threadCount}
timeoutSeconds={timeoutSeconds}
topicFilterPattern={topicFilterPattern}
useFqnFilter={useFqnFilter}
onCancel={handleCancelClick}
onChange={handleStateChange}
onNext={handleNext}
onUseFqnFilterClick={() => setUseFqnFilter((pre) => !pre)}
/>
)}
{activeIngestionStep === 2 && (
<DBTConfigFormBuilder
cancelText={t('label.cancel')}
data={dbtConfigSource || {}}
data={state}
formType={status}
gcsType={gcsConfigType}
handleGcsTypeChange={(type) => setGcsConfigType(type)}
handleIngestionName={(val) => setIngestionName(val)}
handleSourceChange={(src) => setDbtConfigSourceType(src)}
ingestionName={ingestionName}
okText={t('label.next')}
source={dbtConfigSourceType}
onCancel={handleCancelClick}
onChange={handleStateChange}
onSubmit={(dbtConfigData) => {
setDbtConfigSource(dbtConfigData);
handleStateChange({
dbtConfigSource: dbtConfigData,
});
handleNext();
}}
/>
@ -834,18 +677,16 @@ const AddIngestion = ({
{activeIngestionStep === 4 && (
<ScheduleInterval
handleRepeatFrequencyChange={(value: string) =>
setRepeatFrequency(value)
}
includePeriodOptions={
pipelineType === PipelineType.DataInsight ? ['day'] : undefined
}
repeatFrequency={repeatFrequency}
repeatFrequency={state.repeatFrequency}
status={saveState}
submitButtonLabel={
isUndefined(data) ? t('label.add-deploy') : t('label.submit')
}
onBack={handlePrev}
onChange={handleStateChange}
onDeploy={handleScheduleIntervalDeployClick}
/>
)}
@ -854,7 +695,7 @@ const AddIngestion = ({
<SuccessScreen
handleDeployClick={handleDeployClick}
handleViewServiceClick={handleViewServiceClick}
name={ingestionName}
name={state.ingestionName}
showDeployButton={showDeployButton}
showIngestionButton={false}
state={status}
@ -864,7 +705,7 @@ const AddIngestion = ({
<DeployIngestionLoaderModal
action={ingestionAction}
ingestionName={ingestionName}
ingestionName={state.ingestionName}
isDeployed={isIngestionDeployed}
isIngestionCreated={isIngestionCreated}
progress={ingestionProgress}

143
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
View File

@ -15,9 +15,14 @@ import { findAllByText, findByTestId, render } from '@testing-library/react';
import React from 'react';
import { FormSubmitType } from '../../../enums/form.enum';
import { ServiceCategory } from '../../../enums/service.enum';
import { PipelineType } from '../../../generated/entity/services/ingestionPipelines/ingestionPipeline';
import { ProfileSampleType } from '../../../generated/metadataIngestion/databaseServiceProfilerPipeline';
import { ConfigureIngestionProps } from '../addIngestion.interface';
import {
PipelineType,
ProfileSampleType,
} from '../../../generated/entity/services/ingestionPipelines/ingestionPipeline';
import {
AddIngestionState,
ConfigureIngestionProps,
} from '../addIngestion.interface';
import ConfigureIngestion from './ConfigureIngestion';
jest.mock('../../common/FilterPattern/FilterPattern', () => {
@ -31,86 +36,74 @@ jest.mock('../../common/toggle-switch/ToggleSwitchV1', () => {
});
const mockConfigureIngestion: ConfigureIngestionProps = {
ingestionName: '',
databaseFilterPattern: {
includes: [],
excludes: [],
},
dashboardFilterPattern: {
includes: [],
excludes: [],
},
chartFilterPattern: {
includes: [],
excludes: [],
},
schemaFilterPattern: {
includes: [],
excludes: [],
},
tableFilterPattern: {
includes: [],
excludes: [],
},
topicFilterPattern: {
includes: [],
excludes: [],
},
pipelineFilterPattern: {
includes: [],
excludes: [],
},
mlModelFilterPattern: {
includes: [],
excludes: [],
},
includeLineage: false,
includeView: false,
includeTags: false,
pipelineType: PipelineType.Metadata,
formType: FormSubmitType.EDIT,
queryLogDuration: 1,
resultLimit: 100,
stageFileLocation: '',
markDeletedTables: false,
showDashboardFilter: false,
showDatabaseFilter: false,
showSchemaFilter: false,
showTableFilter: false,
showTopicFilter: false,
showChartFilter: false,
showPipelineFilter: false,
showMlModelFilter: false,
handleIncludeLineage: jest.fn(),
handleIncludeView: jest.fn(),
handleIncludeTags: jest.fn(),
handleIngestionName: jest.fn(),
handleMarkDeletedTables: jest.fn(),
handleProfileSample: jest.fn(),
handleQueryLogDuration: jest.fn(),
handleResultLimit: jest.fn(),
handleStageFileLocation: jest.fn(),
getIncludeValue: jest.fn(),
getExcludeValue: jest.fn(),
handleShowFilter: jest.fn(),
onCancel: jest.fn(),
onNext: jest.fn(),
handleProfileSampleType: jest.fn(),
profileSample: 1,
profileSampleType: ProfileSampleType.Percentage,
serviceCategory: ServiceCategory.DATABASE_SERVICES,
enableDebugLog: false,
handleEnableDebugLog: jest.fn(),
ingestSampleData: false,
handleIngestSampleData: jest.fn(),
databaseServiceNames: [''],
handleDatasetServiceName: jest.fn(),
threadCount: 5,
handleThreadCount: jest.fn(),
timeoutSeconds: 43200,
handleTimeoutSeconds: jest.fn(),
useFqnFilter: false,
onUseFqnFilterClick: jest.fn(),
onChange: jest.fn(),
data: {
ingestionName: '',
databaseFilterPattern: {
includes: [],
excludes: [],
},
dashboardFilterPattern: {
includes: [],
excludes: [],
},
chartFilterPattern: {
includes: [],
excludes: [],
},
schemaFilterPattern: {
includes: [],
excludes: [],
},
tableFilterPattern: {
includes: [],
excludes: [],
},
topicFilterPattern: {
includes: [],
excludes: [],
},
pipelineFilterPattern: {
includes: [],
excludes: [],
},
mlModelFilterPattern: {
includes: [],
excludes: [],
},
includeLineage: false,
includeView: false,
includeTags: false,
queryLogDuration: 1,
resultLimit: 100,
stageFileLocation: '',
markDeletedTables: false,
showDashboardFilter: false,
showDatabaseFilter: false,
showSchemaFilter: false,
showTableFilter: false,
showTopicFilter: false,
showChartFilter: false,
showPipelineFilter: false,
showMlModelFilter: false,
profileSample: 1,
profileSampleType: ProfileSampleType.Percentage,
enableDebugLog: false,
ingestSampleData: false,
databaseServiceNames: [''],
threadCount: 5,
timeoutSeconds: 43200,
useFqnFilter: false,
} as unknown as AddIngestionState,
};
describe('Test ConfigureIngestion component', () => {

289
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
View File

@ -13,7 +13,7 @@
import { Form, InputNumber, Select, Typography } from 'antd';
import { isNil } from 'lodash';
import React, { Fragment, useRef } from 'react';
import React, { Fragment, useMemo, useRef } from 'react';
import { useTranslation } from 'react-i18next';
import { PROFILE_SAMPLE_OPTIONS } from '../../../constants/profiler.constant';
import { FilterPatternEnum } from '../../../enums/filterPattern.enum';
@ -29,78 +29,174 @@ import { EditorContentRef } from '../../common/rich-text-editor/RichTextEditor.i
import ToggleSwitchV1 from '../../common/toggle-switch/ToggleSwitchV1';
import { Field } from '../../Field/Field';
import SliderWithInput from '../../SliderWithInput/SliderWithInput';
import { ConfigureIngestionProps } from '../addIngestion.interface';
import {
AddIngestionState,
ConfigureIngestionProps,
ShowFilter,
} from '../addIngestion.interface';
const ConfigureIngestion = ({
ingestionName,
description = '',
databaseServiceNames,
databaseFilterPattern,
dashboardFilterPattern,
schemaFilterPattern,
tableFilterPattern,
topicFilterPattern,
chartFilterPattern,
pipelineFilterPattern,
mlModelFilterPattern,
includeLineage,
includeView,
includeTags,
markDeletedTables,
markAllDeletedTables,
serviceCategory,
pipelineType,
showDatabaseFilter,
ingestSampleData,
showDashboardFilter,
showSchemaFilter,
showTableFilter,
showTopicFilter,
showChartFilter,
showPipelineFilter,
showMlModelFilter,
queryLogDuration,
stageFileLocation,
threadCount,
timeoutSeconds,
resultLimit,
enableDebugLog,
profileSample,
handleEnableDebugLog,
data,
formType,
getExcludeValue,
getIncludeValue,
handleIngestionName,
handleDescription,
handleShowFilter,
handleIncludeLineage,
handleIncludeView,
handleIncludeTags,
handleMarkDeletedTables,
handleMarkAllDeletedTables,
handleIngestSampleData,
handleDatasetServiceName,
handleQueryLogDuration,
handleProfileSample,
handleStageFileLocation,
handleResultLimit,
handleThreadCount,
handleTimeoutSeconds,
useFqnFilter,
onUseFqnFilterClick,
onCancel,
onChange,
onNext,
formType,
profileSampleType,
handleProfileSampleType,
pipelineType,
serviceCategory,
}: ConfigureIngestionProps) => {
const { t } = useTranslation();
const markdownRef = useRef<EditorContentRef>();
const {
chartFilterPattern,
dashboardFilterPattern,
databaseFilterPattern,
databaseServiceNames,
description,
enableDebugLog,
includeLineage,
includeTags,
includeView,
ingestionName,
ingestSampleData,
markAllDeletedTables,
markDeletedTables,
mlModelFilterPattern,
pipelineFilterPattern,
profileSample,
profileSampleType,
queryLogDuration,
resultLimit,
schemaFilterPattern,
showChartFilter,
showDashboardFilter,
showDatabaseFilter,
showMlModelFilter,
showPipelineFilter,
showSchemaFilter,
showTableFilter,
showTopicFilter,
stageFileLocation,
tableFilterPattern,
threadCount,
timeoutSeconds,
topicFilterPattern,
useFqnFilter,
} = useMemo(
() => ({
chartFilterPattern: data.chartFilterPattern,
dashboardFilterPattern: data.dashboardFilterPattern,
databaseFilterPattern: data.databaseFilterPattern,
databaseServiceNames: data.databaseServiceNames,
description: data.description,
enableDebugLog: data.enableDebugLog,
includeLineage: data.includeLineage,
includeTags: data.includeTags,
includeView: data.includeView,
ingestionName: data.ingestionName,
ingestSampleData: data.ingestSampleData,
markAllDeletedTables: data.markAllDeletedTables,
markDeletedTables: data.markDeletedTables,
mlModelFilterPattern: data.mlModelFilterPattern,
pipelineFilterPattern: data.pipelineFilterPattern,
profileSample: data.profileSample,
profileSampleType: data.profileSampleType,
queryLogDuration: data.queryLogDuration,
resultLimit: data.resultLimit,
schemaFilterPattern: data.schemaFilterPattern,
showChartFilter: data.showChartFilter,
showDashboardFilter: data.showDashboardFilter,
showDatabaseFilter: data.showDatabaseFilter,
showMlModelFilter: data.showMlModelFilter,
showPipelineFilter: data.showPipelineFilter,
showSchemaFilter: data.showSchemaFilter,
showTableFilter: data.showTableFilter,
showTopicFilter: data.showTopicFilter,
stageFileLocation: data.stageFileLocation,
tableFilterPattern: data.tableFilterPattern,
threadCount: data.threadCount,
timeoutSeconds: data.timeoutSeconds,
topicFilterPattern: data.topicFilterPattern,
useFqnFilter: data.useFqnFilter,
}),
[data]
);
const toggleField = (field: keyof AddIngestionState) =>
onChange({ [field]: !data[field] });
const handleValueParseInt =
(property: keyof AddIngestionState) =>
(event: React.ChangeEvent<HTMLInputElement>) =>
onChange({
[property]: parseInt(event.target.value),
});
const handleValueChange =
(property: keyof AddIngestionState) =>
(event: React.ChangeEvent<HTMLInputElement>) =>
onChange({
[property]: event.target.value,
});
const handleProfileSample = (profileSample: number | undefined | null) =>
onChange({
profileSample: profileSample ?? undefined,
});
const handleProfileSampleTypeChange = (value: ProfileSampleType) => {
handleProfileSampleType(value);
onChange({
profileSampleType: value,
});
handleProfileSample(undefined);
};
const handleDashBoardServiceNames = (inputValue: string) => {
const separator = ',';
const databaseNames = inputValue.includes(separator)
? inputValue.split(separator)
: Array(inputValue);
if (databaseNames) {
onChange({
databaseServiceNames: databaseNames,
});
}
};
const handleEnableDebugLogCheck = () => toggleField('enableDebugLog');
const handleIncludeLineage = () => toggleField('includeLineage');
const handleIncludeTags = () => toggleField('includeTags');
const handleIncludeViewToggle = () => toggleField('includeView');
const handleIngestSampleToggle = () => toggleField('ingestSampleData');
const handleMarkAllDeletedTables = () => toggleField('markAllDeletedTables');
const handleMarkDeletedTables = () => toggleField('markDeletedTables');
const handleFqnFilter = () => toggleField('useFqnFilter');
const handleQueryLogDuration = handleValueParseInt('queryLogDuration');
const handleResultLimit = handleValueParseInt('resultLimit');
const handleStageFileLocation = handleValueChange('stageFileLocation');
const handleThreadCount = handleValueParseInt('threadCount');
const handleTimeoutSeconds = handleValueParseInt('timeoutSeconds');
const handleIngestionName = handleValueChange('ingestionName');
const getIngestSampleToggle = (label: string, desc: string) => {
return (
<>
@ -109,7 +205,7 @@ const ConfigureIngestion = ({
<label>{label}</label>
<ToggleSwitchV1
checked={ingestSampleData}
handleCheck={handleIngestSampleData}
handleCheck={handleIngestSampleToggle}
testId="ingest-sample-data"
/>
</div>
@ -127,7 +223,7 @@ const ConfigureIngestion = ({
<label>{t('label.enable-debug-log')}</label>
<ToggleSwitchV1
checked={enableDebugLog}
handleCheck={handleEnableDebugLog}
handleCheck={handleEnableDebugLogCheck}
testId="enable-debug-log"
/>
</div>
@ -169,9 +265,7 @@ const ConfigureIngestion = ({
</Typography.Paragraph>
<SliderWithInput
value={profileSample || 100}
onChange={(value: number | null) =>
handleProfileSample(value ?? undefined)
}
onChange={handleProfileSample}
/>
</>
)}
@ -188,7 +282,7 @@ const ConfigureIngestion = ({
name: t('label.row-count-lowercase'),
})}
value={profileSample}
onChange={(value) => handleProfileSample(value ?? undefined)}
onChange={handleProfileSample}
/>
</>
)}
@ -212,7 +306,7 @@ const ConfigureIngestion = ({
placeholder="5"
type="number"
value={threadCount}
onChange={(e) => handleThreadCount(parseInt(e.target.value))}
onChange={handleThreadCount}
/>
</div>
);
@ -233,7 +327,7 @@ const ConfigureIngestion = ({
placeholder="43200"
type="number"
value={timeoutSeconds}
onChange={(e) => handleTimeoutSeconds(parseInt(e.target.value))}
onChange={handleTimeoutSeconds}
/>
</div>
);
@ -250,7 +344,7 @@ const ConfigureIngestion = ({
</label>
<ToggleSwitchV1
checked={includeView}
handleCheck={handleIncludeView}
handleCheck={handleIncludeViewToggle}
testId="include-views"
/>
</div>
@ -286,11 +380,7 @@ const ConfigureIngestion = ({
<label>{t('label.mark-deleted-table-plural')}</label>
<ToggleSwitchV1
checked={markDeletedTables}
handleCheck={() => {
if (handleMarkDeletedTables) {
handleMarkDeletedTables();
}
}}
handleCheck={handleMarkDeletedTables}
testId="mark-deleted"
/>
</div>
@ -306,11 +396,7 @@ const ConfigureIngestion = ({
<label>{t('label.mark-all-deleted-table-plural')}</label>
<ToggleSwitchV1
checked={markAllDeletedTables}
handleCheck={() => {
if (handleMarkAllDeletedTables) {
handleMarkAllDeletedTables();
}
}}
handleCheck={handleMarkAllDeletedTables}
testId="mark-deleted-filter-only"
/>
</div>
@ -357,7 +443,7 @@ const ConfigureIngestion = ({
<label>{t('label.use-fqn-for-filtering')}</label>
<ToggleSwitchV1
checked={useFqnFilter}
handleCheck={onUseFqnFilterClick}
handleCheck={handleFqnFilter}
testId="include-lineage"
/>
</div>
@ -369,18 +455,6 @@ const ConfigureIngestion = ({
);
};
const handleDashBoardServiceNames = (inputValue: string) => {
const separator = ',';
const databaseNames = inputValue.includes(separator)
? inputValue.split(separator)
: Array(inputValue);
if (databaseNames) {
handleDatasetServiceName(databaseNames);
}
};
const getDashboardDBServiceName = () => {
return (
<Field>
@ -413,7 +487,7 @@ const ConfigureIngestion = ({
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.DATABASE)
handleShowFilter(value, ShowFilter.showDatabaseFilter)
}
includePattern={databaseFilterPattern?.includes ?? []}
type={FilterPatternEnum.DATABASE}
@ -424,7 +498,7 @@ const ConfigureIngestion = ({
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.SCHEMA)
handleShowFilter(value, ShowFilter.showSchemaFilter)
}
includePattern={schemaFilterPattern?.includes ?? []}
type={FilterPatternEnum.SCHEMA}
@ -435,7 +509,7 @@ const ConfigureIngestion = ({
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.TABLE)
handleShowFilter(value, ShowFilter.showTableFilter)
}
includePattern={tableFilterPattern?.includes ?? []}
showSeparator={false}
@ -465,7 +539,7 @@ const ConfigureIngestion = ({
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.DASHBOARD)
handleShowFilter(value, ShowFilter.showDashboardFilter)
}
includePattern={dashboardFilterPattern.includes ?? []}
type={FilterPatternEnum.DASHBOARD}
@ -476,7 +550,7 @@ const ConfigureIngestion = ({
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.CHART)
handleShowFilter(value, ShowFilter.showChartFilter)
}
includePattern={chartFilterPattern.includes ?? []}
showSeparator={false}
@ -497,7 +571,7 @@ const ConfigureIngestion = ({
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.TOPIC)
handleShowFilter(value, ShowFilter.showTopicFilter)
}
includePattern={topicFilterPattern.includes ?? []}
showSeparator={false}
@ -522,7 +596,7 @@ const ConfigureIngestion = ({
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.PIPELINE)
handleShowFilter(value, ShowFilter.showPipelineFilter)
}
includePattern={pipelineFilterPattern.includes ?? []}
showSeparator={false}
@ -542,7 +616,7 @@ const ConfigureIngestion = ({
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.MLMODEL)
handleShowFilter(value, ShowFilter.showMlModelFilter)
}
includePattern={mlModelFilterPattern.includes ?? []}
showSeparator={false}
@ -574,7 +648,7 @@ const ConfigureIngestion = ({
name="name"
type="text"
value={ingestionName}
onChange={(e) => handleIngestionName(e.target.value)}
onChange={handleIngestionName}
/>
{getSeparator('')}
</Field>
@ -602,7 +676,7 @@ const ConfigureIngestion = ({
name="query-log-duration"
type="number"
value={queryLogDuration}
onChange={(e) => handleQueryLogDuration(parseInt(e.target.value))}
onChange={handleQueryLogDuration}
/>
{getSeparator('')}
</Field>
@ -622,7 +696,7 @@ const ConfigureIngestion = ({
name="stage-file-location"
type="text"
value={stageFileLocation}
onChange={(e) => handleStageFileLocation(e.target.value)}
onChange={handleStageFileLocation}
/>
{getSeparator('')}
</Field>
@ -642,7 +716,7 @@ const ConfigureIngestion = ({
name="result-limit"
type="number"
value={resultLimit}
onChange={(e) => handleResultLimit(parseInt(e.target.value))}
onChange={handleResultLimit}
/>
{getSeparator('')}
</Field>
@ -670,7 +744,7 @@ const ConfigureIngestion = ({
name="query-log-duration"
type="number"
value={queryLogDuration}
onChange={(e) => handleQueryLogDuration(parseInt(e.target.value))}
onChange={handleQueryLogDuration}
/>
{getSeparator('')}
</Field>
@ -690,7 +764,7 @@ const ConfigureIngestion = ({
name="result-limit"
type="number"
value={resultLimit}
onChange={(e) => handleResultLimit(parseInt(e.target.value))}
onChange={handleResultLimit}
/>
{getSeparator('')}
</Field>
@ -717,7 +791,7 @@ const ConfigureIngestion = ({
name="name"
type="text"
value={ingestionName}
onChange={(e) => handleIngestionName(e.target.value)}
onChange={handleIngestionName}
/>
{getSeparator('')}
</Field>
@ -776,8 +850,9 @@ const ConfigureIngestion = ({
};
const handleNext = () => {
handleDescription &&
handleDescription(markdownRef.current?.getEditorContent() || '');
onChange({
description: markdownRef.current?.getEditorContent() || '',
});
onNext();
};

2
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.test.tsx
View File

@ -29,10 +29,10 @@ jest.mock('../../common/toggle-switch/ToggleSwitchV1', () => {
const mockScheduleIntervalProps: ScheduleIntervalProps = {
status: 'initial',
repeatFrequency: '',
handleRepeatFrequencyChange: jest.fn(),
onBack: jest.fn(),
onDeploy: jest.fn(),
submitButtonLabel: 'Add',
onChange: jest.fn(),
};
describe('Test ScheduleInterval component', () => {

16
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ScheduleInterval.tsx
View File

@ -21,14 +21,18 @@ import Loader from '../../Loader/Loader';
import { ScheduleIntervalProps } from '../addIngestion.interface';
const ScheduleInterval = ({
status,
repeatFrequency,
handleRepeatFrequencyChange,
submitButtonLabel,
onBack,
onDeploy,
includePeriodOptions,
onBack,
onChange,
onDeploy,
repeatFrequency,
status,
submitButtonLabel,
}: ScheduleIntervalProps) => {
const handleRepeatFrequencyChange = (repeatFrequency: string) =>
onChange({
repeatFrequency: repeatFrequency,
});
const { t } = useTranslation();
return (

134
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
View File

@ -15,18 +15,23 @@ import { LoadingState } from 'Models';
import { FilterPatternEnum } from '../../enums/filterPattern.enum';
import { FormSubmitType } from '../../enums/form.enum';
import { ServiceCategory } from '../../enums/service.enum';
import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
import {
ConfigClass,
CreateIngestionPipeline,
DbtConfig,
} from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
import { ProfileSampleType } from '../../generated/entity/data/table';
import {
FilterPattern,
IngestionPipeline,
PipelineType,
} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
import {
DbtConfig,
DbtPipelineClass,
} from '../../generated/metadataIngestion/dbtPipeline';
import { DbtPipelineClass } from '../../generated/metadataIngestion/dbtPipeline';
import { DataObj } from '../../interface/service.interface';
import {
DBT_SOURCES,
GCS_CONFIG,
} from '../common/DBTConfigFormBuilder/DBTFormEnum';
export interface AddIngestionProps {
activeIngestionStep: number;
@ -58,72 +63,22 @@ export interface AddIngestionProps {
}
export interface ConfigureIngestionProps {
data: AddIngestionState;
formType: FormSubmitType;
ingestionName: string;
description?: string;
databaseServiceNames: string[];
serviceCategory: ServiceCategory;
databaseFilterPattern: FilterPattern;
dashboardFilterPattern: FilterPattern;
schemaFilterPattern: FilterPattern;
tableFilterPattern: FilterPattern;
topicFilterPattern: FilterPattern;
chartFilterPattern: FilterPattern;
pipelineFilterPattern: FilterPattern;
mlModelFilterPattern: FilterPattern;
includeLineage: boolean;
includeView: boolean;
includeTags: boolean;
markDeletedTables?: boolean;
markAllDeletedTables?: boolean;
enableDebugLog: boolean;
profileSample?: number;
profileSampleType?: ProfileSampleType;
ingestSampleData: boolean;
useFqnFilter: boolean;
pipelineType: PipelineType;
showDatabaseFilter: boolean;
showDashboardFilter: boolean;
showSchemaFilter: boolean;
showTableFilter: boolean;
showTopicFilter: boolean;
showChartFilter: boolean;
showPipelineFilter: boolean;
showMlModelFilter: boolean;
threadCount: number;
queryLogDuration: number;
stageFileLocation: string;
resultLimit: number;
timeoutSeconds: number;
handleIngestionName: (value: string) => void;
handleDatasetServiceName: (value: string[]) => void;
handleDescription?: (value: string) => void;
handleIncludeLineage: () => void;
onUseFqnFilterClick: () => void;
handleIncludeView: () => void;
handleIncludeTags: () => void;
handleMarkDeletedTables?: () => void;
handleMarkAllDeletedTables?: () => void;
handleEnableDebugLog: () => void;
handleIngestSampleData: () => void;
getIncludeValue: (value: string[], type: FilterPatternEnum) => void;
getExcludeValue: (value: string[], type: FilterPatternEnum) => void;
handleShowFilter: (value: boolean, type: FilterPatternEnum) => void;
handleProfileSample: (value?: number) => void;
handleQueryLogDuration: (value: number) => void;
handleProfileSampleType: (value: ProfileSampleType) => void;
handleStageFileLocation: (value: string) => void;
handleResultLimit: (value: number) => void;
handleThreadCount: (value: number) => void;
handleTimeoutSeconds: (value: number) => void;
getIncludeValue: (value: string[], type: FilterPatternEnum) => void;
handleShowFilter: (value: boolean, type: string) => void;
onCancel: () => void;
onChange: (newState: Partial<AddIngestionState>) => void;
onNext: () => void;
pipelineType: PipelineType;
serviceCategory: ServiceCategory;
}
export type ScheduleIntervalProps = {
onChange: (newState: Partial<AddIngestionState>) => void;
status: LoadingState;
repeatFrequency: string;
handleRepeatFrequencyChange: (value: string) => void;
includePeriodOptions?: string[];
submitButtonLabel: string;
onBack: () => void;
@ -133,3 +88,58 @@ export type ScheduleIntervalProps = {
// Todo: Need to refactor below type, as per schema change #9575
export type ModifiedDbtConfig = DbtConfig &
Pick<DbtPipelineClass, 'dbtUpdateDescriptions'>;
export interface AddIngestionState {
chartFilterPattern: FilterPattern;
dashboardFilterPattern: FilterPattern;
databaseFilterPattern: FilterPattern;
databaseServiceNames: string[];
dbtConfigSource: ModifiedDbtConfig;
dbtConfigSourceType: DBT_SOURCES;
description: string;
enableDebugLog: boolean;
gcsConfigType: GCS_CONFIG | undefined;
includeLineage: boolean;
includeTags: boolean;
includeView: boolean;
ingestionName: string;
ingestSampleData: boolean;
markAllDeletedTables: boolean | undefined;
markDeletedTables: boolean | undefined;
metadataToESConfig: ConfigClass | undefined;
mlModelFilterPattern: FilterPattern;
pipelineFilterPattern: FilterPattern;
profileSample: number | undefined;
profileSampleType: ProfileSampleType;
queryLogDuration: number;
repeatFrequency: string;
resultLimit: number;
saveState: LoadingState;
schemaFilterPattern: FilterPattern;
showChartFilter: boolean;
showDashboardFilter: boolean;
showDatabaseFilter: boolean;
showDeployModal: boolean;
showMlModelFilter: boolean;
showPipelineFilter: boolean;
showSchemaFilter: boolean;
showTableFilter: boolean;
showTopicFilter: boolean;
stageFileLocation: string;
tableFilterPattern: FilterPattern;
threadCount: number;
timeoutSeconds: number;
topicFilterPattern: FilterPattern;
useFqnFilter: boolean;
}
export enum ShowFilter {
showChartFilter = 'showChartFilter',
showDashboardFilter = 'showDashboardFilter',
showDatabaseFilter = 'showDatabaseFilter',
showMlModelFilter = 'showMlModelFilter',
showPipelineFilter = 'showPipelineFilter',
showSchemaFilter = 'showSchemaFilter',
showTableFilter = 'showTableFilter',
showTopicFilter = 'showTopicFilter',
}

6
openmetadata-ui/src/main/resources/ui/src/components/SliderWithInput/SliderWithInput.tsx
View File

@ -12,7 +12,7 @@
*/
import { Col, InputNumber, Row, Slider } from 'antd';
import React from 'react';
import React, { useCallback } from 'react';
import { SliderWithInputProps } from './SliderWithInput.interface';
const SliderWithInput = ({
@ -20,6 +20,8 @@ const SliderWithInput = ({
onChange,
className,
}: SliderWithInputProps) => {
const formatter = useCallback((value) => `${value}%`, [value]);
return (
<Row className={className} data-testid="percentage-input" gutter={20}>
<Col span={20}>
@ -38,7 +40,7 @@ const SliderWithInput = ({
<Col span={4}>
<InputNumber
data-testid="slider-input"
formatter={(value) => `${value}%`}
formatter={formatter}
max={100}
min={0}
step={1}

200
openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ProfilerSettingsModal.tsx
View File

@ -29,7 +29,14 @@ import { AxiosError } from 'axios';
import classNames from 'classnames';
import 'codemirror/addon/fold/foldgutter.css';
import { isEmpty, isEqual, isUndefined, omit, startCase } from 'lodash';
import React, { useEffect, useMemo, useState } from 'react';
import React, {
Reducer,
useCallback,
useEffect,
useMemo,
useReducer,
useState,
} from 'react';
import { Controlled as CodeMirror } from 'react-codemirror2';
import { useTranslation } from 'react-i18next';
import {
@ -42,16 +49,18 @@ import {
SUPPORTED_PARTITION_TYPE,
} from '../../../constants/profiler.constant';
import {
ColumnProfilerConfig,
PartitionProfilerConfig,
ProfileSampleType,
TableProfilerConfig,
} from '../../../generated/entity/data/table';
import jsonData from '../../../jsons/en';
import { reducerWithoutAction } from '../../../utils/CommonUtils';
import SVGIcons, { Icons } from '../../../utils/SvgUtils';
import { showErrorToast, showSuccessToast } from '../../../utils/ToastUtils';
import SliderWithInput from '../../SliderWithInput/SliderWithInput';
import { ProfilerSettingsModalProps } from '../TableProfiler.interface';
import {
ProfilerSettingModalState,
ProfilerSettingsModalProps,
} from '../TableProfiler.interface';
import '../tableProfiler.less';
const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
@ -62,20 +71,32 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
}) => {
const { t } = useTranslation();
const [form] = Form.useForm();
const [data, setData] = useState<TableProfilerConfig>();
const [sqlQuery, setSqlQuery] = useState<string>('');
const [profileSample, setProfileSample] = useState<number | undefined>(100);
const [excludeCol, setExcludeCol] = useState<string[]>([]);
const [includeCol, setIncludeCol] = useState<ColumnProfilerConfig[]>(
DEFAULT_INCLUDE_PROFILE
);
const [isLoading, setIsLoading] = useState(false);
const [enablePartition, setEnablePartition] = useState(false);
const [partitionData, setPartitionData] = useState<PartitionProfilerConfig>();
const [selectedProfileSampleType, setSelectedProfileSampleType] = useState<
ProfileSampleType | undefined
>(ProfileSampleType.Percentage);
const initialState: ProfilerSettingModalState = useMemo(
() => ({
data: undefined,
sqlQuery: '',
profileSample: 100,
excludeCol: [],
includeCol: DEFAULT_INCLUDE_PROFILE,
enablePartition: false,
partitionData: undefined,
selectedProfileSampleType: ProfileSampleType.Percentage,
}),
[]
);
const [state, dispatch] = useReducer<
Reducer<ProfilerSettingModalState, Partial<ProfilerSettingModalState>>
>(reducerWithoutAction, initialState);
const handleStateChange = useCallback(
(newState: Partial<ProfilerSettingModalState>) => {
dispatch(newState);
},
[]
);
const selectOptions = useMemo(() => {
return columns.map(({ name }) => ({
@ -131,12 +152,13 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
profileSampleType,
excludeColumns,
} = tableProfilerConfig;
setSqlQuery(profileQuery || '');
setProfileSample(profileSample);
setExcludeCol(excludeColumns || []);
setSelectedProfileSampleType(
profileSampleType || ProfileSampleType.Percentage
);
handleStateChange({
sqlQuery: profileQuery || '',
profileSample: profileSample,
excludeCol: excludeColumns || [],
selectedProfileSampleType:
profileSampleType || ProfileSampleType.Percentage,
});
const profileSampleTypeCheck =
profileSampleType === ProfileSampleType.Percentage;
@ -162,10 +184,15 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
return col;
});
form.setFieldsValue({ includeColumns: includeColValue });
setIncludeCol(includeColValue);
handleStateChange({
includeCol: includeColValue,
});
}
if (partitioning) {
setEnablePartition(partitioning.enablePartitioning || false);
handleStateChange({
enablePartition: partitioning.enablePartitioning || false,
});
form.setFieldsValue({ ...partitioning });
}
};
@ -176,7 +203,10 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
if (response) {
const { tableProfilerConfig } = response;
if (tableProfilerConfig) {
setData(tableProfilerConfig);
handleStateChange({
data: tableProfilerConfig,
});
updateInitialConfig(tableProfilerConfig);
}
} else {
@ -193,10 +223,13 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
};
const getIncludesColumns = () => {
const includeCols = includeCol.filter(
const includeCols = state.includeCol.filter(
({ columnName }) => !isUndefined(columnName)
);
setIncludeCol(includeCols);
handleStateChange({
includeCol: includeCols,
});
return includeCols.map((col) => {
if (col.metrics && col.metrics[0] === 'all') {
@ -209,7 +242,9 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
});
};
const handleSave: FormProps['onFinish'] = async (data) => {
const handleSave: FormProps['onFinish'] = useCallback(async (data) => {
const { excludeCol, sqlQuery, includeCol, enablePartition, partitionData } =
state;
setIsLoading(true);
const { profileSamplePercentage, profileSampleRows, profileSampleType } =
data;
@ -252,11 +287,55 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
} finally {
setIsLoading(false);
}
};
const handleCancel = () => {
}, []);
const handleCancel = useCallback(() => {
const { data } = state;
data && updateInitialConfig(data);
onVisibilityChange(false);
};
}, []);
const handleProfileSampleType = useCallback(
(selectedProfileSampleType) =>
handleStateChange({
selectedProfileSampleType,
}),
[]
);
const handleProfileSample = useCallback(
(value) =>
handleStateChange({
profileSample: Number(value),
}),
[]
);
const handleCodeMirrorChange = useCallback(
(_Editor, _EditorChange, value) => {
handleStateChange({
sqlQuery: value,
});
},
[]
);
const handleIncludeColumnsProfiler = useCallback((_, data) => {
handleStateChange({
includeCol: data.includeColumns,
partitionData: omit(data, 'includeColumns'),
});
}, []);
const handleChange =
(field: keyof ProfilerSettingModalState) =>
(value: ProfilerSettingModalState[keyof ProfilerSettingModalState]) =>
handleStateChange({
[field]: value,
});
const handleExcludeCol = handleChange('excludeCol');
const handleEnablePartition = handleChange('enablePartition');
useEffect(() => {
fetchProfileConfig();
@ -292,8 +371,8 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
data-testid="configure-ingestion-container"
form={form}
initialValues={{
profileSampleType: selectedProfileSampleType,
profileSamplePercentage: profileSample || 100,
profileSampleType: state?.selectedProfileSampleType,
profileSamplePercentage: state?.profileSample || 100,
}}
layout="vertical">
<Form.Item
@ -305,11 +384,12 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
className="w-full"
data-testid="profile-sample"
options={PROFILE_SAMPLE_OPTIONS}
onChange={setSelectedProfileSampleType}
onChange={handleProfileSampleType}
/>
</Form.Item>
{selectedProfileSampleType === ProfileSampleType.Percentage ? (
{state?.selectedProfileSampleType ===
ProfileSampleType.Percentage ? (
<Form.Item
className="m-b-0"
label={t('label.profile-sample-type', {
@ -318,8 +398,8 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
name="profileSamplePercentage">
<SliderWithInput
className="p-x-xs"
value={profileSample || 0}
onChange={(value) => setProfileSample(Number(value))}
value={state?.profileSample || 0}
onChange={handleProfileSample}
/>
</Form.Item>
) : (
@ -351,13 +431,9 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
className="profiler-setting-sql-editor"
data-testid="profiler-setting-sql-editor"
options={codeMirrorOption}
value={sqlQuery}
onBeforeChange={(_Editor, _EditorChange, value) => {
setSqlQuery(value);
}}
onChange={(_Editor, _EditorChange, value) => {
setSqlQuery(value);
}}
value={state?.sqlQuery}
onBeforeChange={handleCodeMirrorChange}
onChange={handleCodeMirrorChange}
/>
</Col>
<Col data-testid="exclude-column-container" span={24}>
@ -371,8 +447,8 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
options={selectOptions}
placeholder={t('label.select-column-plural-to-exclude')}
size="middle"
value={excludeCol}
onChange={(value) => setExcludeCol(value)}
value={state?.excludeCol}
onChange={handleExcludeCol}
/>
</Col>
@ -382,16 +458,13 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
form={form}
id="profiler-setting-form"
initialValues={{
includeColumns: includeCol,
...data?.partitioning,
includeColumns: state?.includeCol,
...state?.data?.partitioning,
}}
layout="vertical"
name="includeColumnsProfiler"
onFinish={handleSave}
onValuesChange={(_, data) => {
setIncludeCol(data.includeColumns);
setPartitionData(omit(data, 'includeColumns'));
}}>
onValuesChange={handleIncludeColumnsProfiler}>
<List name="includeColumns">
{(fields, { add, remove }) => (
<>
@ -409,7 +482,8 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
</div>
<div
className={classNames({
'tw-max-h-40 tw-overflow-y-auto': includeCol.length > 1,
'tw-max-h-40 tw-overflow-y-auto':
state?.includeCol.length > 1,
})}
data-testid="include-column-container">
{fields.map(({ key, name, ...restField }) => (
@ -466,10 +540,10 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
<Space size={12}>
<p>{t('label.enable-partition')}</p>
<Switch
checked={enablePartition}
checked={state?.enablePartition}
data-testid="enable-partition-switch"
disabled={isPartitionDisabled}
onChange={(value) => setEnablePartition(value)}
onChange={handleEnablePartition}
/>
</Space>
</Form.Item>
@ -492,7 +566,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
name="partitionColumnName"
rules={[
{
required: enablePartition,
required: state?.enablePartition,
message: t('message.field-text-is-required', {
fieldText: t('label.column-entity', {
entity: t('label.name'),
@ -504,7 +578,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
allowClear
className="w-full"
data-testid="column-name"
disabled={!enablePartition}
disabled={!state?.enablePartition}
options={partitionColumnOptions}
placeholder={t('message.select-column-name')}
size="middle"
@ -525,7 +599,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
name="partitionIntervalType"
rules={[
{
required: enablePartition,
required: state?.enablePartition,
message: t('message.field-text-is-required', {
fieldText: t('label.interval-type'),
}),
@ -535,7 +609,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
allowClear
className="w-full"
data-testid="interval-type"
disabled={!enablePartition}
disabled={!state?.enablePartition}
options={INTERVAL_TYPE_OPTIONS}
placeholder={t('message.select-interval-type')}
size="middle"
@ -554,7 +628,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
name="partitionInterval"
rules={[
{
required: enablePartition,
required: state?.enablePartition,
message: t('message.field-text-is-required', {
fieldText: t('label.interval'),
}),
@ -563,7 +637,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
<InputNumber
className="w-full"
data-testid="interval-required"
disabled={!enablePartition}
disabled={!state?.enablePartition}
placeholder={t('message.enter-interval')}
size="middle"
/>
@ -583,7 +657,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
name="partitionIntervalUnit"
rules={[
{
required: enablePartition,
required: state?.enablePartition,
message: t('message.field-text-is-required', {
fieldText: t('label.interval-unit'),
}),
@ -593,7 +667,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
allowClear
className="w-full"
data-testid="select-interval-unit"
disabled={!enablePartition}
disabled={!state?.enablePartition}
options={INTERVAL_UNIT_OPTIONS}
placeholder={t('message.select-interval-unit')}
size="middle"

20
openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts
View File

@ -13,7 +13,14 @@
import { PROFILER_FILTER_RANGE } from '../../constants/profiler.constant';
import { SystemProfile } from '../../generated/api/data/createTableProfile';
import { Column, TableProfile } from '../../generated/entity/data/table';
import {
Column,
ColumnProfilerConfig,
PartitionProfilerConfig,
ProfileSampleType,
TableProfile,
TableProfilerConfig,
} from '../../generated/entity/data/table';
import { TestCase } from '../../generated/tests/testCase';
import { OperationPermission } from '../PermissionProvider/PermissionProvider.interface';
@ -76,3 +83,14 @@ export type TableProfilerData = {
export type TableProfilerChartProps = {
selectedTimeRange: keyof typeof PROFILER_FILTER_RANGE;
};
export interface ProfilerSettingModalState {
data: TableProfilerConfig | undefined;
sqlQuery: string;
profileSample: number | undefined;
excludeCol: string[];
includeCol: ColumnProfilerConfig[];
enablePartition: boolean;
partitionData: PartitionProfilerConfig | undefined;
selectedProfileSampleType: ProfileSampleType | undefined;
}

15
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigForm.interface.ts
View File

@ -17,7 +17,10 @@ import {
GCSCredentialsValues,
SCredentials,
} from '../../../generated/metadataIngestion/dbtPipeline';
import { ModifiedDbtConfig } from '../../AddIngestion/addIngestion.interface';
import {
AddIngestionState,
ModifiedDbtConfig,
} from '../../AddIngestion/addIngestion.interface';
import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum';
export interface DBTFormCommonProps {
@ -29,13 +32,9 @@ export interface DBTFormCommonProps {
export interface DBTConfigFormProps extends DBTFormCommonProps {
formType: FormSubmitType;
data: DbtConfig;
gcsType?: GCS_CONFIG;
source?: DBT_SOURCES;
handleGcsTypeChange?: (type: GCS_CONFIG) => void;
handleSourceChange?: (src: DBT_SOURCES) => void;
ingestionName: string;
handleIngestionName: (value: string) => void;
data: AddIngestionState;
onChange: (newState: Partial<AddIngestionState>) => void;
}
export type DbtConfigCloud = Pick<

102
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.test.tsx
View File

@ -15,6 +15,8 @@ import { fireEvent, getByTestId, render } from '@testing-library/react';
import i18n from 'i18next';
import React from 'react';
import { FormSubmitType } from '../../../enums/form.enum';
import { AddIngestionState } from '../../AddIngestion/addIngestion.interface';
import { DBTConfigFormProps } from './DBTConfigForm.interface';
import DBTConfigFormBuilder from './DBTConfigFormBuilder';
import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum';
@ -148,23 +150,26 @@ jest.mock('./DBTGCSConfig', () => ({
const mockCancel = jest.fn();
const mockSubmit = jest.fn();
const mockCatalogChange = jest.fn();
const mockManifestChange = jest.fn();
const mockIngestionName = jest.fn();
const mockOnChange = jest.fn();
const mockState = {
dbtConfigSource: '',
dbtConfigSourceType: DBT_SOURCES.local,
ingestionName: i18n.t('label.dbt-uppercase'),
gcsConfigType: GCS_CONFIG.GCSValues,
} as unknown as AddIngestionState;
const mockProps = {
data: mockData,
data: mockState,
okText: i18n.t('label.next'),
cancelText: i18n.t('label.cancel'),
gcsType: GCS_CONFIG.GCSValues,
handleGcsTypeChange: mockCatalogChange,
handleSourceChange: mockManifestChange,
onCancel: mockCancel,
onSubmit: mockSubmit,
formType: FormSubmitType.ADD,
handleIngestionName: mockIngestionName,
ingestionName: i18n.t('label.dbt-uppercase'),
};
onChange: mockOnChange,
} as DBTConfigFormProps;
describe('Test DBT Config Form Builder', () => {
it('Form should render with default dbt source', async () => {
@ -178,7 +183,14 @@ describe('Test DBT Config Form Builder', () => {
it('Form should render with local dbt source', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={DBT_SOURCES.local} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: DBT_SOURCES.local,
} as AddIngestionState
}
/>
);
const selectSource = getByTestId(container, 'dbt-source');
const dbtLocal = getByTestId(container, 'dbt-local');
@ -189,7 +201,14 @@ describe('Test DBT Config Form Builder', () => {
it('Form should render with http dbt source', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={DBT_SOURCES.http} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: DBT_SOURCES.http,
} as AddIngestionState
}
/>
);
const selectSource = getByTestId(container, 'dbt-source');
const dbtHttp = getByTestId(container, 'dbt-http');
@ -200,7 +219,14 @@ describe('Test DBT Config Form Builder', () => {
it('Form should render with s3 dbt source', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={DBT_SOURCES.s3} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: DBT_SOURCES.s3,
} as AddIngestionState
}
/>
);
const selectSource = getByTestId(container, 'dbt-source');
const dbtS3 = getByTestId(container, 'dbt-s3');
@ -211,7 +237,14 @@ describe('Test DBT Config Form Builder', () => {
it('Form should render with gcs dbt source', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={DBT_SOURCES.gcs} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: DBT_SOURCES.gcs,
} as AddIngestionState
}
/>
);
const selectSource = getByTestId(container, 'dbt-source');
const dbtGCS = getByTestId(container, 'dbt-gcs');
@ -222,7 +255,14 @@ describe('Test DBT Config Form Builder', () => {
it('Form should render with no dbt source', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={'' as DBT_SOURCES} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: '' as DBT_SOURCES,
} as AddIngestionState
}
/>
);
const selectSource = getByTestId(container, 'dbt-source');
const dbtNone = getByTestId(container, 'dbt-source-none');
@ -233,7 +273,14 @@ describe('Test DBT Config Form Builder', () => {
it('should change dbt local fields', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={DBT_SOURCES.local} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: DBT_SOURCES.local,
} as AddIngestionState
}
/>
);
const dbtLocal = getByTestId(container, 'dbt-local');
@ -244,7 +291,14 @@ describe('Test DBT Config Form Builder', () => {
it('should change dbt http fields', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={DBT_SOURCES.http} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: DBT_SOURCES.http,
} as AddIngestionState
}
/>
);
const dbtHttp = getByTestId(container, 'dbt-http');
@ -255,7 +309,14 @@ describe('Test DBT Config Form Builder', () => {
it('should change dbt s3 fields', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={DBT_SOURCES.s3} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: DBT_SOURCES.s3,
} as AddIngestionState
}
/>
);
const dbtS3 = getByTestId(container, 'dbt-s3');
@ -266,7 +327,14 @@ describe('Test DBT Config Form Builder', () => {
it('should change dbt gcs fields', async () => {
const { container } = render(
<DBTConfigFormBuilder {...mockProps} source={DBT_SOURCES.gcs} />
<DBTConfigFormBuilder
{...mockProps}
data={
{
dbtConfigSourceType: DBT_SOURCES.gcs,
} as AddIngestionState
}
/>
);
const dbtGCS = getByTestId(container, 'dbt-gcs');

121
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx
View File

@ -12,7 +12,13 @@
*/
import { Button } from 'antd';
import React, { Fragment, FunctionComponent, useEffect, useState } from 'react';
import React, {
Fragment,
FunctionComponent,
useEffect,
useMemo,
useState,
} from 'react';
import { useTranslation } from 'react-i18next';
import { FormSubmitType } from '../../../enums/form.enum';
import {
@ -25,28 +31,42 @@ import { Field } from '../../Field/Field';
import { DBTCloudConfig } from './DBTCloudConfig';
import { DBTConfigFormProps } from './DBTConfigForm.interface';
import { DBTSources } from './DBTFormConstants';
import { DBT_SOURCES } from './DBTFormEnum';
import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum';
import { DBTGCSConfig } from './DBTGCSConfig';
import { DBTHttpConfig } from './DBTHttpConfig';
import { DBTLocalConfig } from './DBTLocalConfig';
import { DBTS3Config } from './DBTS3Config';
const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
data,
okText,
cancelText,
gcsType,
source = DBT_SOURCES.local,
handleGcsTypeChange,
handleSourceChange,
onCancel,
onSubmit,
data,
formType,
ingestionName,
handleIngestionName,
okText,
onCancel,
onChange,
onSubmit,
}: DBTConfigFormProps) => {
const { t } = useTranslation();
const [dbtConfig, setDbtConfig] = useState<ModifiedDbtConfig>(data);
// const [dbtConfig, setDbtConfig] = useState<ModifiedDbtConfig>(data);
const { dbtConfigSource, gcsConfigType, ingestionName, dbtConfigSourceType } =
useMemo(
() => ({
ingestionName: data.ingestionName,
gcsConfigType: data.gcsConfigType,
dbtConfigSourceType: data.dbtConfigSourceType,
dbtConfigSource: data.dbtConfigSource,
}),
[
data.ingestionName,
data.gcsConfigType,
data.dbtConfigSourceType,
data.dbtConfigSource,
]
);
const [dbtConfig, setDbtConfig] =
useState<ModifiedDbtConfig>(dbtConfigSource);
const updateDbtConfig = (
key: keyof ModifiedDbtConfig,
@ -61,10 +81,10 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
return (
<DBTCloudConfig
cancelText={cancelText}
dbtCloudAccountId={dbtConfig.dbtCloudAccountId}
dbtCloudAuthToken={dbtConfig.dbtCloudAuthToken}
dbtCloudProjectId={dbtConfig.dbtCloudProjectId}
dbtUpdateDescriptions={dbtConfig.dbtUpdateDescriptions}
dbtCloudAccountId={dbtConfig?.dbtCloudAccountId}
dbtCloudAuthToken={dbtConfig?.dbtCloudAuthToken}
dbtCloudProjectId={dbtConfig?.dbtCloudProjectId}
dbtUpdateDescriptions={dbtConfig?.dbtUpdateDescriptions}
handleCloudAccountIdChange={(val) => {
updateDbtConfig('dbtCloudAccountId', val);
}}
@ -88,10 +108,10 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
return (
<DBTLocalConfig
cancelText={cancelText}
dbtCatalogFilePath={dbtConfig.dbtCatalogFilePath}
dbtManifestFilePath={dbtConfig.dbtManifestFilePath}
dbtRunResultsFilePath={dbtConfig.dbtRunResultsFilePath}
dbtUpdateDescriptions={dbtConfig.dbtUpdateDescriptions}
dbtCatalogFilePath={dbtConfig?.dbtCatalogFilePath}
dbtManifestFilePath={dbtConfig?.dbtManifestFilePath}
dbtRunResultsFilePath={dbtConfig?.dbtRunResultsFilePath}
dbtUpdateDescriptions={dbtConfig?.dbtUpdateDescriptions}
handleCatalogFilePathChange={(val) => {
updateDbtConfig('dbtCatalogFilePath', val);
}}
@ -115,10 +135,10 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
return (
<DBTHttpConfig
cancelText={cancelText}
dbtCatalogHttpPath={dbtConfig.dbtCatalogHttpPath}
dbtManifestHttpPath={dbtConfig.dbtManifestHttpPath}
dbtRunResultsHttpPath={dbtConfig.dbtRunResultsHttpPath}
dbtUpdateDescriptions={dbtConfig.dbtUpdateDescriptions}
dbtCatalogHttpPath={dbtConfig?.dbtCatalogHttpPath}
dbtManifestHttpPath={dbtConfig?.dbtManifestHttpPath}
dbtRunResultsHttpPath={dbtConfig?.dbtRunResultsHttpPath}
dbtUpdateDescriptions={dbtConfig?.dbtUpdateDescriptions}
handleCatalogHttpPathChange={(val) => {
updateDbtConfig('dbtCatalogHttpPath', val);
}}
@ -142,9 +162,9 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
return (
<DBTS3Config
cancelText={cancelText}
dbtPrefixConfig={dbtConfig.dbtPrefixConfig}
dbtSecurityConfig={dbtConfig.dbtSecurityConfig}
dbtUpdateDescriptions={dbtConfig.dbtUpdateDescriptions}
dbtPrefixConfig={dbtConfig?.dbtPrefixConfig}
dbtSecurityConfig={dbtConfig?.dbtSecurityConfig}
dbtUpdateDescriptions={dbtConfig?.dbtUpdateDescriptions}
handlePrefixConfigChange={(val) => {
updateDbtConfig('dbtPrefixConfig', val);
}}
@ -161,17 +181,33 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
);
};
const handleGcsTypeChange = (type: GCS_CONFIG) =>
onChange({
gcsConfigType: type,
});
const handleOnchange = (event: React.ChangeEvent<HTMLInputElement>) =>
onChange({
ingestionName: event.target.value,
});
const handleDbtConfigSourceType = (
event: React.ChangeEvent<HTMLSelectElement>
) => {
onChange({
dbtConfigSourceType: event.target.value as DBT_SOURCES,
});
};
const getGCSConfigFields = () => {
return (
<DBTGCSConfig
cancelText={cancelText}
dbtPrefixConfig={dbtConfig.dbtPrefixConfig}
dbtSecurityConfig={dbtConfig.dbtSecurityConfig}
dbtUpdateDescriptions={dbtConfig.dbtUpdateDescriptions}
gcsType={gcsType}
handleGcsTypeChange={(type) => {
handleGcsTypeChange && handleGcsTypeChange(type);
}}
dbtPrefixConfig={dbtConfig?.dbtPrefixConfig}
dbtSecurityConfig={dbtConfig?.dbtSecurityConfig}
dbtUpdateDescriptions={dbtConfig?.dbtUpdateDescriptions}
gcsType={gcsConfigType}
handleGcsTypeChange={handleGcsTypeChange}
handlePrefixConfigChange={(val) => {
updateDbtConfig('dbtPrefixConfig', val);
}}
@ -189,7 +225,7 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
};
const getFields = () => {
switch (source) {
switch (dbtConfigSourceType) {
case DBT_SOURCES.cloud: {
return getCloudConfigFields();
}
@ -236,8 +272,8 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
};
useEffect(() => {
setDbtConfig(data);
}, [data, source, gcsType]);
setDbtConfig(dbtConfigSource);
}, [data, dbtConfigSourceType, gcsConfigType]);
return (
<Fragment>
@ -256,7 +292,7 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
name="name"
type="text"
value={ingestionName}
onChange={(e) => handleIngestionName(e.target.value)}
onChange={handleOnchange}
/>
{getSeparator('')}
</Field>
@ -275,11 +311,8 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
placeholder={t('label.select-field', {
field: t('label.dbt-source'),
})}
value={source}
onChange={(e) => {
handleSourceChange &&
handleSourceChange(e.target.value as DBT_SOURCES);
}}>
value={dbtConfigSourceType}
onChange={handleDbtConfigSourceType}>
{DBTSources.map((option, i) => (
<option key={i} value={option.value}>
{option.name}

29
openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx
View File

@ -47,6 +47,7 @@ import React from 'react';
import { Trans } from 'react-i18next';
import { reactLocalStorage } from 'reactjs-localstorage';
import AppState from '../AppState';
import { AddIngestionState } from '../components/AddIngestion/addIngestion.interface';
import { FQN_SEPARATOR_CHAR } from '../constants/char.constants';
import {
getTeamAndUserDetailsPath,
@ -907,6 +908,34 @@ export const getFilterPatternDocsLinks = (type: FilterPatternEnum) => {
}
};
/**
* It takes a string and returns a string
* @param {FilterPatternEnum} type - FilterPatternEnum
* @returns A function that takes in a type and returns a keyof AddIngestionState
*/
export const getFilterTypes = (
type: FilterPatternEnum
): keyof AddIngestionState => {
switch (type) {
case FilterPatternEnum.CHART:
return 'chartFilterPattern' as keyof AddIngestionState;
case FilterPatternEnum.DASHBOARD:
return 'dashboardFilterPattern' as keyof AddIngestionState;
case FilterPatternEnum.DATABASE:
return 'databaseFilterPattern' as keyof AddIngestionState;
case FilterPatternEnum.MLMODEL:
return 'mlModelFilterPattern' as keyof AddIngestionState;
case FilterPatternEnum.PIPELINE:
return 'pipelineFilterPattern' as keyof AddIngestionState;
case FilterPatternEnum.SCHEMA:
return 'schemaFilterPattern' as keyof AddIngestionState;
case FilterPatternEnum.TABLE:
return 'tableFilterPattern' as keyof AddIngestionState;
default:
return 'topicFilterPattern' as keyof AddIngestionState;
}
};
/**
* It takes a state and an action, and returns a new state with the action merged into it
* @param {S} state - S - The current state of the reducer.