yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-06 04:26:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Add additional fields to the DBT config form and support DB filter pattern (#4810)

Browse Source
This commit is contained in:
darth-coder00 2022-05-10 11:46:01 +05:30 committed by GitHub
parent 10dd966019
commit ced532812a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 314 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
View File

@ -101,6 +101,11 @@ const AddIngestion = ({
(data?.source.sourceConfig.config as ConfigClass)?.dashboardFilterPattern
)
);
const [showDatabaseFilter, setShowDatabaseFilter] = useState(
!isUndefined(
(data?.source.sourceConfig.config as ConfigClass)?.databaseFilterPattern
)
);
const [showSchemaFilter, setShowSchemaFilter] = useState(
!isUndefined(
(data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern
@ -137,11 +142,7 @@ const AddIngestion = ({
>(showDBTConfig ? (configData as DbtConfigSource) : undefined);
const sourceTypeData = useMemo(
() =>
getSourceTypeFromConfig(
configData as DbtConfigSource | undefined,
status === FormSubmitType.ADD ? DBT_SOURCES.local : ('' as DBT_SOURCES)
),
() => getSourceTypeFromConfig(configData as DbtConfigSource | undefined),
[configData]
);
const [dbtConfigSourceType, setDbtConfigSourceType] = useState<
@ -170,13 +171,18 @@ const AddIngestion = ({
true
);
const [enableDebugLog, setEnableDebugLog] = useState(
isUndefined(data?.loggerLevel) ?? data?.loggerLevel === LogLevels.Debug
isUndefined(data?.loggerLevel) || data?.loggerLevel === LogLevels.Debug
);
const [dashboardFilterPattern, setDashboardFilterPattern] =
useState<FilterPattern>(
(data?.source.sourceConfig.config as ConfigClass)
?.dashboardFilterPattern ?? INITIAL_FILTER_PATTERN
);
const [databaseFilterPattern, setDatabaseFilterPattern] =
useState<FilterPattern>(
(data?.source.sourceConfig.config as ConfigClass)
?.databaseFilterPattern ?? INITIAL_FILTER_PATTERN
);
const [schemaFilterPattern, setSchemaFilterPattern] = useState<FilterPattern>(
(data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern ??
INITIAL_FILTER_PATTERN
@ -229,6 +235,10 @@ const AddIngestion = ({
includes: value,
});
break;
case FilterPatternEnum.DATABASE:
setDatabaseFilterPattern({ ...databaseFilterPattern, includes: value });
break;
case FilterPatternEnum.SCHEMA:
setSchemaFilterPattern({ ...schemaFilterPattern, includes: value });
@ -260,6 +270,10 @@ const AddIngestion = ({
excludes: value,
});
break;
case FilterPatternEnum.DATABASE:
setDatabaseFilterPattern({ ...databaseFilterPattern, excludes: value });
break;
case FilterPatternEnum.SCHEMA:
setSchemaFilterPattern({ ...schemaFilterPattern, excludes: value });
@ -289,6 +303,10 @@ const AddIngestion = ({
case FilterPatternEnum.DASHBOARD:
setShowDashboardFilter(value);
break;
case FilterPatternEnum.DATABASE:
setShowDatabaseFilter(value);
break;
case FilterPatternEnum.SCHEMA:
setShowSchemaFilter(value);
@ -377,6 +395,7 @@ const AddIngestion = ({
enableDataProfiler: enableDataProfiler,
generateSampleData: ingestSampleData,
includeViews: includeView,
databaseFilterPattern: getFilterPatternData(databaseFilterPattern),
schemaFilterPattern: getFilterPatternData(schemaFilterPattern),
tableFilterPattern: getFilterPatternData(tableFilterPattern),
chartFilterPattern: getFilterPatternData(chartFilterPattern),
@ -529,6 +548,7 @@ const AddIngestion = ({
<ConfigureIngestion
chartFilterPattern={chartFilterPattern}
dashboardFilterPattern={dashboardFilterPattern}
databaseFilterPattern={databaseFilterPattern}
description={description}
enableDataProfiler={enableDataProfiler}
enableDebugLog={enableDebugLog}
@ -559,6 +579,7 @@ const AddIngestion = ({
serviceCategory={serviceCategory}
showChartFilter={showChartFilter}
showDashboardFilter={showDashboardFilter}
showDatabaseFilter={showDatabaseFilter}
showFqnFilter={showFqnFilter}
showSchemaFilter={showSchemaFilter}
showTableFilter={showTableFilter}

7
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
View File

@ -30,6 +30,10 @@ jest.mock('../../common/toggle-switch/ToggleSwitchV1', () => {
const mockConfigureIngestion: ConfigureIngestionProps = {
ingestionName: '',
databaseFilterPattern: {
includes: [],
excludes: [],
},
dashboardFilterPattern: {
includes: [],
excludes: [],
@ -63,6 +67,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
ingestSampleData: false,
markDeletedTables: false,
showDashboardFilter: false,
showDatabaseFilter: false,
showSchemaFilter: false,
showTableFilter: false,
showTopicFilter: false,
@ -111,7 +116,7 @@ describe('Test ConfigureIngestion component', () => {
expect(configureIngestionContainer).toBeInTheDocument();
expect(backButton).toBeInTheDocument();
expect(nextButton).toBeInTheDocument();
expect(filterPatternComponents.length).toBe(2);
expect(filterPatternComponents.length).toBe(3);
expect(toggleSwitchs.length).toBe(5);
});
});

13
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
View File

@ -28,6 +28,7 @@ import { ConfigureIngestionProps } from '../addIngestion.interface';
const ConfigureIngestion = ({
ingestionName,
description = '',
databaseFilterPattern,
dashboardFilterPattern,
schemaFilterPattern,
tableFilterPattern,
@ -40,6 +41,7 @@ const ConfigureIngestion = ({
enableDataProfiler,
ingestSampleData,
pipelineType,
showDatabaseFilter,
showDashboardFilter,
showSchemaFilter,
showTableFilter,
@ -154,6 +156,17 @@ const ConfigureIngestion = ({
case ServiceCategory.DATABASE_SERVICES:
return (
<Fragment>
<FilterPattern
checked={showDatabaseFilter}
excludePattern={databaseFilterPattern?.excludes ?? []}
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleChecked={(value) =>
handleShowFilter(value, FilterPatternEnum.DATABASE)
}
includePattern={databaseFilterPattern?.includes ?? []}
type={FilterPatternEnum.DATABASE}
/>
<FilterPattern
checked={showSchemaFilter}
excludePattern={schemaFilterPattern?.excludes ?? []}

2
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
View File

@ -56,6 +56,7 @@ export interface ConfigureIngestionProps {
ingestionName: string;
description?: string;
serviceCategory: ServiceCategory;
databaseFilterPattern: FilterPattern;
dashboardFilterPattern: FilterPattern;
schemaFilterPattern: FilterPattern;
tableFilterPattern: FilterPattern;
@ -68,6 +69,7 @@ export interface ConfigureIngestionProps {
enableDebugLog: boolean;
ingestSampleData: boolean;
pipelineType: PipelineType;
showDatabaseFilter: boolean;
showDashboardFilter: boolean;
showSchemaFilter: boolean;
showTableFilter: boolean;

5
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigForm.interface.ts
View File

@ -43,7 +43,10 @@ export type DbtConfigHttp = Pick<
'dbtCatalogHttpPath' | 'dbtManifestHttpPath'
>;
export type DbtConfigS3GCS = Pick<DbtConfigSource, 'dbtSecurityConfig'>;
export type DbtConfigS3GCS = Pick<
DbtConfigSource,
'dbtSecurityConfig' | 'dbtPrefixConfig'
>;
export type DbtS3Creds = Pick<
SCredentials,

34
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.test.tsx
View File

@ -28,6 +28,11 @@ const mockSecurityConfigGCSValue = {
gcsConfig: {},
};
const mockPrefixConfig = {
dbtBucketName: 'Test Bucket',
dbtObjectPrefix: 'Test Prefix',
};
// const mockSecurityConfigGCSPath = {
// gcsConfig: 'gcsConfigPath',
// };
@ -41,6 +46,7 @@ const mockData = {
...mockSecurityConfigS3,
...mockSecurityConfigGCSValue,
},
dbtPrefixConfig: mockPrefixConfig,
};
jest.mock('./DBTLocalConfig', () => ({
@ -80,26 +86,34 @@ jest.mock('./DBTHttpConfig', () => ({
jest.mock('./DBTS3Config', () => ({
DBTS3Config: jest
.fn()
.mockImplementation(({ handleSecurityConfigChange }) => (
<div
data-testid="dbt-s3"
onClick={() => {
handleSecurityConfigChange(mockSecurityConfigS3);
}}>
DBT S3 Config
</div>
)),
.mockImplementation(
({ handleSecurityConfigChange, handlePrefixConfigChange }) => (
<div
data-testid="dbt-s3"
onClick={() => {
handleSecurityConfigChange(mockSecurityConfigS3);
handlePrefixConfigChange(mockPrefixConfig);
}}>
DBT S3 Config
</div>
)
),
}));
jest.mock('./DBTGCSConfig', () => ({
DBTGCSConfig: jest
.fn()
.mockImplementation(
({ handleSecurityConfigChange, handleGcsTypeChange }) => (
({
handleSecurityConfigChange,
handlePrefixConfigChange,
handleGcsTypeChange,
}) => (
<div
data-testid="dbt-gcs"
onClick={() => {
handleSecurityConfigChange(mockSecurityConfigS3);
handlePrefixConfigChange(mockPrefixConfig);
}}
onMouseDown={() => {
handleGcsTypeChange(GCS_CONFIG.GCSValues);

12
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx
View File

@ -13,6 +13,7 @@
import React, { Fragment, FunctionComponent, useEffect, useState } from 'react';
import {
DBTBucketDetails,
DbtConfigSource,
SCredentials,
} from '../../../generated/metadataIngestion/databaseServiceMetadataPipeline';
@ -42,7 +43,7 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
const updateDbtConfig = (
key: keyof DbtConfigSource,
val?: string | SCredentials
val?: string | SCredentials | DBTBucketDetails
) => {
setDbtConfig((pre) => {
return { ...pre, [key]: val };
@ -91,7 +92,11 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
return (
<DBTS3Config
cancelText={cancelText}
dbtPrefixConfig={dbtConfig.dbtPrefixConfig}
dbtSecurityConfig={dbtConfig.dbtSecurityConfig}
handlePrefixConfigChange={(val) => {
updateDbtConfig('dbtPrefixConfig', val);
}}
handleSecurityConfigChange={(val) => {
updateDbtConfig('dbtSecurityConfig', val);
}}
@ -106,11 +111,14 @@ const DBTConfigFormBuilder: FunctionComponent<DBTConfigFormProps> = ({
return (
<DBTGCSConfig
cancelText={cancelText}
dbtPrefixConfig={dbtConfig.dbtPrefixConfig}
dbtSecurityConfig={dbtConfig.dbtSecurityConfig}
gcsType={gcsType}
handleGcsTypeChange={(type) => {
handleGcsTypeChange && handleGcsTypeChange(type);
setDbtConfig(data);
}}
handlePrefixConfigChange={(val) => {
updateDbtConfig('dbtPrefixConfig', val);
}}
handleSecurityConfigChange={(val) => {
updateDbtConfig('dbtSecurityConfig', val);

10
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTFormConstants.ts
View File

@ -25,23 +25,23 @@ import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum';
export const DBTSources: Array<DropDownListItem> = [
{
name: 'None',
name: 'No Config Source',
value: '',
},
{
name: 'DBT Local Config Source',
name: 'Local Config Source',
value: DBT_SOURCES.local,
},
{
name: 'DBT HTTP Config Source',
name: 'HTTP Config Source',
value: DBT_SOURCES.http,
},
{
name: 'DBT S3 Config Source',
name: 'S3 Config Source',
value: DBT_SOURCES.s3,
},
{
name: 'DBT GCS Config Source',
name: 'GCS Config Source',
value: DBT_SOURCES.gcs,
},
];

58
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.test.tsx
View File

@ -18,6 +18,7 @@ import { DBTGCSConfig } from './DBTGCSConfig';
const mockCancel = jest.fn();
const mockSubmit = jest.fn();
const mockPrefixConfigChange = jest.fn();
const mockSecurityConfigChange = jest.fn();
const mockProps = {
@ -26,6 +27,7 @@ const mockProps = {
gcsType: GCS_CONFIG.GCSValues,
onCancel: mockCancel,
onSubmit: mockSubmit,
handlePrefixConfigChange: mockPrefixConfigChange,
handleSecurityConfigChange: mockSecurityConfigChange,
};
@ -49,6 +51,8 @@ describe('Test DBT GCS Config Form', () => {
container,
'client-x509-certificate-uri'
);
const inputBucketName = getByTestId(container, 'dbt-bucket-name');
const inputObjPrefix = getByTestId(container, 'dbt-object-prefix');
expect(selectGcsConfig).toBeInTheDocument();
expect(inputCredsType).toBeInTheDocument();
@ -61,6 +65,8 @@ describe('Test DBT GCS Config Form', () => {
expect(inputTokenUri).toBeInTheDocument();
expect(inputAuthCertUri).toBeInTheDocument();
expect(inputClientCertUri).toBeInTheDocument();
expect(inputBucketName).toBeInTheDocument();
expect(inputObjPrefix).toBeInTheDocument();
});
it('Fields should render for gcs credential path', async () => {
@ -237,6 +243,34 @@ describe('Test DBT GCS Config Form', () => {
expect(inputCredsPath).toHaveValue('GcsCredPath');
});
it('dbt-bucket-name should render data', async () => {
const { container } = render(
<DBTGCSConfig
{...mockProps}
dbtPrefixConfig={{
dbtBucketName: 'Test Bucket',
}}
/>
);
const inputBucketName = getByTestId(container, 'dbt-bucket-name');
expect(inputBucketName).toHaveValue('Test Bucket');
});
it('dbt-object-prefix should render data', async () => {
const { container } = render(
<DBTGCSConfig
{...mockProps}
dbtPrefixConfig={{
dbtObjectPrefix: 'Test Prefix',
}}
/>
);
const inputObjPrefix = getByTestId(container, 'dbt-object-prefix');
expect(inputObjPrefix).toHaveValue('Test Prefix');
});
it('security config should change', async () => {
const { container } = render(<DBTGCSConfig {...mockProps} />);
const inputCredsType = getByTestId(container, 'credential-type');
@ -319,6 +353,26 @@ describe('Test DBT GCS Config Form', () => {
expect(mockSecurityConfigChange).toBeCalledTimes(10);
});
it('prefix config should change', async () => {
const { container } = render(<DBTGCSConfig {...mockProps} />);
const inputBucketName = getByTestId(container, 'dbt-bucket-name');
const inputObjPrefix = getByTestId(container, 'dbt-object-prefix');
fireEvent.change(inputBucketName, {
target: {
value: 'Test Bucket',
},
});
fireEvent.change(inputObjPrefix, {
target: {
value: 'Test Prefix',
},
});
expect(mockPrefixConfigChange).toBeCalledTimes(2);
});
it('should show errors on submit', async () => {
const { container } = render(<DBTGCSConfig {...mockProps} />);
const submitBtn = getByTestId(container, 'submit-btn');
@ -332,6 +386,10 @@ describe('Test DBT GCS Config Form', () => {
const { container } = render(
<DBTGCSConfig
{...mockProps}
dbtPrefixConfig={{
dbtBucketName: 'Test Bucket',
dbtObjectPrefix: 'Test Prefix',
}}
dbtSecurityConfig={{
gcsConfig: {
type: 'CredsType',

59
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.tsx
View File

@ -11,7 +11,7 @@
* limitations under the License.
*/
import { isEmpty, isString } from 'lodash';
import { isEmpty, isObject, isString } from 'lodash';
import React, {
Fragment,
FunctionComponent,
@ -20,6 +20,7 @@ import React, {
useState,
} from 'react';
import {
DBTBucketDetails,
DbtConfigSource,
GCSCredentialsValues,
SCredentials,
@ -48,10 +49,12 @@ interface Props extends DBTFormCommonProps, DbtConfigS3GCS {
gcsType?: GCS_CONFIG;
handleGcsTypeChange?: (type: GCS_CONFIG) => void;
handleSecurityConfigChange: (value?: SCredentials) => void;
handlePrefixConfigChange: (value: DBTBucketDetails) => void;
}
export const DBTGCSConfig: FunctionComponent<Props> = ({
dbtSecurityConfig,
dbtPrefixConfig,
gcsType = GCS_CONFIG.GCSValues,
okText,
cancelText,
@ -59,13 +62,16 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
onSubmit,
handleGcsTypeChange,
handleSecurityConfigChange,
handlePrefixConfigChange,
}: Props) => {
const isMounted = useRef<boolean>(false);
const updateGCSCredsConfig = (
key: keyof GCSCredentialsValues,
val: string
) => {
const gcsConfig = dbtSecurityConfig?.gcsConfig || {};
const gcsConfig = isObject(dbtSecurityConfig?.gcsConfig)
? dbtSecurityConfig?.gcsConfig
: {};
const updatedCreds: SCredentials = {
gcsConfig: {
...(gcsConfig as GCSCredentialsValues),
@ -75,6 +81,14 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
handleSecurityConfigChange(updatedCreds);
};
const updateDbtBucket = (key: keyof DBTBucketDetails, val: string) => {
const updatedBucket: DBTBucketDetails = {
...dbtPrefixConfig,
[key]: val,
};
handlePrefixConfigChange(updatedBucket);
};
const updateGCSCredsPath = (val: string) => {
const updatedCreds: SCredentials = {
gcsConfig: val,
@ -114,7 +128,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
};
const handleSubmit = () => {
const submitData = { dbtSecurityConfig };
const submitData = { dbtSecurityConfig, dbtPrefixConfig };
if (validate(submitData)) {
onSubmit(submitData);
}
@ -395,7 +409,44 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
{gcsType === GCS_CONFIG.GCSValues
? gcsCredConfigs(dbtSecurityConfig?.gcsConfig as GCSCredentialsValues)
: gcsCredPath(dbtSecurityConfig?.gcsConfig as string)}
<Field>
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="dbt-bucket-name">
DBT Bucket Name
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Name of the bucket where the dbt files are stored.
</p>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="dbt-bucket-name"
id="dbt-bucket-name"
name="dbt-bucket-name"
type="text"
value={dbtPrefixConfig?.dbtBucketName}
onChange={(e) => updateDbtBucket('dbtBucketName', e.target.value)}
/>
</Field>
<Field>
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="dbt-object-prefix">
DBT Object Prefix
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Path of the folder where the dbt files are stored.
</p>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="dbt-object-prefix"
id="dbt-object-prefix"
name="dbt-object-prefix"
type="text"
value={dbtPrefixConfig?.dbtObjectPrefix}
onChange={(e) => updateDbtBucket('dbtObjectPrefix', e.target.value)}
/>
</Field>
{getSeparator('')}
<Field className="tw-flex tw-justify-end">

58
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.test.tsx
View File

@ -17,6 +17,7 @@ import { DBTS3Config } from './DBTS3Config';
const mockCancel = jest.fn();
const mockSubmit = jest.fn();
const mockPrefixConfigChange = jest.fn();
const mockSecurityConfigChange = jest.fn();
const mockProps = {
@ -24,6 +25,7 @@ const mockProps = {
cancelText: 'Back',
onCancel: mockCancel,
onSubmit: mockSubmit,
handlePrefixConfigChange: mockPrefixConfigChange,
handleSecurityConfigChange: mockSecurityConfigChange,
};
@ -35,12 +37,16 @@ describe('Test DBT S3 Config Form', () => {
const inputRegion = getByTestId(container, 'aws-region');
const inputSessionToken = getByTestId(container, 'aws-session-token');
const inputEndpointUrl = getByTestId(container, 'endpoint-url');
const inputBucketName = getByTestId(container, 'dbt-bucket-name');
const inputObjPrefix = getByTestId(container, 'dbt-object-prefix');
expect(inputAccessKeyId).toBeInTheDocument();
expect(inputSecretKey).toBeInTheDocument();
expect(inputRegion).toBeInTheDocument();
expect(inputSessionToken).toBeInTheDocument();
expect(inputEndpointUrl).toBeInTheDocument();
expect(inputBucketName).toBeInTheDocument();
expect(inputObjPrefix).toBeInTheDocument();
});
it('access-key should render data', async () => {
@ -100,6 +106,34 @@ describe('Test DBT S3 Config Form', () => {
expect(inputEndpointUrl).toHaveValue('EndpointUrl');
});
it('dbt-bucket-name should render data', async () => {
const { container } = render(
<DBTS3Config
{...mockProps}
dbtPrefixConfig={{
dbtBucketName: 'Test Bucket',
}}
/>
);
const inputBucketName = getByTestId(container, 'dbt-bucket-name');
expect(inputBucketName).toHaveValue('Test Bucket');
});
it('dbt-object-prefix should render data', async () => {
const { container } = render(
<DBTS3Config
{...mockProps}
dbtPrefixConfig={{
dbtObjectPrefix: 'Test Prefix',
}}
/>
);
const inputObjPrefix = getByTestId(container, 'dbt-object-prefix');
expect(inputObjPrefix).toHaveValue('Test Prefix');
});
it('security config should change', async () => {
const { container } = render(<DBTS3Config {...mockProps} />);
const inputAccessKeyId = getByTestId(container, 'aws-access-key-id');
@ -141,6 +175,26 @@ describe('Test DBT S3 Config Form', () => {
expect(mockSecurityConfigChange).toBeCalledTimes(5);
});
it('prefix config should change', async () => {
const { container } = render(<DBTS3Config {...mockProps} />);
const inputBucketName = getByTestId(container, 'dbt-bucket-name');
const inputObjPrefix = getByTestId(container, 'dbt-object-prefix');
fireEvent.change(inputBucketName, {
target: {
value: 'Test Bucket',
},
});
fireEvent.change(inputObjPrefix, {
target: {
value: 'Test Prefix',
},
});
expect(mockPrefixConfigChange).toBeCalledTimes(2);
});
it('should show errors on submit', async () => {
const { container } = render(<DBTS3Config {...mockProps} />);
const submitBtn = getByTestId(container, 'submit-btn');
@ -154,6 +208,10 @@ describe('Test DBT S3 Config Form', () => {
const { container } = render(
<DBTS3Config
{...mockProps}
dbtPrefixConfig={{
dbtBucketName: 'Test Bucket',
dbtObjectPrefix: 'Test Prefix',
}}
dbtSecurityConfig={{
awsAccessKeyId: 'AccessKeyId',
awsSecretAccessKey: 'SecretKey',

52
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.tsx
View File

@ -13,6 +13,7 @@
import React, { Fragment, FunctionComponent, useState } from 'react';
import {
DBTBucketDetails,
DbtConfigSource,
SCredentials,
} from '../../../generated/metadataIngestion/databaseServiceMetadataPipeline';
@ -35,15 +36,18 @@ import {
interface Props extends DBTFormCommonProps, DbtConfigS3GCS {
handleSecurityConfigChange: (value: SCredentials) => void;
handlePrefixConfigChange: (value: DBTBucketDetails) => void;
}
export const DBTS3Config: FunctionComponent<Props> = ({
dbtSecurityConfig,
dbtPrefixConfig,
okText,
cancelText,
onCancel,
onSubmit,
handleSecurityConfigChange,
handlePrefixConfigChange,
}: Props) => {
const updateS3Creds = (key: keyof SCredentials, val: string) => {
const updatedCreds: SCredentials = {
@ -54,6 +58,14 @@ export const DBTS3Config: FunctionComponent<Props> = ({
handleSecurityConfigChange(updatedCreds);
};
const updateDbtBucket = (key: keyof DBTBucketDetails, val: string) => {
const updatedBucket: DBTBucketDetails = {
...dbtPrefixConfig,
[key]: val,
};
handlePrefixConfigChange(updatedBucket);
};
const [errors, setErrors] = useState<ErrorDbtS3>();
const validate = (data: DbtConfigSource) => {
const { isValid, errors: reqErrors } = validateDbtS3Config(
@ -68,7 +80,7 @@ export const DBTS3Config: FunctionComponent<Props> = ({
};
const handleSubmit = () => {
const submitData = { dbtSecurityConfig };
const submitData = { dbtSecurityConfig, dbtPrefixConfig };
if (validate(submitData)) {
onSubmit(submitData);
}
@ -173,6 +185,44 @@ export const DBTS3Config: FunctionComponent<Props> = ({
/>
{errors?.endPointURL && errorMsg(errors.endPointURL)}
</Field>
<Field>
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="dbt-bucket-name">
DBT Bucket Name
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Name of the bucket where the dbt files are stored.
</p>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="dbt-bucket-name"
id="dbt-bucket-name"
name="dbt-bucket-name"
type="text"
value={dbtPrefixConfig?.dbtBucketName}
onChange={(e) => updateDbtBucket('dbtBucketName', e.target.value)}
/>
</Field>
<Field>
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="dbt-object-prefix">
DBT Object Prefix
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Path of the folder where the dbt files are stored.
</p>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="dbt-object-prefix"
id="dbt-object-prefix"
name="dbt-object-prefix"
type="text"
value={dbtPrefixConfig?.dbtObjectPrefix}
onChange={(e) => updateDbtBucket('dbtObjectPrefix', e.target.value)}
/>
</Field>
{getSeparator('')}
<Field className="tw-flex tw-justify-end">

1
openmetadata-ui/src/main/resources/ui/src/enums/filterPattern.enum.ts
View File

@ -12,6 +12,7 @@
*/
export enum FilterPatternEnum {
DATABASE = 'database',
SCHEMA = 'schema',
TABLE = 'table',
CHART = 'chart',