yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-17 03:38:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix (ui): Add strategy and threshold to the SQL test #11848 (#11946)

Browse Source
This commit is contained in:
Shailesh Parmar 2023-06-12 16:29:11 +05:30 committed by GitHub
parent b2ae9d498c
commit 5e7fbf9e4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 250 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/EditTestCaseModal.tsx
View File

@ -26,7 +26,6 @@ import React, {
import { useTranslation } from 'react-i18next';
import { getTableDetailsByFQN } from 'rest/tableAPI';
import { getTestDefinitionById, updateTestCaseById } from 'rest/testAPI';
import { CSMode } from '../../enums/codemirror.enum';
import { TestCaseParameterValue } from '../../generated/tests/testCase';
import {
TestDataType,
@ -38,7 +37,6 @@ import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
import RichTextEditor from '../common/rich-text-editor/RichTextEditor';
import { EditorContentRef } from '../common/rich-text-editor/RichTextEditor.interface';
import Loader from '../Loader/Loader';
import SchemaEditor from '../schema-editor/SchemaEditor';
import { EditTestCaseModalProps } from './AddDataQualityTest.interface';
import ParameterForm from './components/ParameterForm';
@ -52,12 +50,6 @@ const EditTestCaseModal: React.FC<EditTestCaseModalProps> = ({
const [form] = Form.useForm();
const [selectedDefinition, setSelectedDefinition] =
useState<TestDefinition>();
const [sqlQuery, setSqlQuery] = useState(
testCase?.parameterValues?.[0] ?? {
name: 'sqlExpression',
value: '',
}
);
const [isLoading, setIsLoading] = useState(true);
const [isLoadingOnSave, setIsLoadingOnSave] = useState(false);
const [table, setTable] = useState<Table>();
@ -70,34 +62,12 @@ const EditTestCaseModal: React.FC<EditTestCaseModalProps> = ({
);
const GenerateParamsField = useCallback(() => {
if (selectedDefinition && selectedDefinition.parameterDefinition) {
const name = selectedDefinition.parameterDefinition[0]?.name;
if (name === 'sqlExpression') {
return (
<Form.Item
data-testid="sql-editor-container"
key={name}
label={t('label.sql-uppercase-query')}
name={name}
tooltip={t('message.sql-query-tooltip')}>
<SchemaEditor
className="custom-query-editor query-editor-h-200"
mode={{ name: CSMode.SQL }}
options={{
readOnly: false,
}}
value={sqlQuery.value || ''}
onChange={(value) => setSqlQuery((pre) => ({ ...pre, value }))}
/>
</Form.Item>
);
}
if (selectedDefinition?.parameterDefinition) {
return <ParameterForm definition={selectedDefinition} table={table} />;
}
return;
}, [testCase, selectedDefinition, sqlQuery, table]);
}, [testCase, selectedDefinition, table]);
const fetchTestDefinitionById = async () => {
setIsLoading(true);
@ -120,19 +90,18 @@ const EditTestCaseModal: React.FC<EditTestCaseModalProps> = ({
}) => {
const paramsValue = selectedDefinition?.parameterDefinition?.[0];
const parameterValues =
paramsValue?.name === 'sqlExpression'
? [sqlQuery]
: Object.entries(value.params || {}).map(([key, value]) => ({
name: key,
value:
paramsValue?.dataType === TestDataType.Array
? // need to send array as string formate
JSON.stringify(
(value as { value: string }[]).map((data) => data.value)
)
: value,
}));
const parameterValues = Object.entries(value.params || {}).map(
([key, value]) => ({
name: key,
value:
paramsValue?.dataType === TestDataType.Array
? // need to send array as string formate
JSON.stringify(
(value as { value: string }[]).map((data) => data.value)
)
: value,
})
);
return {
parameterValues: parameterValues as TestCaseParameterValue[],
@ -201,12 +170,7 @@ const EditTestCaseModal: React.FC<EditTestCaseModalProps> = ({
getEntityFqnFromEntityLink(testCase?.entityLink, isColumn)
),
});
setSqlQuery(
testCase?.parameterValues?.[0] ?? {
name: 'sqlExpression',
value: '',
}
);
const isContainsColumnName = testCase.parameterValues?.find(
(value) => value.name === 'columnName'
);
@ -240,7 +204,6 @@ const EditTestCaseModal: React.FC<EditTestCaseModalProps> = ({
<Form
className="tw-h-70vh tw-overflow-auto"
form={form}
initialValues={{ sqlExpression: sqlQuery.value }}
layout="vertical"
name="tableTestForm"
onFinish={handleFormSubmit}>

33
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/ParameterForm.test.tsx
View File

@ -15,12 +15,17 @@ import { act, render, screen } from '@testing-library/react';
import { TestDefinition } from 'generated/tests/testDefinition';
import {
MOCK_TABLE_COLUMN_NAME_TO_EXIST,
MOCK_TABLE_CUSTOM_SQL_QUERY,
MOCK_TABLE_ROW_INSERTED_COUNT_TO_BE_BETWEEN,
MOCK_TABLE_WITH_DATE_TIME_COLUMNS,
} from 'mocks/TestSuite.mock';
import React from 'react';
import ParameterForm from './ParameterForm';
jest.mock('components/schema-editor/SchemaEditor', () => {
return jest.fn().mockReturnValue(<div>SchemaEditor</div>);
});
describe('ParameterForm component test', () => {
it('Select box should render if "columnName" field is present and table data provided', async () => {
await act(async () => {
@ -81,4 +86,32 @@ describe('ParameterForm component test', () => {
MOCK_TABLE_COLUMN_NAME_TO_EXIST.parameterDefinition.length
);
});
it('Query editor should render if "sqlExpression" field is present', async () => {
await act(async () => {
render(
<ParameterForm
definition={MOCK_TABLE_CUSTOM_SQL_QUERY as TestDefinition}
/>
);
});
const sqlEditor = await screen.findByText('SchemaEditor');
expect(sqlEditor).toBeInTheDocument();
});
it('Should render select box if optionValues are provided', async () => {
await act(async () => {
render(
<ParameterForm
definition={MOCK_TABLE_CUSTOM_SQL_QUERY as TestDefinition}
/>
);
});
const selectBox = await screen.findByRole('combobox');
expect(selectBox).toBeInTheDocument();
});
});

261
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/ParameterForm.tsx
View File

@ -14,7 +14,9 @@
import { PlusOutlined } from '@ant-design/icons';
import { Button, Form, Input, InputNumber, Select, Switch } from 'antd';
import 'codemirror/addon/fold/foldgutter.css';
import SchemaEditor from 'components/schema-editor/SchemaEditor';
import { SUPPORTED_PARTITION_TYPE_FOR_DATE_TIME } from 'constants/profiler.constant';
import { CSMode } from 'enums/codemirror.enum';
import { isUndefined } from 'lodash';
import React from 'react';
import { useTranslation } from 'react-i18next';
@ -37,135 +39,160 @@ const ParameterForm: React.FC<ParameterFormProps> = ({ definition, table }) => {
})}`}
/>
);
switch (data.dataType) {
case TestDataType.String:
if (
!isUndefined(table) &&
definition.name === 'tableRowInsertedCountToBeBetween' &&
data.name === 'columnName'
) {
const partitionColumnOptions = table.columns.reduce(
(result, column) => {
if (
SUPPORTED_PARTITION_TYPE_FOR_DATE_TIME.includes(column.dataType)
) {
return [
...result,
{
value: column.name,
label: column.name,
},
];
}
if (data.optionValues?.length) {
Field = (
<Select
placeholder={`${t('label.please-select-entity', {
entity: data.displayName,
})}`}>
{data.optionValues.map((value) => (
<Select.Option key={value}>{value}</Select.Option>
))}
</Select>
);
} else {
switch (data.dataType) {
case TestDataType.String:
if (
!isUndefined(table) &&
definition.name === 'tableRowInsertedCountToBeBetween' &&
data.name === 'columnName'
) {
const partitionColumnOptions = table.columns.reduce(
(result, column) => {
if (
SUPPORTED_PARTITION_TYPE_FOR_DATE_TIME.includes(
column.dataType
)
) {
return [
...result,
{
value: column.name,
label: column.name,
},
];
}
return result;
},
[] as { value: string; label: string }[]
);
return result;
},
[] as { value: string; label: string }[]
);
Field = (
<Select
options={partitionColumnOptions}
placeholder={t('message.select-column-name')}
/>
);
} else if (data.name === 'sqlExpression') {
Field = (
<SchemaEditor
className="custom-query-editor query-editor-h-200"
mode={{ name: CSMode.SQL }}
options={{
readOnly: false,
}}
/>
);
} else {
Field = (
<Input
placeholder={`${t('message.enter-a-field', {
field: data.displayName,
})}`}
/>
);
}
break;
case TestDataType.Number:
case TestDataType.Int:
case TestDataType.Decimal:
case TestDataType.Double:
case TestDataType.Float:
Field = (
<Select
options={partitionColumnOptions}
placeholder={t('message.select-column-name')}
/>
);
} else {
Field = (
<Input
<InputNumber
className="tw-w-full"
placeholder={`${t('message.enter-a-field', {
field: data.displayName,
})}`}
/>
);
}
break;
case TestDataType.Number:
case TestDataType.Int:
case TestDataType.Decimal:
case TestDataType.Double:
case TestDataType.Float:
Field = (
<InputNumber
className="tw-w-full"
placeholder={`${t('message.enter-a-field', {
field: data.displayName,
})}`}
/>
);
break;
case TestDataType.Boolean:
Field = <Switch />;
break;
case TestDataType.Boolean:
Field = <Switch />;
break;
case TestDataType.Array:
case TestDataType.Set:
Field = (
<Input
placeholder={`${t('message.enter-comma-separated-field', {
field: data.displayName,
})}`}
/>
);
break;
case TestDataType.Array:
case TestDataType.Set:
Field = (
<Input
placeholder={`${t('message.enter-comma-separated-field', {
field: data.displayName,
})}`}
/>
);
return (
<Form.List
initialValue={[{ value: '' }]}
key={data.name}
name={data.name || ''}>
{(fields, { add, remove }) => (
<Form.Item
key={data.name}
label={
<span>
<span className="tw-mr-3">{data.displayName}:</span>
<Button
icon={<PlusOutlined />}
size="small"
type="primary"
onClick={() => add()}
/>
</span>
}
name={data.name}
tooltip={data.description}>
{fields.map(({ key, name, ...restField }) => (
<div className="d-flex tw-gap-2 tw-w-full" key={key}>
<Form.Item
className="tw-w-11/12 tw-mb-4"
{...restField}
name={[name, 'value']}
rules={[
{
required: data.required,
message: `${t('message.field-text-is-required', {
fieldText: data.displayName,
})}`,
},
]}>
<Input
placeholder={`${t('message.enter-a-field', {
field: data.displayName,
})}`}
return (
<Form.List
initialValue={[{ value: '' }]}
key={data.name}
name={data.name || ''}>
{(fields, { add, remove }) => (
<Form.Item
key={data.name}
label={
<span>
<span className="tw-mr-3">{data.displayName}:</span>
<Button
icon={<PlusOutlined />}
size="small"
type="primary"
onClick={() => add()}
/>
</Form.Item>
<Button
icon={
<SVGIcons
alt="delete"
className="tw-w-4"
icon={Icons.DELETE}
</span>
}
name={data.name}
tooltip={data.description}>
{fields.map(({ key, name, ...restField }) => (
<div className="d-flex tw-gap-2 tw-w-full" key={key}>
<Form.Item
className="tw-w-11/12 tw-mb-4"
{...restField}
name={[name, 'value']}
rules={[
{
required: data.required,
message: `${t('message.field-text-is-required', {
fieldText: data.displayName,
})}`,
},
]}>
<Input
placeholder={`${t('message.enter-a-field', {
field: data.displayName,
})}`}
/>
}
type="text"
onClick={() => remove(name)}
/>
</div>
))}
</Form.Item>
)}
</Form.List>
);
</Form.Item>
<Button
icon={
<SVGIcons
alt="delete"
className="tw-w-4"
icon={Icons.DELETE}
/>
}
type="text"
onClick={() => remove(name)}
/>
</div>
))}
</Form.Item>
)}
</Form.List>
);
}
}
return (

55
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/TestCaseForm.tsx
View File

@ -21,7 +21,6 @@ import { useParams } from 'react-router-dom';
import { getListTestCase, getListTestDefinitions } from 'rest/testAPI';
import { getEntityName } from 'utils/EntityUtils';
import { API_RES_MAX_SIZE } from '../../../constants/constants';
import { CSMode } from '../../../enums/codemirror.enum';
import { ProfilerDashboardType } from '../../../enums/table.enum';
import {
TestCase,
@ -42,7 +41,6 @@ import { generateEntityLink } from '../../../utils/TableUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
import RichTextEditor from '../../common/rich-text-editor/RichTextEditor';
import { EditorContentRef } from '../../common/rich-text-editor/RichTextEditor.interface';
import SchemaEditor from '../../schema-editor/SchemaEditor';
import { TestCaseFormProps } from '../AddDataQualityTest.interface';
import ParameterForm from './ParameterForm';
@ -62,10 +60,6 @@ const TestCaseForm: React.FC<TestCaseFormProps> = ({
initialValue?.testDefinition
);
const [testCases, setTestCases] = useState<TestCase[]>([]);
const [sqlQuery, setSqlQuery] = useState({
name: 'sqlExpression',
value: initialValue?.parameterValues?.[0]?.value || '',
});
const fetchAllTestDefinitions = async () => {
try {
@ -112,33 +106,11 @@ const TestCaseForm: React.FC<TestCaseFormProps> = ({
const GenerateParamsField = useCallback(() => {
const selectedDefinition = getSelectedTestDefinition();
if (selectedDefinition && selectedDefinition.parameterDefinition) {
const name = selectedDefinition.parameterDefinition[0]?.name;
if (name === 'sqlExpression') {
return (
<Form.Item
data-testid="sql-editor-container"
key={name}
label={t('label.sql-uppercase-query')}
name={name}
tooltip={t('message.queries-result-test')}>
<SchemaEditor
className="custom-query-editor query-editor-h-200"
mode={{ name: CSMode.SQL }}
options={{
readOnly: false,
}}
value={sqlQuery.value || ''}
onChange={(value) => setSqlQuery((pre) => ({ ...pre, value }))}
/>
</Form.Item>
);
}
return <ParameterForm definition={selectedDefinition} table={table} />;
}
return;
}, [selectedTestType, initialValue, testDefinitions, sqlQuery]);
}, [selectedTestType, initialValue, testDefinitions]);
const createTestCaseObj = (value: {
testName: string;
@ -148,19 +120,18 @@ const TestCaseForm: React.FC<TestCaseFormProps> = ({
const selectedDefinition = getSelectedTestDefinition();
const paramsValue = selectedDefinition?.parameterDefinition?.[0];
const parameterValues =
paramsValue?.name === 'sqlExpression'
? [sqlQuery]
: Object.entries(value.params || {}).map(([key, value]) => ({
name: key,
value:
paramsValue?.dataType === TestDataType.Array
? // need to send array as string formate
JSON.stringify(
(value as { value: string }[]).map((data) => data.value)
)
: value,
}));
const parameterValues = Object.entries(value.params || {}).map(
([key, value]) => ({
name: key,
value:
paramsValue?.dataType === TestDataType.Array
? // need to send array as string formate
JSON.stringify(
(value as { value: string }[]).map((data) => data.value)
)
: value,
})
);
return {
name: value.testName,

4
openmetadata-ui/src/main/resources/ui/src/components/Team/TeamDetails/TeamDetailsV1.tsx
View File

@ -55,7 +55,7 @@ import {
import { ExtraInfo } from 'Models';
import AddAttributeModal from 'pages/RolesPage/AddAttributeModal/AddAttributeModal';
import { ImportType } from 'pages/teams/ImportTeamsPage/ImportTeamsPage.interface';
import { default as Qs, default as QueryString } from 'qs';
import Qs from 'qs';
import React, {
Fragment,
useCallback,
@ -665,7 +665,7 @@ const TeamDetailsV1 = ({
currentTeam.name,
EntityAction.IMPORT
),
search: QueryString.stringify({ type: ImportType.TEAMS }),
search: Qs.stringify({ type: ImportType.TEAMS }),
});
}, []);

4
openmetadata-ui/src/main/resources/ui/src/components/schema-editor/SchemaEditor.tsx
View File

@ -33,7 +33,7 @@ type Mode = {
};
const SchemaEditor = ({
value,
value = '',
className = '',
mode = {
name: CSMode.JAVASCRIPT,
@ -43,7 +43,7 @@ const SchemaEditor = ({
editorClass,
onChange,
}: {
value: string;
value?: string;
className?: string;
mode?: Mode;
readOnly?: boolean;

39
openmetadata-ui/src/main/resources/ui/src/mocks/TestSuite.mock.ts
View File

@ -788,6 +788,45 @@ export const MOCK_TABLE_ROW_INSERTED_COUNT_TO_BE_BETWEEN = {
deleted: false,
};
export const MOCK_TABLE_CUSTOM_SQL_QUERY = {
id: '9fdc266a-f262-4607-aafb-34562926ab3c',
name: 'tableCustomSQLQuery',
displayName: 'Custom SQL Query',
fullyQualifiedName: 'tableCustomSQLQuery',
description: 'Test if a custom SQL returns 0 row or `COUNT(<x>) == 0`',
entityType: 'TABLE',
testPlatforms: ['OpenMetadata'],
supportedDataTypes: [],
parameterDefinition: [
{
name: 'sqlExpression',
displayName: 'SQL Expression',
dataType: 'STRING',
description: 'SQL expression to run against the table',
required: true,
optionValues: [],
},
{
name: 'strategy',
displayName: 'Strategy',
dataType: 'ARRAY',
description:
'Strategy to use to run the custom SQL query (i.e. `SELECT COUNT(<col>)` or `SELECT <col> (defaults to COUNT)',
required: false,
optionValues: ['ROWS', 'COUNT'],
},
{
name: 'threshold',
displayName: 'Threshold',
dataType: 'NUMBER',
description:
'Threshold to use to determine if the test passes or fails (defaults to 0).',
required: false,
optionValues: [],
},
],
};
export const MOCK_TABLE_COLUMN_NAME_TO_EXIST = {
id: '6d4e4673-fd7f-4b37-811e-7645c3c17e93',
name: 'tableColumnNameToExist',

4
openmetadata-ui/src/main/resources/ui/src/pages/teams/ImportTeamsPage/ImportTeamsPage.test.tsx
View File

@ -198,7 +198,7 @@ describe('ImportTeamsPage', () => {
).toBeInTheDocument();
});
it('On click of cancel, redirect to team tab of teams page', async () => {
it('Should redirect to team tab of teams page on clicking of cancel', async () => {
act(() => {
render(<ImportTeamsPage />);
});
@ -242,7 +242,7 @@ describe('ImportTeamsPage', () => {
expect(await screen.findByText('UserImportResult')).toBeInTheDocument();
});
it('On click of cancel, redirect to users tab of teams page', async () => {
it('Should redirect to users tab of teams page on clicking of cancel', async () => {
mockLocation.search = '?type=users';
act(() => {
render(<ImportTeamsPage />);