yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-22 07:58:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Fixes #5678] Implement frontend logic for data quality input field (#5682)

Browse Source 
* Added input fields for new table tests

* Added forms for data quality tests

* Fix error in test validation for new tests

* fixed failing test

* Added toggle form to force test to run on column order or no

Co-authored-by: Shailesh Parmar <shailesh.parmar.webdev@gmail.com>
This commit is contained in:
Teddy 2022-06-28 14:58:29 +02:00 committed by GitHub
parent 408dd02be9
commit 7a131b115f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 267 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx
View File

@ -78,6 +78,11 @@ const ColumnTestForm = ({
const [forbiddenValues, setForbiddenValues] = useState<(string | number)[]>(
data?.testCase?.config?.forbiddenValues || ['']
);
const [allowedValues, setAllowedValues] = useState<(string | number)[]>(
data?.testCase?.config?.allowedValues || ['']
);
const [isShowError, setIsShowError] = useState({
testName: false,
columnName: false,
@ -85,6 +90,7 @@ const ColumnTestForm = ({
minOrMax: false,
missingCountValue: false,
values: false,
inSetValues: false,
minMaxValue: false,
allTestAdded: false,
notSupported: false,
@ -119,6 +125,23 @@ const ColumnTestForm = ({
setIsShowError({ ...isShowError, values: false });
};
const addInSetValueFields = () => {
setAllowedValues([...allowedValues, '']);
};
const removeInSetValueFields = (i: number) => {
const newFormValues = [...allowedValues];
newFormValues.splice(i, 1);
setAllowedValues(newFormValues);
};
const handleInSetValueFieldsChange = (i: number, value: string) => {
const newFormValues = [...allowedValues];
newFormValues[i] = value;
setAllowedValues(newFormValues);
setIsShowError({ ...isShowError, values: false });
};
const addMatchFields = () => {
setMissingValueMatch([...missingValueMatch, '']);
};
@ -194,6 +217,9 @@ const ColumnTestForm = ({
switch (columnTest) {
case ColumnTestType.ColumnValueLengthsToBeBetween:
case ColumnTestType.ColumnValuesToBeBetween:
case ColumnTestType.ColumnValueMaxToBeBetween:
case ColumnTestType.ColumnValueMinToBeBetween:
case ColumnTestType.ColumnValuesSumToBeBetween:
errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue);
if (!isUndefined(minValue) && !isUndefined(maxValue)) {
errMsg.minMaxValue = (+minValue as number) > (+maxValue as number);
@ -213,6 +239,13 @@ const ColumnTestForm = ({
break;
}
case ColumnTestType.ColumnValuesToBeInSet: {
const actualValues = allowedValues.filter((v) => !isEmpty(v));
errMsg.inSetValues = actualValues.length < 1;
break;
}
case ColumnTestType.ColumnValuesToMatchRegex:
errMsg.regex = isEmpty(regex);
@ -237,6 +270,24 @@ const ColumnTestForm = ({
maxValue: !isEmpty(maxValue) ? maxValue : undefined,
};
case ColumnTestType.ColumnValueMaxToBeBetween:
return {
minValueForMaxInCol: !isEmpty(minValue) ? minValue : undefined,
maxValueForMaxInCol: !isEmpty(maxValue) ? maxValue : undefined,
};
case ColumnTestType.ColumnValueMinToBeBetween:
return {
minValueForMinInCol: !isEmpty(minValue) ? minValue : undefined,
maxValueForMinInCol: !isEmpty(maxValue) ? maxValue : undefined,
};
case ColumnTestType.ColumnValuesSumToBeBetween:
return {
minValueForColSum: !isEmpty(minValue) ? minValue : undefined,
maxValueForColSum: !isEmpty(maxValue) ? maxValue : undefined,
};
case ColumnTestType.ColumnValuesMissingCountToBeEqual: {
const filterMatchValue = missingValueMatch.filter(
(value) => !isEmpty(value)
@ -254,6 +305,11 @@ const ColumnTestForm = ({
forbiddenValues: forbiddenValues.filter((v) => !isEmpty(v)),
};
case ColumnTestType.ColumnValuesToBeInSet:
return {
allowedValues: allowedValues.filter((v) => !isEmpty(v)),
};
case ColumnTestType.ColumnValuesToMatchRegex:
return {
regex: regex,
@ -307,12 +363,14 @@ const ColumnTestForm = ({
isAcceptedTypeIsString.current =
columnDataType === 'varchar' || columnDataType === 'boolean';
setForbiddenValues(['']);
setAllowedValues(['']);
setColumnTest(value as ColumnTestType);
errorMsg.columnName = false;
errorMsg.regex = false;
errorMsg.minOrMax = false;
errorMsg.missingCountValue = false;
errorMsg.values = false;
errorMsg.inSetValues = false;
errorMsg.minMaxValue = false;
errorMsg.testName = false;
@ -342,6 +400,7 @@ const ColumnTestForm = ({
isAcceptedTypeIsString.current =
columnDataType === 'varchar' || columnDataType === 'boolean';
setForbiddenValues(['']);
setAllowedValues(['']);
setColumnName(value);
handleTestTypeOptionChange(value);
errorMsg.allTestAdded =
@ -555,10 +614,66 @@ const ColumnTestForm = ({
);
};
const getColumnValuesToBeInSetField = () => {
return (
<div data-testid="in-set-field">
<div className="tw-flex tw-items-center tw-mt-6">
<p className="w-form-label tw-mr-3">{requiredField('Values')}</p>
<Button
className="tw-h-5 tw-px-2"
size="x-small"
theme="primary"
variant="contained"
onClick={addInSetValueFields}>
<FontAwesomeIcon icon="plus" />
</Button>
</div>
{allowedValues.map((value, i) => (
<div className="tw-flex tw-items-center" key={i}>
<div className="tw-w-11/12">
<Field>
<input
className="tw-form-inputs tw-form-inputs-padding"
id={`option-key-${i}`}
name="key"
placeholder="Values to be in the set"
type={isAcceptedTypeIsString.current ? 'text' : 'number'}
value={value}
onChange={(e) =>
handleInSetValueFieldsChange(i, e.target.value)
}
/>
</Field>
</div>
<button
className="focus:tw-outline-none tw-mt-3 tw-w-1/12"
onClick={(e) => {
removeInSetValueFields(i);
e.preventDefault();
}}>
<SVGIcons
alt="delete"
icon="icon-delete"
title="Delete"
width="16px"
/>
</button>
</div>
))}
{isShowError.values && errorMsg('Value is required.')}
</div>
);
};
const getColumnTestConfig = () => {
switch (columnTest) {
case ColumnTestType.ColumnValueLengthsToBeBetween:
case ColumnTestType.ColumnValuesToBeBetween:
case ColumnTestType.ColumnValueMaxToBeBetween:
case ColumnTestType.ColumnValueMinToBeBetween:
case ColumnTestType.ColumnValuesSumToBeBetween:
return getMinMaxField();
case ColumnTestType.ColumnValuesMissingCountToBeEqual:
@ -567,6 +682,9 @@ const ColumnTestForm = ({
case ColumnTestType.ColumnValuesToBeNotInSet:
return getColumnValuesToBeNotInSetField();
case ColumnTestType.ColumnValuesToBeInSet:
return getColumnValuesToBeInSetField();
case ColumnTestType.ColumnValuesToMatchRegex:
return getColumnValuesToMatchRegexFields();

6
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.test.tsx
View File

@ -25,7 +25,8 @@ describe('Test TableTestForm component', () => {
const tableTestType = await findByTestId(container, 'tableTestType');
const cancelButton = await findByTestId(container, 'cancel-test');
const saveButton = await findByTestId(container, 'save-test');
const value = await findByTestId(container, 'value');
const min = await findByTestId(container, 'min');
const max = await findByTestId(container, 'max');
const description = await findByText(
container,
/MarkdownWithPreview component/i
@ -33,7 +34,8 @@ describe('Test TableTestForm component', () => {
expect(tableTestType).toBeInTheDocument();
expect(description).toBeInTheDocument();
expect(value).toBeInTheDocument();
expect(min).toBeInTheDocument();
expect(max).toBeInTheDocument();
expect(cancelButton).toBeInTheDocument();
expect(saveButton).toBeInTheDocument();
});

147
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.tsx
View File

@ -27,6 +27,7 @@ import {
} from '../../../utils/CommonUtils';
import { Button } from '../../buttons/Button/Button';
import RichTextEditor from '../../common/rich-text-editor/RichTextEditor';
import ToggleSwitchV1 from '../../common/toggle-switch/ToggleSwitchV1';
type Props = {
data?: TableTest;
@ -67,12 +68,26 @@ const TableTestForm = ({
const [value, setValue] = useState<number | undefined>(
data?.testCase.config?.value || data?.testCase.config?.columnCount
);
const [columnName, setcolumnName] = useState<string | undefined>(
data?.testCase?.config?.columnName
);
const [columnNameSet, setcolumnNameSet] = useState<string | undefined>(
data?.testCase?.config?.columnNames
);
const [columnNameSetOrdered, setcolumnNameSetOrdered] = useState<boolean>(
data?.testCase?.config?.ordered !== undefined
? data?.testCase?.config?.ordered
: false
);
const [isShowError, setIsShowError] = useState({
minOrMax: false,
values: false,
minMaxValue: false,
allTestAdded: false,
tableTest: false,
columnName: false,
columnNameSet: false,
columnNameSetOrdered: false,
});
const [testTypeOptions, setTestTypeOptions] = useState<string[]>();
@ -100,19 +115,29 @@ const TableTestForm = ({
const validateForm = () => {
const errMsg = cloneDeep(isShowError);
const isTableRowCountToBeBetweenTest =
tableTest === TableTestType.TableRowCountToBeBetween;
errMsg.tableTest = isEmpty(tableTest);
if (isTableRowCountToBeBetweenTest) {
errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue);
if (!isUndefined(minValue) && !isUndefined(maxValue)) {
errMsg.minMaxValue = (+minValue as number) > (+maxValue as number);
}
} else {
errMsg.values = isEmpty(value);
switch (tableTest) {
case TableTestType.TableRowCountToBeBetween:
case TableTestType.TableColumnCountToBeBetween:
errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue);
if (!isUndefined(minValue) && !isUndefined(maxValue)) {
errMsg.minMaxValue = (+minValue as number) > (+maxValue as number);
}
break;
case TableTestType.TableColumnNameToExist:
errMsg.columnName = isEmpty(columnName);
break;
case TableTestType.TableColumnToMatchSet:
errMsg.columnNameSet = isEmpty(columnNameSet);
break;
default:
errMsg.values = isEmpty(value);
}
setIsShowError(errMsg);
@ -128,6 +153,23 @@ const TableTestForm = ({
minValue: isEmpty(minValue) ? undefined : minValue,
};
case TableTestType.TableColumnCountToBeBetween:
return {
minColValue: isEmpty(minValue) ? undefined : minValue,
maxColValue: isEmpty(maxValue) ? undefined : maxValue,
};
case TableTestType.TableColumnNameToExist:
return {
columnName: isEmpty(columnName) ? undefined : columnName,
};
case TableTestType.TableColumnToMatchSet:
return {
columnNames: isEmpty(columnNameSet) ? undefined : columnNameSet,
ordered: columnNameSetOrdered,
};
case TableTestType.TableColumnCountToEqual:
return {
columnCount: isEmpty(value) ? undefined : value,
@ -196,6 +238,19 @@ const TableTestForm = ({
break;
case 'column-name':
setcolumnName(value as unknown as string);
errorMsg.columnName = false;
break;
case 'column-name-set':
setcolumnNameSet(value as unknown as string);
errorMsg.columnNameSet = false;
errorMsg.columnNameSetOrdered = false;
break;
default:
break;
}
@ -224,6 +279,56 @@ const TableTestForm = ({
);
};
const getValueColTextField = () => {
return (
<Field>
<label className="tw-block tw-form-label" htmlFor="column-name">
Column Name
</label>
<input
className="tw-form-inputs tw-form-inputs-padding"
data-testid="column-name"
id="columnn-ame"
name="column-name"
placeholder="col1"
type="text"
value={columnName}
onChange={handleValidation}
/>
{isShowError.columnName && errorMsg('Value is required.')}
</Field>
);
};
const getValueColSetTextField = () => {
return (
<Field>
<label className="tw-block tw-form-label" htmlFor="column-name-set">
Column Names
</label>
<input
className="tw-form-inputs tw-form-inputs-padding tw-mb-4"
data-testid="column-name-set"
id="column-name-set"
name="column-name-set"
placeholder="col1, col2, col3"
type="text"
value={columnNameSet}
onChange={handleValidation}
/>
<div className="tw-flex tw-gap-1">
<label>Match Column Input Order</label>
<ToggleSwitchV1
checked={columnNameSetOrdered}
handleCheck={() => setcolumnNameSetOrdered((pre) => !pre)}
testId="enable-ordered-column-names"
/>
</div>
{isShowError.columnNameSet && errorMsg('Value is required.')}
</Field>
);
};
const getMinMaxField = () => {
return (
<Fragment>
@ -266,6 +371,20 @@ const TableTestForm = ({
);
};
const getTableTestConfig = () => {
switch (tableTest) {
case TableTestType.TableRowCountToBeBetween:
case TableTestType.TableColumnCountToBeBetween:
return getMinMaxField();
case TableTestType.TableColumnNameToExist:
return getValueColTextField();
case TableTestType.TableColumnToMatchSet:
return getValueColSetTextField();
default:
return getValueField();
}
};
return (
<Fragment>
<p className="tw-font-medium tw-px-4">
@ -314,11 +433,7 @@ const TableTestForm = ({
/>
</Field>
<Field>
{tableTest === TableTestType.TableRowCountToBeBetween
? getMinMaxField()
: getValueField()}
</Field>
<Field>{getTableTestConfig()}</Field>
</div>
<Field className="tw-flex tw-justify-end">
<Button

16
openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx
View File

@ -498,16 +498,25 @@ export const filteredColumnTestOption = (dataType: string) => {
(test) => test !== ColumnTestType.ColumnValueLengthsToBeBetween
);
case 'varchar':
case 'varchar': {
const excluded = [
ColumnTestType.ColumnValuesToBeBetween,
ColumnTestType.ColumnValuesSumToBeBetween,
ColumnTestType.ColumnValueMinToBeBetween,
ColumnTestType.ColumnValueMaxToBeBetween,
];
return Object.values(ColumnTestType).filter(
(test) => test !== ColumnTestType.ColumnValuesToBeBetween
(test) => !excluded.includes(test)
);
}
case 'timestamp':
case 'date': {
const excluded = [
ColumnTestType.ColumnValuesToBeNotInSet,
ColumnTestType.ColumnValueLengthsToBeBetween,
ColumnTestType.ColumnValuesSumToBeBetween,
];
return Object.values(ColumnTestType).filter(
@ -519,6 +528,9 @@ export const filteredColumnTestOption = (dataType: string) => {
ColumnTestType.ColumnValuesToBeNotInSet,
ColumnTestType.ColumnValueLengthsToBeBetween,
ColumnTestType.ColumnValuesToBeBetween,
ColumnTestType.ColumnValuesSumToBeBetween,
ColumnTestType.ColumnValueMinToBeBetween,
ColumnTestType.ColumnValueMaxToBeBetween,
];
return Object.values(ColumnTestType).filter(