From 7a131b115fd56fa311aa560c6a9a08ff9bdc1c3e Mon Sep 17 00:00:00 2001 From: Teddy Date: Tue, 28 Jun 2022 14:58:29 +0200 Subject: [PATCH] [Fixes #5678] Implement frontend logic for data quality input field (#5682) * Added input fields for new table tests * Added forms for data quality tests * Fix error in test validation for new tests * fixed failing test * Added toggle form to force test to run on column order or no Co-authored-by: Shailesh Parmar --- .../Forms/ColumnTestForm.tsx | 118 ++++++++++++++ .../Forms/TableTestForm.test.tsx | 6 +- .../Forms/TableTestForm.tsx | 147 ++++++++++++++++-- .../resources/ui/src/utils/EntityUtils.tsx | 16 +- 4 files changed, 267 insertions(+), 20 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx index e0dff594631..892fa89405f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx @@ -78,6 +78,11 @@ const ColumnTestForm = ({ const [forbiddenValues, setForbiddenValues] = useState<(string | number)[]>( data?.testCase?.config?.forbiddenValues || [''] ); + + const [allowedValues, setAllowedValues] = useState<(string | number)[]>( + data?.testCase?.config?.allowedValues || [''] + ); + const [isShowError, setIsShowError] = useState({ testName: false, columnName: false, @@ -85,6 +90,7 @@ const ColumnTestForm = ({ minOrMax: false, missingCountValue: false, values: false, + inSetValues: false, minMaxValue: false, allTestAdded: false, notSupported: false, @@ -119,6 +125,23 @@ const ColumnTestForm = ({ setIsShowError({ ...isShowError, values: false }); }; + const addInSetValueFields = () => { + setAllowedValues([...allowedValues, '']); + }; + + const removeInSetValueFields = (i: number) => { + const newFormValues = [...allowedValues]; + newFormValues.splice(i, 1); + setAllowedValues(newFormValues); + }; + + const handleInSetValueFieldsChange = (i: number, value: string) => { + const newFormValues = [...allowedValues]; + newFormValues[i] = value; + setAllowedValues(newFormValues); + setIsShowError({ ...isShowError, values: false }); + }; + const addMatchFields = () => { setMissingValueMatch([...missingValueMatch, '']); }; @@ -194,6 +217,9 @@ const ColumnTestForm = ({ switch (columnTest) { case ColumnTestType.ColumnValueLengthsToBeBetween: case ColumnTestType.ColumnValuesToBeBetween: + case ColumnTestType.ColumnValueMaxToBeBetween: + case ColumnTestType.ColumnValueMinToBeBetween: + case ColumnTestType.ColumnValuesSumToBeBetween: errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue); if (!isUndefined(minValue) && !isUndefined(maxValue)) { errMsg.minMaxValue = (+minValue as number) > (+maxValue as number); @@ -213,6 +239,13 @@ const ColumnTestForm = ({ break; } + case ColumnTestType.ColumnValuesToBeInSet: { + const actualValues = allowedValues.filter((v) => !isEmpty(v)); + errMsg.inSetValues = actualValues.length < 1; + + break; + } + case ColumnTestType.ColumnValuesToMatchRegex: errMsg.regex = isEmpty(regex); @@ -237,6 +270,24 @@ const ColumnTestForm = ({ maxValue: !isEmpty(maxValue) ? maxValue : undefined, }; + case ColumnTestType.ColumnValueMaxToBeBetween: + return { + minValueForMaxInCol: !isEmpty(minValue) ? minValue : undefined, + maxValueForMaxInCol: !isEmpty(maxValue) ? maxValue : undefined, + }; + + case ColumnTestType.ColumnValueMinToBeBetween: + return { + minValueForMinInCol: !isEmpty(minValue) ? minValue : undefined, + maxValueForMinInCol: !isEmpty(maxValue) ? maxValue : undefined, + }; + + case ColumnTestType.ColumnValuesSumToBeBetween: + return { + minValueForColSum: !isEmpty(minValue) ? minValue : undefined, + maxValueForColSum: !isEmpty(maxValue) ? maxValue : undefined, + }; + case ColumnTestType.ColumnValuesMissingCountToBeEqual: { const filterMatchValue = missingValueMatch.filter( (value) => !isEmpty(value) @@ -254,6 +305,11 @@ const ColumnTestForm = ({ forbiddenValues: forbiddenValues.filter((v) => !isEmpty(v)), }; + case ColumnTestType.ColumnValuesToBeInSet: + return { + allowedValues: allowedValues.filter((v) => !isEmpty(v)), + }; + case ColumnTestType.ColumnValuesToMatchRegex: return { regex: regex, @@ -307,12 +363,14 @@ const ColumnTestForm = ({ isAcceptedTypeIsString.current = columnDataType === 'varchar' || columnDataType === 'boolean'; setForbiddenValues(['']); + setAllowedValues(['']); setColumnTest(value as ColumnTestType); errorMsg.columnName = false; errorMsg.regex = false; errorMsg.minOrMax = false; errorMsg.missingCountValue = false; errorMsg.values = false; + errorMsg.inSetValues = false; errorMsg.minMaxValue = false; errorMsg.testName = false; @@ -342,6 +400,7 @@ const ColumnTestForm = ({ isAcceptedTypeIsString.current = columnDataType === 'varchar' || columnDataType === 'boolean'; setForbiddenValues(['']); + setAllowedValues(['']); setColumnName(value); handleTestTypeOptionChange(value); errorMsg.allTestAdded = @@ -555,10 +614,66 @@ const ColumnTestForm = ({ ); }; + const getColumnValuesToBeInSetField = () => { + return ( +
+
+

{requiredField('Values')}

+ +
+ + {allowedValues.map((value, i) => ( +
+
+ + + handleInSetValueFieldsChange(i, e.target.value) + } + /> + +
+ +
+ ))} + + {isShowError.values && errorMsg('Value is required.')} +
+ ); + }; + const getColumnTestConfig = () => { switch (columnTest) { case ColumnTestType.ColumnValueLengthsToBeBetween: case ColumnTestType.ColumnValuesToBeBetween: + case ColumnTestType.ColumnValueMaxToBeBetween: + case ColumnTestType.ColumnValueMinToBeBetween: + case ColumnTestType.ColumnValuesSumToBeBetween: return getMinMaxField(); case ColumnTestType.ColumnValuesMissingCountToBeEqual: @@ -567,6 +682,9 @@ const ColumnTestForm = ({ case ColumnTestType.ColumnValuesToBeNotInSet: return getColumnValuesToBeNotInSetField(); + case ColumnTestType.ColumnValuesToBeInSet: + return getColumnValuesToBeInSetField(); + case ColumnTestType.ColumnValuesToMatchRegex: return getColumnValuesToMatchRegexFields(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.test.tsx index c8c24966dae..0194be09caa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.test.tsx @@ -25,7 +25,8 @@ describe('Test TableTestForm component', () => { const tableTestType = await findByTestId(container, 'tableTestType'); const cancelButton = await findByTestId(container, 'cancel-test'); const saveButton = await findByTestId(container, 'save-test'); - const value = await findByTestId(container, 'value'); + const min = await findByTestId(container, 'min'); + const max = await findByTestId(container, 'max'); const description = await findByText( container, /MarkdownWithPreview component/i @@ -33,7 +34,8 @@ describe('Test TableTestForm component', () => { expect(tableTestType).toBeInTheDocument(); expect(description).toBeInTheDocument(); - expect(value).toBeInTheDocument(); + expect(min).toBeInTheDocument(); + expect(max).toBeInTheDocument(); expect(cancelButton).toBeInTheDocument(); expect(saveButton).toBeInTheDocument(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.tsx index 624a344e4e8..5e074f83309 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.tsx @@ -27,6 +27,7 @@ import { } from '../../../utils/CommonUtils'; import { Button } from '../../buttons/Button/Button'; import RichTextEditor from '../../common/rich-text-editor/RichTextEditor'; +import ToggleSwitchV1 from '../../common/toggle-switch/ToggleSwitchV1'; type Props = { data?: TableTest; @@ -67,12 +68,26 @@ const TableTestForm = ({ const [value, setValue] = useState( data?.testCase.config?.value || data?.testCase.config?.columnCount ); + const [columnName, setcolumnName] = useState( + data?.testCase?.config?.columnName + ); + const [columnNameSet, setcolumnNameSet] = useState( + data?.testCase?.config?.columnNames + ); + const [columnNameSetOrdered, setcolumnNameSetOrdered] = useState( + data?.testCase?.config?.ordered !== undefined + ? data?.testCase?.config?.ordered + : false + ); const [isShowError, setIsShowError] = useState({ minOrMax: false, values: false, minMaxValue: false, allTestAdded: false, tableTest: false, + columnName: false, + columnNameSet: false, + columnNameSetOrdered: false, }); const [testTypeOptions, setTestTypeOptions] = useState(); @@ -100,19 +115,29 @@ const TableTestForm = ({ const validateForm = () => { const errMsg = cloneDeep(isShowError); - - const isTableRowCountToBeBetweenTest = - tableTest === TableTestType.TableRowCountToBeBetween; - errMsg.tableTest = isEmpty(tableTest); - if (isTableRowCountToBeBetweenTest) { - errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue); - if (!isUndefined(minValue) && !isUndefined(maxValue)) { - errMsg.minMaxValue = (+minValue as number) > (+maxValue as number); - } - } else { - errMsg.values = isEmpty(value); + switch (tableTest) { + case TableTestType.TableRowCountToBeBetween: + case TableTestType.TableColumnCountToBeBetween: + errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue); + if (!isUndefined(minValue) && !isUndefined(maxValue)) { + errMsg.minMaxValue = (+minValue as number) > (+maxValue as number); + } + + break; + case TableTestType.TableColumnNameToExist: + errMsg.columnName = isEmpty(columnName); + + break; + + case TableTestType.TableColumnToMatchSet: + errMsg.columnNameSet = isEmpty(columnNameSet); + + break; + + default: + errMsg.values = isEmpty(value); } setIsShowError(errMsg); @@ -128,6 +153,23 @@ const TableTestForm = ({ minValue: isEmpty(minValue) ? undefined : minValue, }; + case TableTestType.TableColumnCountToBeBetween: + return { + minColValue: isEmpty(minValue) ? undefined : minValue, + maxColValue: isEmpty(maxValue) ? undefined : maxValue, + }; + + case TableTestType.TableColumnNameToExist: + return { + columnName: isEmpty(columnName) ? undefined : columnName, + }; + + case TableTestType.TableColumnToMatchSet: + return { + columnNames: isEmpty(columnNameSet) ? undefined : columnNameSet, + ordered: columnNameSetOrdered, + }; + case TableTestType.TableColumnCountToEqual: return { columnCount: isEmpty(value) ? undefined : value, @@ -196,6 +238,19 @@ const TableTestForm = ({ break; + case 'column-name': + setcolumnName(value as unknown as string); + errorMsg.columnName = false; + + break; + + case 'column-name-set': + setcolumnNameSet(value as unknown as string); + errorMsg.columnNameSet = false; + errorMsg.columnNameSetOrdered = false; + + break; + default: break; } @@ -224,6 +279,56 @@ const TableTestForm = ({ ); }; + const getValueColTextField = () => { + return ( + + + + {isShowError.columnName && errorMsg('Value is required.')} + + ); + }; + + const getValueColSetTextField = () => { + return ( + + + +
+ + setcolumnNameSetOrdered((pre) => !pre)} + testId="enable-ordered-column-names" + /> +
+ {isShowError.columnNameSet && errorMsg('Value is required.')} + + ); + }; + const getMinMaxField = () => { return ( @@ -266,6 +371,20 @@ const TableTestForm = ({ ); }; + const getTableTestConfig = () => { + switch (tableTest) { + case TableTestType.TableRowCountToBeBetween: + case TableTestType.TableColumnCountToBeBetween: + return getMinMaxField(); + case TableTestType.TableColumnNameToExist: + return getValueColTextField(); + case TableTestType.TableColumnToMatchSet: + return getValueColSetTextField(); + default: + return getValueField(); + } + }; + return (

@@ -314,11 +433,7 @@ const TableTestForm = ({ /> - - {tableTest === TableTestType.TableRowCountToBeBetween - ? getMinMaxField() - : getValueField()} - + {getTableTestConfig()}