[Fixes #5678] Implement frontend logic for data quality input field (#5682)

* Added input fields for new table tests

* Added forms for data quality tests

* Fix error in test validation for new tests

* fixed failing test

* Added toggle form to force test to run on column order or no

Co-authored-by: Shailesh Parmar <shailesh.parmar.webdev@gmail.com>

openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx

@@ -78,6 +78,11 @@ const ColumnTestForm = ({
   const [forbiddenValues, setForbiddenValues] = useState<(string | number)[]>(
     data?.testCase?.config?.forbiddenValues || ['']
   );
+
+  const [allowedValues, setAllowedValues] = useState<(string | number)[]>(
+    data?.testCase?.config?.allowedValues || ['']
+  );
+
   const [isShowError, setIsShowError] = useState({
     testName: false,
     columnName: false,
@@ -85,6 +90,7 @@ const ColumnTestForm = ({
     minOrMax: false,
     missingCountValue: false,
     values: false,
+    inSetValues: false,
     minMaxValue: false,
     allTestAdded: false,
     notSupported: false,
@@ -119,6 +125,23 @@ const ColumnTestForm = ({
     setIsShowError({ ...isShowError, values: false });
   };
 
+  const addInSetValueFields = () => {
+    setAllowedValues([...allowedValues, '']);
+  };
+
+  const removeInSetValueFields = (i: number) => {
+    const newFormValues = [...allowedValues];
+    newFormValues.splice(i, 1);
+    setAllowedValues(newFormValues);
+  };
+
+  const handleInSetValueFieldsChange = (i: number, value: string) => {
+    const newFormValues = [...allowedValues];
+    newFormValues[i] = value;
+    setAllowedValues(newFormValues);
+    setIsShowError({ ...isShowError, values: false });
+  };
+
   const addMatchFields = () => {
     setMissingValueMatch([...missingValueMatch, '']);
   };
@@ -194,6 +217,9 @@ const ColumnTestForm = ({
     switch (columnTest) {
       case ColumnTestType.ColumnValueLengthsToBeBetween:
       case ColumnTestType.ColumnValuesToBeBetween:
+      case ColumnTestType.ColumnValueMaxToBeBetween:
+      case ColumnTestType.ColumnValueMinToBeBetween:
+      case ColumnTestType.ColumnValuesSumToBeBetween:
         errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue);
         if (!isUndefined(minValue) && !isUndefined(maxValue)) {
           errMsg.minMaxValue = (+minValue as number) > (+maxValue as number);
@@ -213,6 +239,13 @@ const ColumnTestForm = ({
         break;
       }
 
+      case ColumnTestType.ColumnValuesToBeInSet: {
+        const actualValues = allowedValues.filter((v) => !isEmpty(v));
+        errMsg.inSetValues = actualValues.length < 1;
+
+        break;
+      }
+
       case ColumnTestType.ColumnValuesToMatchRegex:
         errMsg.regex = isEmpty(regex);
 
@@ -237,6 +270,24 @@ const ColumnTestForm = ({
           maxValue: !isEmpty(maxValue) ? maxValue : undefined,
         };
 
+      case ColumnTestType.ColumnValueMaxToBeBetween:
+        return {
+          minValueForMaxInCol: !isEmpty(minValue) ? minValue : undefined,
+          maxValueForMaxInCol: !isEmpty(maxValue) ? maxValue : undefined,
+        };
+
+      case ColumnTestType.ColumnValueMinToBeBetween:
+        return {
+          minValueForMinInCol: !isEmpty(minValue) ? minValue : undefined,
+          maxValueForMinInCol: !isEmpty(maxValue) ? maxValue : undefined,
+        };
+
+      case ColumnTestType.ColumnValuesSumToBeBetween:
+        return {
+          minValueForColSum: !isEmpty(minValue) ? minValue : undefined,
+          maxValueForColSum: !isEmpty(maxValue) ? maxValue : undefined,
+        };
+
       case ColumnTestType.ColumnValuesMissingCountToBeEqual: {
         const filterMatchValue = missingValueMatch.filter(
           (value) => !isEmpty(value)
@@ -254,6 +305,11 @@ const ColumnTestForm = ({
           forbiddenValues: forbiddenValues.filter((v) => !isEmpty(v)),
         };
 
+      case ColumnTestType.ColumnValuesToBeInSet:
+        return {
+          allowedValues: allowedValues.filter((v) => !isEmpty(v)),
+        };
+
       case ColumnTestType.ColumnValuesToMatchRegex:
         return {
           regex: regex,
@@ -307,12 +363,14 @@ const ColumnTestForm = ({
         isAcceptedTypeIsString.current =
           columnDataType === 'varchar' || columnDataType === 'boolean';
         setForbiddenValues(['']);
+        setAllowedValues(['']);
         setColumnTest(value as ColumnTestType);
         errorMsg.columnName = false;
         errorMsg.regex = false;
         errorMsg.minOrMax = false;
         errorMsg.missingCountValue = false;
         errorMsg.values = false;
+        errorMsg.inSetValues = false;
         errorMsg.minMaxValue = false;
         errorMsg.testName = false;
 
@@ -342,6 +400,7 @@ const ColumnTestForm = ({
         isAcceptedTypeIsString.current =
           columnDataType === 'varchar' || columnDataType === 'boolean';
         setForbiddenValues(['']);
+        setAllowedValues(['']);
         setColumnName(value);
         handleTestTypeOptionChange(value);
         errorMsg.allTestAdded =
@@ -555,10 +614,66 @@ const ColumnTestForm = ({
     );
   };
 
+  const getColumnValuesToBeInSetField = () => {
+    return (
+      <div data-testid="in-set-field">
+        <div className="tw-flex tw-items-center tw-mt-6">
+          <p className="w-form-label tw-mr-3">{requiredField('Values')}</p>
+          <Button
+            className="tw-h-5 tw-px-2"
+            size="x-small"
+            theme="primary"
+            variant="contained"
+            onClick={addInSetValueFields}>
+            <FontAwesomeIcon icon="plus" />
+          </Button>
+        </div>
+
+        {allowedValues.map((value, i) => (
+          <div className="tw-flex tw-items-center" key={i}>
+            <div className="tw-w-11/12">
+              <Field>
+                <input
+                  className="tw-form-inputs tw-form-inputs-padding"
+                  id={`option-key-${i}`}
+                  name="key"
+                  placeholder="Values to be in the set"
+                  type={isAcceptedTypeIsString.current ? 'text' : 'number'}
+                  value={value}
+                  onChange={(e) =>
+                    handleInSetValueFieldsChange(i, e.target.value)
+                  }
+                />
+              </Field>
+            </div>
+            <button
+              className="focus:tw-outline-none tw-mt-3 tw-w-1/12"
+              onClick={(e) => {
+                removeInSetValueFields(i);
+                e.preventDefault();
+              }}>
+              <SVGIcons
+                alt="delete"
+                icon="icon-delete"
+                title="Delete"
+                width="16px"
+              />
+            </button>
+          </div>
+        ))}
+
+        {isShowError.values && errorMsg('Value is required.')}
+      </div>
+    );
+  };
+
   const getColumnTestConfig = () => {
     switch (columnTest) {
       case ColumnTestType.ColumnValueLengthsToBeBetween:
       case ColumnTestType.ColumnValuesToBeBetween:
+      case ColumnTestType.ColumnValueMaxToBeBetween:
+      case ColumnTestType.ColumnValueMinToBeBetween:
+      case ColumnTestType.ColumnValuesSumToBeBetween:
         return getMinMaxField();
 
       case ColumnTestType.ColumnValuesMissingCountToBeEqual:
@@ -567,6 +682,9 @@ const ColumnTestForm = ({
       case ColumnTestType.ColumnValuesToBeNotInSet:
         return getColumnValuesToBeNotInSetField();
 
+      case ColumnTestType.ColumnValuesToBeInSet:
+        return getColumnValuesToBeInSetField();
+
       case ColumnTestType.ColumnValuesToMatchRegex:
         return getColumnValuesToMatchRegexFields();
 

openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.test.tsx

@@ -25,7 +25,8 @@ describe('Test TableTestForm component', () => {
     const tableTestType = await findByTestId(container, 'tableTestType');
     const cancelButton = await findByTestId(container, 'cancel-test');
     const saveButton = await findByTestId(container, 'save-test');
-    const value = await findByTestId(container, 'value');
+    const min = await findByTestId(container, 'min');
+    const max = await findByTestId(container, 'max');
     const description = await findByText(
       container,
       /MarkdownWithPreview component/i
@@ -33,7 +34,8 @@ describe('Test TableTestForm component', () => {
 
     expect(tableTestType).toBeInTheDocument();
     expect(description).toBeInTheDocument();
-    expect(value).toBeInTheDocument();
+    expect(min).toBeInTheDocument();
+    expect(max).toBeInTheDocument();
     expect(cancelButton).toBeInTheDocument();
     expect(saveButton).toBeInTheDocument();
   });

openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.tsx

@@ -27,6 +27,7 @@ import {
 } from '../../../utils/CommonUtils';
 import { Button } from '../../buttons/Button/Button';
 import RichTextEditor from '../../common/rich-text-editor/RichTextEditor';
+import ToggleSwitchV1 from '../../common/toggle-switch/ToggleSwitchV1';
 
 type Props = {
   data?: TableTest;
@@ -67,12 +68,26 @@ const TableTestForm = ({
   const [value, setValue] = useState<number | undefined>(
     data?.testCase.config?.value || data?.testCase.config?.columnCount
   );
+  const [columnName, setcolumnName] = useState<string | undefined>(
+    data?.testCase?.config?.columnName
+  );
+  const [columnNameSet, setcolumnNameSet] = useState<string | undefined>(
+    data?.testCase?.config?.columnNames
+  );
+  const [columnNameSetOrdered, setcolumnNameSetOrdered] = useState<boolean>(
+    data?.testCase?.config?.ordered !== undefined
+      ? data?.testCase?.config?.ordered
+      : false
+  );
   const [isShowError, setIsShowError] = useState({
     minOrMax: false,
     values: false,
     minMaxValue: false,
     allTestAdded: false,
     tableTest: false,
+    columnName: false,
+    columnNameSet: false,
+    columnNameSetOrdered: false,
   });
   const [testTypeOptions, setTestTypeOptions] = useState<string[]>();
 
@@ -100,19 +115,29 @@ const TableTestForm = ({
 
   const validateForm = () => {
     const errMsg = cloneDeep(isShowError);
-
-    const isTableRowCountToBeBetweenTest =
-      tableTest === TableTestType.TableRowCountToBeBetween;
-
     errMsg.tableTest = isEmpty(tableTest);
 
-    if (isTableRowCountToBeBetweenTest) {
+    switch (tableTest) {
-      errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue);
+      case TableTestType.TableRowCountToBeBetween:
-      if (!isUndefined(minValue) && !isUndefined(maxValue)) {
+      case TableTestType.TableColumnCountToBeBetween:
-        errMsg.minMaxValue = (+minValue as number) > (+maxValue as number);
+        errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue);
-      }
+        if (!isUndefined(minValue) && !isUndefined(maxValue)) {
-    } else {
+          errMsg.minMaxValue = (+minValue as number) > (+maxValue as number);
-      errMsg.values = isEmpty(value);
+        }
+
+        break;
+      case TableTestType.TableColumnNameToExist:
+        errMsg.columnName = isEmpty(columnName);
+
+        break;
+
+      case TableTestType.TableColumnToMatchSet:
+        errMsg.columnNameSet = isEmpty(columnNameSet);
+
+        break;
+
+      default:
+        errMsg.values = isEmpty(value);
     }
 
     setIsShowError(errMsg);
@@ -128,6 +153,23 @@ const TableTestForm = ({
           minValue: isEmpty(minValue) ? undefined : minValue,
         };
 
+      case TableTestType.TableColumnCountToBeBetween:
+        return {
+          minColValue: isEmpty(minValue) ? undefined : minValue,
+          maxColValue: isEmpty(maxValue) ? undefined : maxValue,
+        };
+
+      case TableTestType.TableColumnNameToExist:
+        return {
+          columnName: isEmpty(columnName) ? undefined : columnName,
+        };
+
+      case TableTestType.TableColumnToMatchSet:
+        return {
+          columnNames: isEmpty(columnNameSet) ? undefined : columnNameSet,
+          ordered: columnNameSetOrdered,
+        };
+
       case TableTestType.TableColumnCountToEqual:
         return {
           columnCount: isEmpty(value) ? undefined : value,
@@ -196,6 +238,19 @@ const TableTestForm = ({
 
         break;
 
+      case 'column-name':
+        setcolumnName(value as unknown as string);
+        errorMsg.columnName = false;
+
+        break;
+
+      case 'column-name-set':
+        setcolumnNameSet(value as unknown as string);
+        errorMsg.columnNameSet = false;
+        errorMsg.columnNameSetOrdered = false;
+
+        break;
+
       default:
         break;
     }
@@ -224,6 +279,56 @@ const TableTestForm = ({
     );
   };
 
+  const getValueColTextField = () => {
+    return (
+      <Field>
+        <label className="tw-block tw-form-label" htmlFor="column-name">
+          Column Name
+        </label>
+        <input
+          className="tw-form-inputs tw-form-inputs-padding"
+          data-testid="column-name"
+          id="columnn-ame"
+          name="column-name"
+          placeholder="col1"
+          type="text"
+          value={columnName}
+          onChange={handleValidation}
+        />
+        {isShowError.columnName && errorMsg('Value is required.')}
+      </Field>
+    );
+  };
+
+  const getValueColSetTextField = () => {
+    return (
+      <Field>
+        <label className="tw-block tw-form-label" htmlFor="column-name-set">
+          Column Names
+        </label>
+        <input
+          className="tw-form-inputs tw-form-inputs-padding tw-mb-4"
+          data-testid="column-name-set"
+          id="column-name-set"
+          name="column-name-set"
+          placeholder="col1, col2, col3"
+          type="text"
+          value={columnNameSet}
+          onChange={handleValidation}
+        />
+        <div className="tw-flex tw-gap-1">
+          <label>Match Column Input Order</label>
+          <ToggleSwitchV1
+            checked={columnNameSetOrdered}
+            handleCheck={() => setcolumnNameSetOrdered((pre) => !pre)}
+            testId="enable-ordered-column-names"
+          />
+        </div>
+        {isShowError.columnNameSet && errorMsg('Value is required.')}
+      </Field>
+    );
+  };
+
   const getMinMaxField = () => {
     return (
       <Fragment>
@@ -266,6 +371,20 @@ const TableTestForm = ({
     );
   };
 
+  const getTableTestConfig = () => {
+    switch (tableTest) {
+      case TableTestType.TableRowCountToBeBetween:
+      case TableTestType.TableColumnCountToBeBetween:
+        return getMinMaxField();
+      case TableTestType.TableColumnNameToExist:
+        return getValueColTextField();
+      case TableTestType.TableColumnToMatchSet:
+        return getValueColSetTextField();
+      default:
+        return getValueField();
+    }
+  };
+
   return (
     <Fragment>
       <p className="tw-font-medium tw-px-4">
@@ -314,11 +433,7 @@ const TableTestForm = ({
             />
           </Field>
 
-          <Field>
+          <Field>{getTableTestConfig()}</Field>
-            {tableTest === TableTestType.TableRowCountToBeBetween
-              ? getMinMaxField()
-              : getValueField()}
-          </Field>
         </div>
         <Field className="tw-flex tw-justify-end">
           <Button

openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx

@@ -498,16 +498,25 @@ export const filteredColumnTestOption = (dataType: string) => {
         (test) => test !== ColumnTestType.ColumnValueLengthsToBeBetween
       );
 
-    case 'varchar':
+    case 'varchar': {
+      const excluded = [
+        ColumnTestType.ColumnValuesToBeBetween,
+        ColumnTestType.ColumnValuesSumToBeBetween,
+        ColumnTestType.ColumnValueMinToBeBetween,
+        ColumnTestType.ColumnValueMaxToBeBetween,
+      ];
+
       return Object.values(ColumnTestType).filter(
-        (test) => test !== ColumnTestType.ColumnValuesToBeBetween
+        (test) => !excluded.includes(test)
       );
+    }
 
     case 'timestamp':
     case 'date': {
       const excluded = [
         ColumnTestType.ColumnValuesToBeNotInSet,
         ColumnTestType.ColumnValueLengthsToBeBetween,
+        ColumnTestType.ColumnValuesSumToBeBetween,
       ];
 
       return Object.values(ColumnTestType).filter(
@@ -519,6 +528,9 @@ export const filteredColumnTestOption = (dataType: string) => {
         ColumnTestType.ColumnValuesToBeNotInSet,
         ColumnTestType.ColumnValueLengthsToBeBetween,
         ColumnTestType.ColumnValuesToBeBetween,
+        ColumnTestType.ColumnValuesSumToBeBetween,
+        ColumnTestType.ColumnValueMinToBeBetween,
+        ColumnTestType.ColumnValueMaxToBeBetween,
       ];
 
       return Object.values(ColumnTestType).filter(