diff --git a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/tableAPI.ts b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/tableAPI.ts index b71f1b40685..0ab04df751f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/tableAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/tableAPI.ts @@ -13,6 +13,10 @@ import { AxiosResponse } from 'axios'; import { Table } from 'Models'; +import { ColumnTestType } from '../enums/columnTest.enum'; +import { CreateTableTest } from '../generated/api/tests/createTableTest'; +import { TableTestType } from '../generated/tests/tableTest'; +import { CreateColumnTest } from '../interface/dataQuality.interface'; import { getURLWithQueryFields } from '../utils/APIUtils'; import APIClient from './index'; @@ -111,3 +115,48 @@ export const removeFollower: Function = ( configOptions ); }; + +export const addTableTestCase = (tableId: string, data: CreateTableTest) => { + const configOptions = { + headers: { 'Content-type': 'application/json' }, + }; + + return APIClient.put(`/tables/${tableId}/tableTest`, data, configOptions); +}; + +export const deleteTableTestCase = ( + tableId: string, + tableTestType: TableTestType +): Promise => { + const configOptions = { + headers: { 'Content-type': 'application/json' }, + }; + + return APIClient.delete( + `/tables/${tableId}/tableTest/${tableTestType}`, + configOptions + ); +}; + +export const addColumnTestCase = (tableId: string, data: CreateColumnTest) => { + const configOptions = { + headers: { 'Content-type': 'application/json' }, + }; + + return APIClient.put(`/tables/${tableId}/columnTest`, data, configOptions); +}; + +export const deleteColumnTestCase = ( + tableId: string, + columnName: string, + columnTestType: ColumnTestType +): Promise => { + const configOptions = { + headers: { 'Content-type': 'application/json' }, + }; + + return APIClient.delete( + `/tables/${tableId}/columnTest/${columnName}/${columnTestType}`, + configOptions + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.tsx new file mode 100644 index 00000000000..d327022a9f8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.tsx @@ -0,0 +1,65 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { CreateTableTest } from '../../generated/api/tests/createTableTest'; +import { Table } from '../../generated/entity/data/table'; +import { TableTest } from '../../generated/tests/tableTest'; +import { + CreateColumnTest, + TableTestDataType, +} from '../../interface/dataQuality.interface'; +import ColumnTestForm from './Forms/ColumnTestForm'; +import TableTestForm from './Forms/TableTestForm'; + +type Props = { + data?: TableTestDataType; + testMode: 'table' | 'column'; + columnOptions: Table['columns']; + tableTestCase: TableTest[]; + handleAddTableTestCase: (data: CreateTableTest) => void; + handleAddColumnTestCase: (data: CreateColumnTest) => void; + onFormCancel: () => void; +}; + +const AddDataQualityTest = ({ + tableTestCase, + data, + testMode, + columnOptions = [], + handleAddTableTestCase, + handleAddColumnTestCase, + onFormCancel, +}: Props) => { + return ( +
+ {testMode === 'table' ? ( + + ) : ( + + )} +
+ ); +}; + +export default AddDataQualityTest; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx new file mode 100644 index 00000000000..b444928d9ac --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx @@ -0,0 +1,618 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import classNames from 'classnames'; +import { cloneDeep, isEmpty, isUndefined } from 'lodash'; +import { EditorContentRef } from 'Models'; +import React, { Fragment, useEffect, useRef, useState } from 'react'; +import { ColumnTestType } from '../../../enums/columnTest.enum'; +import { TestCaseExecutionFrequency } from '../../../generated/api/tests/createTableTest'; +import { Table } from '../../../generated/entity/data/table'; +import { + CreateColumnTest, + ModifiedTableColumn, +} from '../../../interface/dataQuality.interface'; +import { + errorMsg, + getCurrentUserId, + requiredField, +} from '../../../utils/CommonUtils'; +import SVGIcons from '../../../utils/SvgUtils'; +import { getDataTypeString } from '../../../utils/TableUtils'; +import { Button } from '../../buttons/Button/Button'; +import MarkdownWithPreview from '../../common/editor/MarkdownWithPreview'; + +type Props = { + data: CreateColumnTest; + column: ModifiedTableColumn[]; + handleAddColumnTestCase: (data: CreateColumnTest) => void; + onFormCancel: () => void; +}; + +export const Field = ({ + children, + className = '', +}: { + children: React.ReactNode; + className?: string; +}) => { + return
{children}
; +}; + +const ColumnTestForm = ({ + data, + column, + handleAddColumnTestCase, + onFormCancel, +}: Props) => { + const markdownRef = useRef(); + const [description] = useState(data?.description || ''); + const isAcceptedTypeIsString = useRef(true); + const [columnTest, setColumnTest] = useState( + data?.testCase?.columnTestType + ); + const [columnOptions, setColumnOptions] = useState([]); + const [testTypeOptions, setTestTypeOptions] = useState([]); + const [minValue, setMinValue] = useState( + data?.testCase?.config?.minValue + ); + const [maxValue, setMaxValue] = useState( + data?.testCase?.config?.maxValue + ); + + const [frequency, setFrequency] = useState( + data?.executionFrequency || TestCaseExecutionFrequency.Daily + ); + const [forbiddenValues, setForbiddenValues] = useState<(string | number)[]>( + data?.testCase?.config?.forbiddenValues || [''] + ); + const [isShowError, setIsShowError] = useState({ + columnName: false, + regex: false, + minOrMax: false, + missingCountValue: false, + values: false, + minMaxValue: false, + }); + + const [columnName, setColumnName] = useState(data?.columnName); + const [missingValueMatch, setMissingValueMatch] = useState( + data?.testCase?.config?.missingValueMatch || '' + ); + const [missingCountValue, setMissingCountValue] = useState< + number | undefined + >(data?.testCase?.config?.missingCountValue); + + const [regex, setRegex] = useState( + data?.testCase?.config?.regex || '' + ); + + const addValueFields = () => { + setForbiddenValues([...forbiddenValues, '']); + }; + + const removeValueFields = (i: number) => { + const newFormValues = [...forbiddenValues]; + newFormValues.splice(i, 1); + setForbiddenValues(newFormValues); + }; + + const handleValueFieldsChange = (i: number, value: string) => { + const newFormValues = [...forbiddenValues]; + newFormValues[i] = value; + setForbiddenValues(newFormValues); + setIsShowError({ ...isShowError, values: false }); + }; + + const handleTestTypeOptionChange = (name: string) => { + const selectedColumn = column.find((d) => d.name === name); + const existingTests = + selectedColumn?.columnTests?.map( + (d: CreateColumnTest) => d.testCase.columnTestType + ) || []; + if (existingTests.length) { + const newTest = Object.values(ColumnTestType).filter( + (d) => !existingTests.includes(d) + ); + setTestTypeOptions(newTest); + setColumnTest(newTest[0]); + } else { + const newTest = Object.values(ColumnTestType); + setTestTypeOptions(newTest); + setColumnTest(newTest[0]); + } + }; + + useEffect(() => { + if (isUndefined(data)) { + const allOption = column.filter((value) => { + return ( + value?.dataType !== 'STRUCT' && + value.columnTests?.length !== Object.values(ColumnTestType).length + ); + }); + setColumnOptions(allOption); + setColumnName(allOption[0]?.name || ''); + handleTestTypeOptionChange(allOption[0]?.name || ''); + } else { + setColumnOptions(column); + setTestTypeOptions(Object.values(ColumnTestType)); + setColumnName(data.columnName || ''); + } + }, [column]); + + const validateForm = () => { + const errMsg = cloneDeep(isShowError); + errMsg.columnName = isEmpty(columnName); + + switch (columnTest) { + case ColumnTestType.columnValueLengthsToBeBetween: + case ColumnTestType.columnValuesToBeBetween: + errMsg.minOrMax = isEmpty(minValue) && isEmpty(maxValue); + if (!isEmpty(minValue) && !isEmpty(maxValue)) { + errMsg.minMaxValue = (minValue as number) > (maxValue as number); + } + + break; + + case ColumnTestType.columnValuesMissingCountToBeEqual: + errMsg.missingCountValue = isEmpty(missingCountValue); + + break; + + case ColumnTestType.columnValuesToBeNotInSet: { + const actualValues = forbiddenValues.filter((v) => !isEmpty(v)); + errMsg.values = actualValues.length < 1; + + break; + } + + case ColumnTestType.columnValuesToMatchRegex: + errMsg.regex = isEmpty(regex); + + break; + } + + setIsShowError(errMsg); + + return !Object.values(errMsg).includes(true); + }; + + const getTestConfi = () => { + switch (columnTest) { + case ColumnTestType.columnValueLengthsToBeBetween: + case ColumnTestType.columnValuesToBeBetween: + return { + minValue: minValue, + maxValue: maxValue, + }; + + case ColumnTestType.columnValuesMissingCountToBeEqual: + return { + missingCountValue: missingCountValue, + missingValueMatch: missingValueMatch, + }; + + case ColumnTestType.columnValuesToBeNotInSet: + return { + forbiddenValues: forbiddenValues.filter((v) => !isEmpty(v)), + }; + + case ColumnTestType.columnValuesToMatchRegex: + return { + regex: regex, + }; + + case ColumnTestType.columnValuesToBeNotNull: + case ColumnTestType.columnValuesToBeUnique: + default: + return {}; + } + }; + + const handleSave = () => { + if (validateForm()) { + const columnTestObj: CreateColumnTest = { + columnName: columnName, + description: markdownRef.current?.getEditorContent() || undefined, + executionFrequency: frequency, + testCase: { + config: getTestConfi(), + columnTestType: columnTest, + }, + owner: { + type: 'user', + id: getCurrentUserId(), + }, + }; + handleAddColumnTestCase(columnTestObj); + } + }; + + const handleValidation = ( + event: React.ChangeEvent + ) => { + const value = event.target.value; + const eleName = event.target.name; + + const errorMsg = cloneDeep(isShowError); + + switch (eleName) { + case 'columTestType': { + const selectedColumn = column.find((d) => d.name === columnName); + const columnDataType = getDataTypeString( + selectedColumn?.dataType as string + ); + isAcceptedTypeIsString.current = + columnDataType === 'varchar' || columnDataType === 'boolean'; + setForbiddenValues(['']); + setColumnTest(value as ColumnTestType); + errorMsg.columnName = false; + errorMsg.regex = false; + errorMsg.minOrMax = false; + errorMsg.missingCountValue = false; + errorMsg.values = false; + errorMsg.minMaxValue = false; + + break; + } + case 'min': { + setMinValue(value as unknown as number); + errorMsg.minOrMax = false; + errorMsg.minMaxValue = false; + + break; + } + + case 'max': { + setMaxValue(value as unknown as number); + errorMsg.minOrMax = false; + errorMsg.minMaxValue = false; + + break; + } + + case 'frequency': + setFrequency(value as TestCaseExecutionFrequency); + + break; + + case 'columnName': { + const selectedColumn = column.find((d) => d.name === value); + const columnDataType = getDataTypeString( + selectedColumn?.dataType as string + ); + isAcceptedTypeIsString.current = + columnDataType === 'varchar' || columnDataType === 'boolean'; + setForbiddenValues(['']); + setColumnName(value); + handleTestTypeOptionChange(value); + errorMsg.columnName = false; + + break; + } + + case 'missingValueMatch': + setMissingValueMatch(value); + + break; + + case 'missingCountValue': + setMissingCountValue(value as unknown as number); + errorMsg.missingCountValue = false; + + break; + + case 'regex': + setRegex(value); + errorMsg.regex = false; + + break; + + default: + break; + } + + setIsShowError(errorMsg); + }; + + const getMinMaxField = () => { + return ( + +
+
+ + +
+
+ + +
+
+ {isShowError.minOrMax && errorMsg('Please enter atleast one value.')} + {isShowError.minMaxValue && + errorMsg('Min value should be lower than Max value.')} +
+ ); + }; + + const getMissingCountToBeEqualFields = () => { + return ( + +
+
+ + + {isShowError.missingCountValue && + errorMsg('Count value is required.')} +
+
+ + +
+
+
+ ); + }; + + const getColumnValuesToMatchRegexFields = () => { + return ( + + + + {isShowError.regex && errorMsg('Regex is required.')} + + ); + }; + + const getColumnValuesToBeNotInSetField = () => { + return ( +
+
+

{requiredField('Values')}

+