diff --git a/ingestion/src/metadata/test_suite/api/workflow.py b/ingestion/src/metadata/test_suite/api/workflow.py index 91817501fdd..579c7284fe1 100644 --- a/ingestion/src/metadata/test_suite/api/workflow.py +++ b/ingestion/src/metadata/test_suite/api/workflow.py @@ -333,7 +333,7 @@ class TestSuiteWorkflow: or self.get_or_create_test_suite_entity_for_cli_workflow() ) test_cases = self.get_test_cases_from_test_suite(test_suites) - if self.processor_config: + if self.processor_config.testSuites: cli_config_test_cases_def = self.get_test_case_from_cli_config() runtime_created_test_cases = self.compare_and_create_test_cases( cli_config_test_cases_def, test_cases diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py index 35bfcfdb037..e6334dd94d2 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py @@ -91,7 +91,7 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource: Union[DatabaseService, MessagingService, PipelineService, DashboardService] ] = None - if service_type == "TestSuite": + if service_type == "testSuite": return WorkflowSource( type=service_type, serviceName=ingestion_pipeline.service.name, diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/registry.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/registry.py index 0d0cca0d840..bd87d846b4e 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/registry.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/registry.py @@ -16,6 +16,9 @@ Add a function for each type from PipelineType from openmetadata_managed_apis.workflows.ingestion.lineage import build_lineage_dag from openmetadata_managed_apis.workflows.ingestion.metadata import build_metadata_dag from openmetadata_managed_apis.workflows.ingestion.profiler import build_profiler_dag +from openmetadata_managed_apis.workflows.ingestion.test_suite import ( + build_test_suite_dag, +) from openmetadata_managed_apis.workflows.ingestion.usage import build_usage_dag from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( @@ -29,3 +32,4 @@ build_registry.add(PipelineType.metadata.value)(build_metadata_dag) build_registry.add(PipelineType.usage.value)(build_usage_dag) build_registry.add(PipelineType.lineage.value)(build_lineage_dag) build_registry.add(PipelineType.profiler.value)(build_profiler_dag) +build_registry.add(PipelineType.TestSuite.value)(build_test_suite_dag) diff --git a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py index 15cffc33704..e3d17470431 100644 --- a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py +++ b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py @@ -306,7 +306,7 @@ class OMetaServiceTest(TestCase): ), service=EntityReference( id=self.test_suite.id, - type="TestSuite", + type="testSuite", name=self.test_suite.name.__root__, ), ) diff --git a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/testAPI.ts b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/testAPI.ts index 439fc6aac3c..77ef8ec1413 100644 --- a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/testAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/testAPI.ts @@ -11,35 +11,49 @@ * limitations under the License. */ +import { CreateTestCase } from '../generated/api/tests/createTestCase'; +import { CreateTestSuite } from '../generated/api/tests/createTestSuite'; import { TestCase, TestCaseResult } from '../generated/tests/testCase'; +import { EntityType, TestDefinition } from '../generated/tests/testDefinition'; +import { TestSuite } from '../generated/tests/testSuite'; import { Include } from '../generated/type/include'; import { Paging } from '../generated/type/paging'; import APIClient from './index'; -export type ListTestCaseParams = { +export type ListParams = { fields?: string; limit?: number; before?: string; after?: string; - entityLink?: string; - testSuiteId?: string; - includeAllTests?: boolean; include?: Include; }; -export type ListTestCaseResultsParams = { - startTs?: number; - endTs?: number; - before?: string; - after?: string; - limit?: number; +export type ListTestCaseParams = ListParams & { + entityLink?: string; + testSuiteId?: string; + includeAllTests?: boolean; }; -const baseUrl = '/testCase'; +export type ListTestDefinitionsParams = ListParams & { + entityType?: EntityType; +}; +export type ListTestCaseResultsParams = Omit< + ListParams, + 'fields' | 'include' +> & { + startTs?: number; + endTs?: number; +}; + +const testCaseUrl = '/testCase'; +const testSuiteUrl = '/testSuite'; +const testDefinitionUrl = '/testDefinition'; + +// testCase section export const getListTestCase = async (params?: ListTestCaseParams) => { const response = await APIClient.get<{ data: TestCase[]; paging: Paging }>( - baseUrl, + testCaseUrl, { params, } @@ -52,7 +66,7 @@ export const getListTestCaseResults = async ( fqn: string, params?: ListTestCaseResultsParams ) => { - const url = `${baseUrl}/${fqn}/testCaseResult`; + const url = `${testCaseUrl}/${fqn}/testCaseResult`; const response = await APIClient.get<{ data: TestCaseResult[]; paging: Paging; @@ -62,3 +76,41 @@ export const getListTestCaseResults = async ( return response.data; }; + +export const createTestCase = async (data: CreateTestCase) => { + const response = await APIClient.post(testCaseUrl, data); + + return response.data; +}; + +// testSuite section +export const getListTestSuites = async (params?: ListParams) => { + const response = await APIClient.get<{ + data: TestSuite[]; + paging: Paging; + }>(testSuiteUrl, { + params, + }); + + return response.data; +}; + +export const createTestSuites = async (data: CreateTestSuite) => { + const response = await APIClient.post(testSuiteUrl, data); + + return response.data; +}; + +// testDefinition Section +export const getListTestDefinitions = async ( + params?: ListTestDefinitionsParams +) => { + const response = await APIClient.get<{ + data: TestDefinition[]; + paging: Paging; + }>(testDefinitionUrl, { + params, + }); + + return response.data; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.interface.ts new file mode 100644 index 00000000000..ae271bb6e6a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.interface.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ReactNode } from 'react'; +import { Table } from '../../generated/entity/data/table'; +import { TestCase } from '../../generated/tests/testCase'; +import { TestDefinition } from '../../generated/tests/testDefinition'; +import { TestSuite } from '../../generated/tests/testSuite'; + +export interface AddDataQualityTestProps { + table: Table; +} + +export interface SelectTestSuiteProps { + initialValue?: SelectTestSuiteType; + onSubmit: (data: SelectTestSuiteType) => void; +} + +export interface TestCaseFormProps { + initialValue?: TestCase; + onSubmit: (data: TestCase) => void; + onCancel: (data: TestCase) => void; +} + +export interface TestSuiteIngestionProps { + testSuite: TestSuite; + onCancel: () => void; +} + +export interface TestSuiteSchedulerProps { + onSubmit: (repeatFrequency: string) => void; + onCancel: () => void; +} + +export interface RightPanelProps { + data: { + title: string; + body: string | ReactNode; + }; +} + +export type SelectTestSuiteType = { + name?: string; + description?: string; + data?: TestSuite; + isNewTestSuite: boolean; +}; + +export interface ParameterFormProps { + definition: TestDefinition; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTestV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTestV1.tsx new file mode 100644 index 00000000000..e896974b244 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTestV1.tsx @@ -0,0 +1,304 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Col, Row } from 'antd'; +import { AxiosError } from 'axios'; +import { isUndefined } from 'lodash'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { useHistory, useParams } from 'react-router-dom'; +import { checkAirflowStatus } from '../../axiosAPIs/ingestionPipelineAPI'; +import { createTestCase, createTestSuites } from '../../axiosAPIs/testAPI'; +import { + getDatabaseDetailsPath, + getDatabaseSchemaDetailsPath, + getServiceDetailsPath, + getTableTabPath, +} from '../../constants/constants'; +import { STEPS_FOR_ADD_TEST_CASE } from '../../constants/profiler.constant'; +import { FqnPart } from '../../enums/entity.enum'; +import { FormSubmitType } from '../../enums/form.enum'; +import { PageLayoutType } from '../../enums/layout.enum'; +import { ServiceCategory } from '../../enums/service.enum'; +import { ProfilerDashboardType } from '../../enums/table.enum'; +import { OwnerType } from '../../enums/user.enum'; +import { CreateTestCase } from '../../generated/api/tests/createTestCase'; +import { TestCase } from '../../generated/tests/testCase'; +import { TestSuite } from '../../generated/tests/testSuite'; +import { + getCurrentUserId, + getEntityName, + getNameFromFQN, + getPartialNameFromTableFQN, +} from '../../utils/CommonUtils'; +import { getSettingPath } from '../../utils/RouterUtils'; +import { serviceTypeLogo } from '../../utils/ServiceUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; +import SuccessScreen from '../common/success-screen/SuccessScreen'; +import TitleBreadcrumb from '../common/title-breadcrumb/title-breadcrumb.component'; +import { TitleBreadcrumbProps } from '../common/title-breadcrumb/title-breadcrumb.interface'; +import PageLayout from '../containers/PageLayout'; +import IngestionStepper from '../IngestionStepper/IngestionStepper.component'; +import { + AddDataQualityTestProps, + SelectTestSuiteType, +} from './AddDataQualityTest.interface'; +import RightPanel from './components/RightPanel'; +import SelectTestSuite from './components/SelectTestSuite'; +import TestCaseForm from './components/TestCaseForm'; +import { INGESTION_DATA, TEST_FORM_DATA } from './rightPanelData'; +import TestSuiteIngestion from './TestSuiteIngestion'; + +const AddDataQualityTestV1: React.FC = ({ table }) => { + const { entityTypeFQN, dashboardType } = useParams>(); + const isColumnFqn = dashboardType === ProfilerDashboardType.COLUMN; + const history = useHistory(); + const [activeServiceStep, setActiveServiceStep] = useState(1); + const [selectedTestSuite, setSelectedTestSuite] = + useState(); + const [testCaseData, setTestCaseData] = useState(); + const [testSuiteData, setTestSuiteData] = useState(); + const [testCaseRes, setTestCaseRes] = useState(); + const [isAirflowRunning, setIsAirflowRunning] = useState(false); + const [addIngestion, setAddIngestion] = useState(false); + + const breadcrumb = useMemo(() => { + const { service, serviceType, fullyQualifiedName = '' } = table; + + const data: TitleBreadcrumbProps['titleLinks'] = [ + { + name: service?.name || '', + url: service + ? getServiceDetailsPath( + service.name || '', + ServiceCategory.DATABASE_SERVICES + ) + : '', + imgSrc: serviceType ? serviceTypeLogo(serviceType) : undefined, + }, + { + name: getPartialNameFromTableFQN(fullyQualifiedName, [ + FqnPart.Database, + ]), + url: getDatabaseDetailsPath(fullyQualifiedName), + }, + { + name: getPartialNameFromTableFQN(fullyQualifiedName, [FqnPart.Schema]), + url: getDatabaseSchemaDetailsPath(fullyQualifiedName), + }, + { + name: getEntityName(table), + url: getTableTabPath(fullyQualifiedName), + }, + ]; + + if (isColumnFqn) { + const colVal = [ + { + name: getNameFromFQN(entityTypeFQN), + url: getTableTabPath(fullyQualifiedName), + }, + { + name: 'Add Column Test', + url: '', + activeTitle: true, + }, + ]; + data.push(...colVal); + } else { + data.push({ + name: 'Add Table Test', + url: '', + activeTitle: true, + }); + } + + return data; + }, [table, entityTypeFQN, isColumnFqn]); + + const handleViewTestSuiteClick = () => { + history.push(getSettingPath()); + }; + + const handleAirflowStatusCheck = (): Promise => { + return new Promise((resolve, reject) => { + checkAirflowStatus() + .then((res) => { + if (res.status === 200) { + setIsAirflowRunning(true); + resolve(); + } else { + setIsAirflowRunning(false); + reject(); + } + }) + .catch(() => { + setIsAirflowRunning(false); + reject(); + }); + }); + }; + + const handleCancelClick = () => { + setActiveServiceStep((pre) => pre - 1); + }; + + const handleTestCaseBack = (testCase: TestCase) => { + setTestCaseData(testCase); + handleCancelClick(); + }; + + const handleSelectTestSuite = (data: SelectTestSuiteType) => { + setSelectedTestSuite(data); + setActiveServiceStep(2); + }; + + const handleFormSubmit = async (data: TestCase) => { + setTestCaseData(data); + if (isUndefined(selectedTestSuite)) return; + try { + const { parameterValues, testDefinition, name, entityLink, description } = + data; + const { isNewTestSuite, data: selectedSuite } = selectedTestSuite; + const owner = { + id: getCurrentUserId(), + type: OwnerType.USER, + }; + const testCasePayload: CreateTestCase = { + name, + description, + entityLink, + parameterValues, + owner, + testDefinition, + testSuite: { + id: selectedSuite?.id || '', + type: 'testSuite', + }, + }; + if (isNewTestSuite && isUndefined(testSuiteData)) { + const testSuitePayload = { + name: selectedTestSuite.name || '', + description: selectedTestSuite.description || '', + owner, + }; + const testSuiteResponse = await createTestSuites(testSuitePayload); + testCasePayload.testSuite.id = testSuiteResponse.id || ''; + setTestSuiteData(testSuiteResponse); + } else if (!isUndefined(testSuiteData)) { + testCasePayload.testSuite.id = testSuiteData.id || ''; + } + + const testCaseResponse = await createTestCase(testCasePayload); + setActiveServiceStep(3); + setTestCaseRes(testCaseResponse); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const RenderSelectedTab = useCallback(() => { + if (activeServiceStep === 2) { + return ( + + ); + } else if (activeServiceStep > 2) { + const successName = selectedTestSuite?.isNewTestSuite + ? `${testSuiteData?.name} & ${testCaseRes?.name}` + : testCaseRes?.name || 'Test case'; + + const successMessage = selectedTestSuite?.isNewTestSuite ? undefined : ( + + + "{successName}" + + + has been created successfully. and will be pickup in next run. + + + ); + + return ( + setAddIngestion(true)} + handleViewServiceClick={handleViewTestSuiteClick} + isAirflowSetup={isAirflowRunning} + name={successName} + showIngestionButton={selectedTestSuite?.isNewTestSuite || false} + state={FormSubmitType.ADD} + successMessage={successMessage} + viewServiceText="View Test Suite" + onCheckAirflowStatus={handleAirflowStatusCheck} + /> + ); + } + + return ( + + ); + }, [activeServiceStep, isAirflowRunning, testCaseRes]); + + useEffect(() => { + handleAirflowStatusCheck(); + }, []); + + return ( + } + layout={PageLayoutType['2ColRTL']} + rightPanel={ + + }> + {addIngestion ? ( + setAddIngestion(false)} + /> + ) : ( + + +
+ {`Add ${isColumnFqn ? 'Column' : 'Table'} Test`} +
+ + + + + {RenderSelectedTab()} + + )} + + ); +}; + +export default AddDataQualityTestV1; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/TestSuiteIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/TestSuiteIngestion.tsx new file mode 100644 index 00000000000..5f8b7cd2583 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/TestSuiteIngestion.tsx @@ -0,0 +1,176 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Col, Row } from 'antd'; +import { AxiosError } from 'axios'; +import { camelCase } from 'lodash'; +import React, { useMemo, useState } from 'react'; +import { useHistory } from 'react-router-dom'; +import { + addIngestionPipeline, + deployIngestionPipelineById, +} from '../../axiosAPIs/ingestionPipelineAPI'; +import { + DEPLOYED_PROGRESS_VAL, + INGESTION_PROGRESS_END_VAL, +} from '../../constants/constants'; +import { FormSubmitType } from '../../enums/form.enum'; +import { IngestionActionMessage } from '../../enums/ingestion.enum'; +import { + ConfigType, + CreateIngestionPipeline, + PipelineType, +} from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; +import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; +import jsonData from '../../jsons/en'; +import { getSettingPath } from '../../utils/RouterUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; +import SuccessScreen from '../common/success-screen/SuccessScreen'; +import DeployIngestionLoaderModal from '../Modals/DeployIngestionLoaderModal/DeployIngestionLoaderModal'; +import { TestSuiteIngestionProps } from './AddDataQualityTest.interface'; +import TestSuiteScheduler from './components/TestSuiteScheduler'; + +const TestSuiteIngestion: React.FC = ({ + testSuite, + onCancel, +}) => { + const history = useHistory(); + const [ingestionData, setIngestionData] = useState(); + const [showDeployModal, setShowDeployModal] = useState(false); + const [showDeployButton, setShowDeployButton] = useState(false); + const [ingestionAction, setIngestionAction] = useState( + IngestionActionMessage.CREATING + ); + const [isIngestionDeployed, setIsIngestionDeployed] = useState(false); + const [isIngestionCreated, setIsIngestionCreated] = useState(false); + const [ingestionProgress, setIngestionProgress] = useState(0); + const getSuccessMessage = useMemo(() => { + const createMessage = showDeployButton + ? 'has been created, but failed to deploy' + : 'has been created and deployed successfully'; + + return ( + + + "{ingestionData?.name ?? 'Test Suite'}" + + {createMessage} + + ); + }, [ingestionData, showDeployButton]); + + const handleIngestionDeploy = (id?: string) => { + setShowDeployModal(true); + + return new Promise((resolve) => { + setIsIngestionCreated(true); + setIngestionProgress(INGESTION_PROGRESS_END_VAL); + setIngestionAction(IngestionActionMessage.DEPLOYING); + + deployIngestionPipelineById(`${id || ingestionData?.id}`) + .then(() => { + setIsIngestionDeployed(true); + setShowDeployButton(false); + setIngestionProgress(DEPLOYED_PROGRESS_VAL); + setIngestionAction(IngestionActionMessage.DEPLOYED); + }) + .catch((err: AxiosError) => { + setShowDeployButton(true); + setIngestionAction(IngestionActionMessage.DEPLOYING_ERROR); + showErrorToast( + err || jsonData['api-error-messages']['deploy-ingestion-error'] + ); + }) + .finally(() => resolve()); + }); + }; + + const createIngestionPipeline = async (repeatFrequency: string) => { + const ingestionPayload: CreateIngestionPipeline = { + airflowConfig: { + scheduleInterval: repeatFrequency, + }, + name: `${testSuite.name}_${PipelineType.TestSuite}`, + pipelineType: PipelineType.TestSuite, + service: { + id: testSuite.id || '', + type: camelCase(PipelineType.TestSuite), + }, + sourceConfig: { + config: { + type: ConfigType.TestSuite, + }, + }, + }; + + const ingestion = await addIngestionPipeline(ingestionPayload); + + setIngestionData(ingestion); + handleIngestionDeploy(ingestion.id).finally(() => + setShowDeployModal(false) + ); + }; + + const handleViewTestSuiteClick = () => { + history.push(getSettingPath()); + }; + + const handleDeployClick = () => { + setShowDeployModal(true); + handleIngestionDeploy?.().finally(() => { + setTimeout(() => setShowDeployModal(false), 500); + }); + }; + + return ( + + +
+ Schedule Ingestion +
+ + + + {isIngestionCreated ? ( + + ) : ( + + )} + + + {showDeployModal && ( + + )} + + ); +}; + +export default TestSuiteIngestion; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/ParameterForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/ParameterForm.tsx new file mode 100644 index 00000000000..55d3bec52cc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/ParameterForm.tsx @@ -0,0 +1,135 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PlusOutlined } from '@ant-design/icons'; +import { Button, Form, Input, InputNumber, Switch } from 'antd'; +import 'codemirror/addon/fold/foldgutter.css'; +import React from 'react'; +import { + TestCaseParameterDefinition, + TestDataType, +} from '../../../generated/tests/testDefinition'; +import SVGIcons, { Icons } from '../../../utils/SvgUtils'; +import '../../TableProfiler/tableProfiler.less'; +import { ParameterFormProps } from '../AddDataQualityTest.interface'; + +const ParameterForm: React.FC = ({ definition }) => { + const prepareForm = (data: TestCaseParameterDefinition) => { + let Field = ; + switch (data.dataType) { + case TestDataType.String: + Field = ; + + break; + case TestDataType.Number: + case TestDataType.Int: + case TestDataType.Decimal: + case TestDataType.Double: + case TestDataType.Float: + Field = ( + + ); + + break; + case TestDataType.Boolean: + Field = ; + + break; + case TestDataType.Array: + case TestDataType.Set: + Field = ( + + ); + + return ( + + {(fields, { add, remove }) => ( + + {data.displayName}: + + + )} + + + + + + + + + ); +}; + +export default SelectTestSuite; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/TestCaseForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/TestCaseForm.tsx new file mode 100644 index 00000000000..ead8d88e4cd --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/TestCaseForm.tsx @@ -0,0 +1,265 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Button, Col, Form, FormProps, Input, Row, Select, Space } from 'antd'; +import { AxiosError } from 'axios'; +import 'codemirror/addon/fold/foldgutter.css'; +import { isEmpty } from 'lodash'; +import { EditorContentRef } from 'Models'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { Controlled as CodeMirror } from 'react-codemirror2'; +import { useParams } from 'react-router-dom'; +import { + getListTestCase, + getListTestDefinitions, +} from '../../../axiosAPIs/testAPI'; +import { API_RES_MAX_SIZE } from '../../../constants/constants'; +import { codeMirrorOption } from '../../../constants/profiler.constant'; +import { ProfilerDashboardType } from '../../../enums/table.enum'; +import { EntityReference } from '../../../generated/entity/services/ingestionPipelines/ingestionPipeline'; +import { + TestCase, + TestCaseParameterValue, +} from '../../../generated/tests/testCase'; +import { + EntityType, + TestDataType, + TestDefinition, +} from '../../../generated/tests/testDefinition'; +import { generateEntityLink } from '../../../utils/TableUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import RichTextEditor from '../../common/rich-text-editor/RichTextEditor'; +import { TestCaseFormProps } from '../AddDataQualityTest.interface'; +import ParameterForm from './ParameterForm'; + +const TestCaseForm: React.FC = ({ + initialValue, + onSubmit, + onCancel, +}) => { + const { entityTypeFQN, dashboardType } = useParams>(); + const isColumnFqn = dashboardType === ProfilerDashboardType.COLUMN; + const [form] = Form.useForm(); + const markdownRef = useRef(); + const [testDefinitions, setTestDefinitions] = useState([]); + const [selectedTestType, setSelectedTestType] = useState( + initialValue?.testDefinition?.id + ); + const [testCases, setTestCases] = useState<{ [key: string]: TestCase }>({}); + const [sqlQuery, setSqlQuery] = useState({ + name: 'sqlExpression', + value: initialValue?.parameterValues?.[0].value || '', + }); + + const fetchAllTestDefinitions = async () => { + try { + const { data } = await getListTestDefinitions({ + limit: API_RES_MAX_SIZE, + entityType: isColumnFqn ? EntityType.Column : EntityType.Table, + }); + + setTestDefinitions(data); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + const fetchAllTestCases = async () => { + try { + const { data } = await getListTestCase({ + fields: 'testDefinition', + limit: API_RES_MAX_SIZE, + entityLink: generateEntityLink(entityTypeFQN, isColumnFqn), + }); + const modifiedData = data.reduce((acc, curr) => { + return { ...acc, [curr.testDefinition.fullyQualifiedName || '']: curr }; + }, {}); + setTestCases(modifiedData); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const GenerateParamsField = useCallback(() => { + const testType = initialValue?.testDefinition?.id ?? selectedTestType; + const selectedDefinition = testDefinitions.find( + (definition) => definition.id === testType + ); + + if (selectedDefinition && selectedDefinition.parameterDefinition) { + const name = selectedDefinition.parameterDefinition[0].name; + if (name === 'sqlExpression') { + return ( + + +

Profile Sample Query

+ { + setSqlQuery((pre) => ({ ...pre, value })); + }} + onChange={(_Editor, _EditorChange, value) => { + setSqlQuery((pre) => ({ ...pre, value })); + }} + /> + + + ); + } + + return ; + } + + return; + }, [selectedTestType, initialValue, testDefinitions, sqlQuery]); + + const createTestCaseObj = (value: { + testName: string; + params: Record; + testTypeId: string; + }) => { + const testType = initialValue?.testDefinition?.id ?? selectedTestType; + const selectedDefinition = testDefinitions.find( + (definition) => definition.id === testType + ); + const paramsValue = selectedDefinition?.parameterDefinition?.[0]; + + const parameterValues = + paramsValue?.name === 'sqlExpression' + ? [sqlQuery] + : Object.entries(value.params || {}).map(([key, value]) => ({ + name: key, + value: + paramsValue?.dataType === TestDataType.Array + ? // need to send array as string formate + JSON.stringify( + (value as { value: string }[]).map((data) => data.value) + ) + : value, + })); + + return { + name: value.testName, + entityLink: generateEntityLink(entityTypeFQN, isColumnFqn), + parameterValues: parameterValues as TestCaseParameterValue[], + testDefinition: { + id: value.testTypeId, + type: 'testDefinition', + }, + description: markdownRef.current?.getEditorContent(), + testSuite: {} as EntityReference, + }; + }; + + const handleFormSubmit: FormProps['onFinish'] = (value) => { + onSubmit(createTestCaseObj(value)); + }; + + const onBack = () => { + const data = form.getFieldsValue(); + onCancel(createTestCaseObj(data)); + }; + + useEffect(() => { + if (testDefinitions.length === 0) { + fetchAllTestDefinitions(); + } + if (isEmpty(testCases)) { + fetchAllTestCases(); + } + }, []); + + return ( +
({ + ...acc, + [curr.name || '']: curr.value, + }), + {} + ), + }} + layout="vertical" + name="tableTestForm" + onFinish={handleFormSubmit} + onValuesChange={(value) => { + if (value.testTypeId) { + setSelectedTestType(value.testTypeId); + } + }}> + { + if ( + Object.values(testCases).some((suite) => suite.name === value) + ) { + return Promise.reject('Name already exist!'); + } + + return Promise.resolve(); + }, + }, + ]}> + + + +