diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/icon-test-suite.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/icon-test-suite.svg new file mode 100644 index 00000000000..15ddb1647b8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/icon-test-suite.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTestV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTestV1.tsx index f72ea4c617f..857871f323f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTestV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTestV1.tsx @@ -55,7 +55,7 @@ import { import RightPanel from './components/RightPanel'; import SelectTestSuite from './components/SelectTestSuite'; import TestCaseForm from './components/TestCaseForm'; -import { INGESTION_DATA, TEST_FORM_DATA } from './rightPanelData'; +import { addTestSuiteRightPanel, INGESTION_DATA } from './rightPanelData'; import TestSuiteIngestion from './TestSuiteIngestion'; const AddDataQualityTestV1: React.FC = ({ table }) => { @@ -126,7 +126,9 @@ const AddDataQualityTestV1: React.FC = ({ table }) => { }, [table, entityTypeFQN, isColumnFqn]); const handleViewTestSuiteClick = () => { - history.push(getTestSuitePath(testSuiteData?.fullyQualifiedName || '')); + history.push( + getTestSuitePath(selectedTestSuite?.data?.fullyQualifiedName || '') + ); }; const handleAirflowStatusCheck = (): Promise => { @@ -226,7 +228,8 @@ const AddDataQualityTestV1: React.FC = ({ table }) => { "{successName}" - has been created successfully. and will be pickup in next run. + has been created successfully. This will be picked up in the next + run. ); @@ -268,7 +271,14 @@ const AddDataQualityTestV1: React.FC = ({ table }) => { data={ addIngestion ? INGESTION_DATA - : TEST_FORM_DATA[activeServiceStep - 1] + : addTestSuiteRightPanel( + activeServiceStep, + selectedTestSuite?.isNewTestSuite, + { + testCase: testCaseData?.name || '', + testSuite: testSuiteData?.name || '', + } + ) } /> }> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/TestSuiteIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/TestSuiteIngestion.tsx index 298354fe466..2722425dca7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/TestSuiteIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/TestSuiteIngestion.tsx @@ -13,7 +13,7 @@ import { Col, Row, Typography } from 'antd'; import { AxiosError } from 'axios'; import { compare } from 'fast-json-patch'; -import { camelCase } from 'lodash'; +import { camelCase, isEmpty } from 'lodash'; import React, { useMemo, useState } from 'react'; import { useHistory, useParams } from 'react-router-dom'; import { @@ -105,7 +105,9 @@ const TestSuiteIngestion: React.FC = ({ const createIngestionPipeline = async (repeatFrequency: string) => { const ingestionPayload: CreateIngestionPipeline = { airflowConfig: { - scheduleInterval: repeatFrequency, + scheduleInterval: isEmpty(repeatFrequency) + ? undefined + : repeatFrequency, }, name: `${testSuite.name}_${PipelineType.TestSuite}`, pipelineType: PipelineType.TestSuite, @@ -131,7 +133,9 @@ const TestSuiteIngestion: React.FC = ({ ...ingestionPipeline, airflowConfig: { ...ingestionPipeline?.airflowConfig, - scheduleInterval: repeatFrequency, + scheduleInterval: isEmpty(repeatFrequency) + ? undefined + : repeatFrequency, }, }; const jsonPatch = compare( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/rightPanelData.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/rightPanelData.ts index a09ec53ddd7..cda435249a8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/rightPanelData.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/rightPanelData.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-len */ /* * Copyright 2022 Collate * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,19 +15,49 @@ export const TEST_FORM_DATA = [ { title: 'Select/Create Test Suite', - body: 'Select existing test suite or Create new Test Suite', + body: 'To create a Table or Column level test for an entity, start by selecting the Test Suite. Select an existing test suite or create a new test suite.', }, { - title: 'Test Case', - body: 'Fill up the relevant details and create test case', + title: 'Create Test Case', + body: 'To create a test case, add a unique name. Select a test type from the options provided. Fill in the details for the parameters that show up for a selected test type. Enter a description (optional).', }, { - title: 'Test case created successfully', - body: 'Visit the newly created test case to take a look at the details. Ensure that you have Airflow set up correctly before heading to ingest metadata.', + title: 'Test Case Created Successfully', + body: '{testCase} has been created successfully. View the Test Suite to check the details of the new created test case. This will be picked up in the next run.', + }, + { + title: 'Test Suite & Test Case Created Successfully', + body: '{testSuite} & {testCase} has been created successfully. In the next step, you can schedule to ingest metadata at the desired frequency. You can also view the Test Suite to check the details of the new created test case.', }, ]; export const INGESTION_DATA = { - title: 'Schedule for Ingestion', - body: 'Scheduling can be set up at an hourly, daily, or weekly cadence. The timezone is in UTC.', + title: 'Scheduler for Tests', + body: 'The data quality tests can be scheduled to run at the desired frequency. The timezone is in UTC.', +}; + +export const addTestSuiteRightPanel = ( + step: number, + isSuiteCreate?: boolean, + data?: { testSuite: string; testCase: string } +) => { + let message = TEST_FORM_DATA[step - 1]; + + if (step === 3) { + if (isSuiteCreate) { + message = TEST_FORM_DATA[step]; + const updatedMessage = message.body + .replace('{testSuite}', data?.testSuite || 'Test Suite') + .replace('{testCase}', data?.testCase || 'Test Case'); + message.body = updatedMessage; + } else { + const updatedMessage = message.body.replace( + '{testCase}', + data?.testCase || 'Test Case' + ); + message.body = updatedMessage; + } + } + + return message; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx index 5cd22458a13..edbdd9a348b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx @@ -255,17 +255,6 @@ const DatasetDetails: React.FC = ({ isProtected: false, position: 5, }, - { - name: 'Data Quality', - icon: { - alt: 'data-quality', - name: 'icon-quality', - title: 'Data Quality', - selectedName: '', - }, - isProtected: false, - position: 6, - }, { name: 'Lineage', icon: { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx index 6b3697eab44..75ea0d14f35 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx @@ -241,7 +241,6 @@ describe('Test MyDataDetailsPage page', () => { const sampleDataTab = await findByTestId(tabs, 'Sample Data'); const queriesTab = await findByTestId(tabs, 'Queries'); const profilerTab = await findByTestId(tabs, 'Profiler'); - const dataQualityTab = await findByTestId(tabs, 'Data Quality'); const lineageTab = await findByTestId(tabs, 'Lineage'); const dbtTab = queryByTestId(tabs, 'DBT'); @@ -254,7 +253,6 @@ describe('Test MyDataDetailsPage page', () => { expect(sampleDataTab).toBeInTheDocument(); expect(queriesTab).toBeInTheDocument(); expect(profilerTab).toBeInTheDocument(); - expect(dataQualityTab).toBeInTheDocument(); expect(lineageTab).toBeInTheDocument(); expect(dbtTab).not.toBeInTheDocument(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyAssetStats/MyAssetStats.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyAssetStats/MyAssetStats.component.tsx index e01194e86ef..19fed2dd0f5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyAssetStats/MyAssetStats.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyAssetStats/MyAssetStats.component.tsx @@ -73,7 +73,7 @@ const MyAssetStats: FunctionComponent = ({ dataTestId: 'mlmodels', }, testSuite: { - icon: Icons.TABLE_GREY, + icon: Icons.TEST_SUITE, data: 'Test Suite', count: entityCounts.testSuiteCount, link: getSettingPath( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx index b026f195169..90a539fac6e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx @@ -46,6 +46,7 @@ import { getPartialNameFromTableFQN, hasEditAccess, } from '../../utils/CommonUtils'; +import { getAddDataQualityTableTestPath } from '../../utils/RouterUtils'; import { serviceTypeLogo } from '../../utils/ServiceUtils'; import { generateEntityLink, @@ -299,7 +300,12 @@ const ProfilerDashboard: React.FC = ({ const handleAddTestClick = () => { history.push( - getTableTabPath(table.fullyQualifiedName || '', 'data-quality') + getAddDataQualityTableTestPath( + isColumnView + ? ProfilerDashboardType.COLUMN + : ProfilerDashboardType.TABLE, + entityTypeFQN || '' + ) ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.tsx index 38a48f87855..d376542e4aa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.tsx @@ -12,16 +12,26 @@ */ import { Card, Col, Row, Statistic } from 'antd'; +import { AxiosError } from 'axios'; import { sortBy } from 'lodash'; import moment from 'moment'; import React, { useEffect, useMemo, useState } from 'react'; +import { useParams } from 'react-router-dom'; +import { getListTestCase } from '../../../axiosAPIs/testAPI'; +import { API_RES_MAX_SIZE } from '../../../constants/constants'; import { INITIAL_COUNT_METRIC_VALUE, INITIAL_MATH_METRIC_VALUE, INITIAL_PROPORTION_METRIC_VALUE, INITIAL_SUM_METRIC_VALUE, + INITIAL_TEST_RESULT_SUMMARY, } from '../../../constants/profiler.constant'; +import { getTableFQNFromColumnFQN } from '../../../utils/CommonUtils'; +import { updateTestResults } from '../../../utils/DataQualityAndProfilerUtils'; +import { generateEntityLink } from '../../../utils/TableUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; import Ellipses from '../../common/Ellipses/Ellipses'; +import { TableTestsType } from '../../TableProfiler/TableProfiler.interface'; import { MetricChartType, ProfilerTabProps, @@ -34,6 +44,7 @@ const ProfilerTab: React.FC = ({ profilerData, tableProfile, }) => { + const { entityTypeFQN } = useParams>(); const [countMetrics, setCountMetrics] = useState( INITIAL_COUNT_METRIC_VALUE ); @@ -46,6 +57,10 @@ const ProfilerTab: React.FC = ({ const [sumMetrics, setSumMetrics] = useState( INITIAL_SUM_METRIC_VALUE ); + const [tableTests, setTableTests] = useState({ + tests: [], + results: INITIAL_TEST_RESULT_SUMMARY, + }); const tableState = useMemo( () => [ @@ -64,23 +79,24 @@ const ProfilerTab: React.FC = ({ ], [tableProfile] ); - const testSummary = useMemo( - () => [ + const testSummary = useMemo(() => { + const { results } = tableTests; + + return [ { title: 'Success', - value: 0, + value: results.success, }, { title: 'Aborted', - value: 0, + value: results.aborted, }, { title: 'Failed', - value: 0, + value: results.failed, }, - ], - [] - ); + ]; + }, [tableTests]); const createMetricsChartData = () => { const updateProfilerData = sortBy(profilerData, 'timestamp'); @@ -171,10 +187,38 @@ const ProfilerTab: React.FC = ({ })); }; + const fetchAllTests = async () => { + const tableFqn = getTableFQNFromColumnFQN(entityTypeFQN); + try { + const { data } = await getListTestCase({ + fields: 'testCaseResult', + entityLink: generateEntityLink(tableFqn), + limit: API_RES_MAX_SIZE, + }); + const tableTests: TableTestsType = { + tests: [], + results: { ...INITIAL_TEST_RESULT_SUMMARY }, + }; + data.forEach((test) => { + updateTestResults( + tableTests.results, + test.testCaseResult?.testCaseStatus || '' + ); + }); + setTableTests(tableTests); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + useEffect(() => { createMetricsChartData(); }, [profilerData]); + useEffect(() => { + fetchAllTests(); + }, []); + return ( diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx index 18e1dcfb3e1..c68a7a8458a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx @@ -84,7 +84,7 @@ const ProfilerDashboardPage = () => { }; const handleTestCaseUpdate = () => { - fetchTestCases(generateEntityLink(entityTypeFQN)); + fetchTestCases(generateEntityLink(entityTypeFQN, isColumnView)); }; const fetchTableEntity = async () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsUtils.tsx index 55c756266fd..3d63d025618 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsUtils.tsx @@ -17,6 +17,7 @@ import React, { ReactNode } from 'react'; import { ReactComponent as BotIcon } from '../../src/assets/svg/bot-profile.svg'; import { ReactComponent as DashboardIcon } from '../../src/assets/svg/dashboard-grey.svg'; import { ReactComponent as RolesIcon } from '../../src/assets/svg/icon-role-grey.svg'; +import { ReactComponent as TestSuite } from '../../src/assets/svg/icon-test-suite.svg'; import { ReactComponent as MlModelIcon } from '../../src/assets/svg/mlmodal.svg'; import { ReactComponent as PipelineIcon } from '../../src/assets/svg/pipeline-grey.svg'; import { ReactComponent as PoliciesIcon } from '../../src/assets/svg/policies.svg'; @@ -162,7 +163,7 @@ export const getGlobalSettingsMenuWithPermission = ( ResourceEntity.TEST_SUITE, permissions ), - icon: , + icon: , }, ], }, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/SvgUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/SvgUtils.tsx index 5457cac1419..b0e5c2e89b5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/SvgUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/SvgUtils.tsx @@ -118,6 +118,7 @@ import IconKey from '../assets/svg/icon-key.svg'; import IconNotNull from '../assets/svg/icon-notnull.svg'; import IconPlusPrimaryOutlined from '../assets/svg/icon-plus-primary-outlined.svg'; import IconRoleGrey from '../assets/svg/icon-role-grey.svg'; +import IconTestSuite from '../assets/svg/icon-test-suite.svg'; import IconTour from '../assets/svg/icon-tour.svg'; import IconUnique from '../assets/svg/icon-unique.svg'; import IconUp from '../assets/svg/icon-up.svg'; @@ -204,6 +205,7 @@ export const Icons = { SQL_BUILDER: 'icon-sql-builder', TEAMS: 'icon-teams', TEAMS_GREY: 'icon-teams-grey', + TEST_SUITE: 'icon-test-suite', WORKFLOWS: 'icon-workflows', MENU: 'icon-menu', FEED: 'icon-feed', @@ -1008,6 +1010,10 @@ const SVGIcons: FunctionComponent = ({ case Icons.DELETE_COLORED: IconComponent = IconDeleteColored; + break; + case Icons.TEST_SUITE: + IconComponent = IconTestSuite; + break; default: IconComponent = null;