diff --git a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/testAPI.ts b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/testAPI.ts new file mode 100644 index 00000000000..439fc6aac3c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/testAPI.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TestCase, TestCaseResult } from '../generated/tests/testCase'; +import { Include } from '../generated/type/include'; +import { Paging } from '../generated/type/paging'; +import APIClient from './index'; + +export type ListTestCaseParams = { + fields?: string; + limit?: number; + before?: string; + after?: string; + entityLink?: string; + testSuiteId?: string; + includeAllTests?: boolean; + include?: Include; +}; + +export type ListTestCaseResultsParams = { + startTs?: number; + endTs?: number; + before?: string; + after?: string; + limit?: number; +}; + +const baseUrl = '/testCase'; + +export const getListTestCase = async (params?: ListTestCaseParams) => { + const response = await APIClient.get<{ data: TestCase[]; paging: Paging }>( + baseUrl, + { + params, + } + ); + + return response.data; +}; + +export const getListTestCaseResults = async ( + fqn: string, + params?: ListTestCaseResultsParams +) => { + const url = `${baseUrl}/${fqn}/testCaseResult`; + const response = await APIClient.get<{ + data: TestCaseResult[]; + paging: Paging; + }>(url, { + params, + }); + + return response.data; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx index ccc3c4f749b..7d80c7a4eb8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx @@ -10,7 +10,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - import { Button, Col, Radio, Row, Select, Space } from 'antd'; import { RadioChangeEvent } from 'antd/lib/radio'; import { AxiosError } from 'axios'; @@ -29,8 +28,13 @@ import { import { PROFILER_FILTER_RANGE } from '../../constants/profiler.constant'; import { EntityType, FqnPart } from '../../enums/entity.enum'; import { ServiceCategory } from '../../enums/service.enum'; +import { ProfilerDashboardType } from '../../enums/table.enum'; import { OwnerType } from '../../enums/user.enum'; -import { Column, Table } from '../../generated/entity/data/table'; +import { + Column, + Table, + TestCaseStatus, +} from '../../generated/entity/data/table'; import { EntityReference } from '../../generated/type/entityReference'; import { LabelType, State } from '../../generated/type/tagLabel'; import jsonData from '../../jsons/en'; @@ -44,6 +48,7 @@ import { } from '../../utils/CommonUtils'; import { serviceTypeLogo } from '../../utils/ServiceUtils'; import { + generateEntityLink, getTagsWithoutTier, getTierTags, getUsagePercentile, @@ -51,6 +56,7 @@ import { import { showErrorToast } from '../../utils/ToastUtils'; import EntityPageInfo from '../common/entityPageInfo/EntityPageInfo'; import PageLayout from '../containers/PageLayout'; +import DataQualityTab from './component/DataQualityTab'; import ProfilerTab from './component/ProfilerTab'; import { ProfilerDashboardProps, @@ -60,23 +66,40 @@ import './profilerDashboard.less'; const ProfilerDashboard: React.FC = ({ table, + testCases, fetchProfilerData, + fetchTestCases, profilerData, onTableChange, }) => { const history = useHistory(); - const { entityTypeFQN } = useParams>(); + const { entityTypeFQN, dashboardType } = useParams>(); + const isColumnView = dashboardType === ProfilerDashboardType.COLUMN; const [follower, setFollower] = useState([]); const [isFollowing, setIsFollowing] = useState(false); const [activeTab, setActiveTab] = useState( - ProfilerDashboardTab.PROFILER + isColumnView + ? ProfilerDashboardTab.PROFILER + : ProfilerDashboardTab.DATA_QUALITY ); + const [selectedTestCaseStatus, setSelectedTestCaseStatus] = + useState(''); const [selectedTimeRange, setSelectedTimeRange] = useState('last3days'); const [activeColumnDetails, setActiveColumnDetails] = useState( {} as Column ); + const tabOptions = useMemo(() => { + return Object.values(ProfilerDashboardTab).filter((value) => { + if (value === ProfilerDashboardTab.PROFILER) { + return isColumnView; + } + + return value; + }); + }, [dashboardType]); + const timeRangeOption = useMemo(() => { return Object.entries(PROFILER_FILTER_RANGE).map(([key, value]) => ({ label: value.title, @@ -84,6 +107,21 @@ const ProfilerDashboard: React.FC = ({ })); }, []); + const testCaseStatusOption = useMemo(() => { + const testCaseStatus: Record[] = Object.values( + TestCaseStatus + ).map((value) => ({ + label: value, + value: value, + })); + testCaseStatus.unshift({ + label: 'All Test', + value: '', + }); + + return testCaseStatus; + }, []); + const tier = useMemo(() => getTierTags(table.tags ?? []), [table]); const breadcrumb = useMemo(() => { const serviceName = getEntityName(table.service); @@ -251,7 +289,10 @@ const ProfilerDashboard: React.FC = ({ const value = e.target.value as ProfilerDashboardTab; if (ProfilerDashboardTab.SUMMARY === value) { history.push(getTableTabPath(table.fullyQualifiedName || '', 'profiler')); + } else if (ProfilerDashboardTab.DATA_QUALITY === value) { + fetchTestCases(generateEntityLink(entityTypeFQN, true)); } + setSelectedTestCaseStatus(''); setActiveTab(value); }; @@ -264,17 +305,35 @@ const ProfilerDashboard: React.FC = ({ const handleTimeRangeChange = (value: keyof typeof PROFILER_FILTER_RANGE) => { if (value !== selectedTimeRange) { setSelectedTimeRange(value); - fetchProfilerData(entityTypeFQN, PROFILER_FILTER_RANGE[value].days); + if (activeTab === ProfilerDashboardTab.PROFILER) { + fetchProfilerData(entityTypeFQN, PROFILER_FILTER_RANGE[value].days); + } } }; + const handleTestCaseStatusChange = (value: string) => { + if (value !== selectedTestCaseStatus) { + setSelectedTestCaseStatus(value); + } + }; + + const getFilterTestCase = () => { + return testCases.filter( + (data) => + selectedTestCaseStatus === '' || + data.testCaseResult?.testCaseStatus === selectedTestCaseStatus + ); + }; + useEffect(() => { if (table) { - const columnName = getNameFromFQN(entityTypeFQN); - const selectedColumn = table.columns.find( - (col) => col.name === columnName - ); - setActiveColumnDetails(selectedColumn || ({} as Column)); + if (isColumnView) { + const columnName = getNameFromFQN(entityTypeFQN); + const selectedColumn = table.columns.find( + (col) => col.name === columnName + ); + setActiveColumnDetails(selectedColumn || ({} as Column)); + } setFollower(table?.followers || []); setIsFollowing( follower.some(({ id }: { id: string }) => id === getCurrentUserId()) @@ -315,18 +374,28 @@ const ProfilerDashboard: React.FC = ({ - + )} + {activeTab === ProfilerDashboardTab.PROFILER && ( + + + + + + + + + {data.parameterValues?.length === 2 && referenceArea()} + {chartData?.information?.map((info, i) => ( + + ))} + + + + ) : ( + + )} + + + + + Name: + {data.displayName || data.name} + + + Parameter: + + + {data.parameterValues?.map((param) => ( + + {param.name}: + {param.value} + + ))} + + + + Description: + + + + + + ); +}; + +export default TestSummary; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/profilerDashboard.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/profilerDashboard.interface.ts index 24a88141735..b57efb27d31 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/profilerDashboard.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/profilerDashboard.interface.ts @@ -16,12 +16,15 @@ import { ColumnProfile, Table, } from '../../generated/entity/data/table'; +import { TestCase } from '../../generated/tests/testCase'; export interface ProfilerDashboardProps { onTableChange: (table: Table) => void; table: Table; + testCases: TestCase[]; profilerData: ColumnProfile[]; fetchProfilerData: (tableId: string, days?: number) => void; + fetchTestCases: (fqn: string) => void; } export type MetricChartType = { @@ -73,3 +76,11 @@ export interface ProfilerSummaryCardProps { }[]; showIndicator?: boolean; } + +export interface DataQualityTabProps { + testCases: TestCase[]; +} + +export interface TestSummaryProps { + data: TestCase; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ColumnProfileTable.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ColumnProfileTable.test.tsx index 0a9c1a1a46a..b937b8162d3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ColumnProfileTable.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ColumnProfileTable.test.tsx @@ -101,6 +101,7 @@ jest.mock('../../../utils/DatasetDetailsUtils'); const mockProps: ColumnProfileTableProps = { columns: MOCK_TABLE.columns, onAddTestClick: jest.fn, + columnTests: [], }; describe('Test ColumnProfileTable component', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ColumnProfileTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ColumnProfileTable.tsx index f80a12152f7..9e0d0947b66 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ColumnProfileTable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/ColumnProfileTable.tsx @@ -13,6 +13,7 @@ import { Button, Space, Table } from 'antd'; import { ColumnsType } from 'antd/lib/table'; +import { isUndefined } from 'lodash'; import React, { FC, useEffect, useMemo, useState } from 'react'; import { Link } from 'react-router-dom'; import { @@ -20,38 +21,35 @@ import { SECONDARY_COLOR, SUCCESS_COLOR, } from '../../../constants/constants'; +import { + DEFAULT_TEST_VALUE, + INITIAL_TEST_RESULT_SUMMARY, +} from '../../../constants/profiler.constant'; +import { ProfilerDashboardType } from '../../../enums/table.enum'; import { Column, ColumnProfile } from '../../../generated/entity/data/table'; -import { TestCaseStatus } from '../../../generated/tests/tableTest'; import { formatNumberWithComma } from '../../../utils/CommonUtils'; +import { updateTestResults } from '../../../utils/DataQualityAndProfilerUtils'; import { getCurrentDatasetTab } from '../../../utils/DatasetDetailsUtils'; import { getProfilerDashboardWithFqnPath } from '../../../utils/RouterUtils'; import Ellipses from '../../common/Ellipses/Ellipses'; import Searchbar from '../../common/searchbar/Searchbar'; import TestIndicator from '../../common/TestIndicator/TestIndicator'; -import { ColumnProfileTableProps } from '../TableProfiler.interface'; +import { + ColumnProfileTableProps, + columnTestResultType, +} from '../TableProfiler.interface'; import ProfilerProgressWidget from './ProfilerProgressWidget'; const ColumnProfileTable: FC = ({ + columnTests, onAddTestClick, columns = [], }) => { const [searchText, setSearchText] = useState(''); const [data, setData] = useState(columns); - // TODO:- Once column level test filter is implemented in test case API, remove this hardcoded value - const testDetails = [ - { - value: 0, - type: TestCaseStatus.Success, - }, - { - value: 0, - type: TestCaseStatus.Aborted, - }, - { - value: 0, - type: TestCaseStatus.Failed, - }, - ]; + const [columnTestSummary, setColumnTestSummary] = + useState(); + const tableColumn: ColumnsType = useMemo(() => { return [ { @@ -62,6 +60,7 @@ const ColumnProfileTable: FC = ({ return ( {name} @@ -130,10 +129,18 @@ const ColumnProfileTable: FC = ({ title: 'Test', dataIndex: 'dataQualityTest', key: 'dataQualityTest', - render: () => { + render: (_, record) => { + const summary = columnTestSummary?.[record.fullyQualifiedName || '']; + const currentResult = summary + ? Object.entries(summary).map(([key, value]) => ({ + value, + type: key, + })) + : DEFAULT_TEST_VALUE; + return ( - {testDetails.map((test, i) => ( + {currentResult.map((test, i) => ( ))} @@ -160,7 +167,7 @@ const ColumnProfileTable: FC = ({ ), }, ]; - }, [columns]); + }, [columns, columnTestSummary]); const handleSearchAction = (searchText: string) => { setSearchText(searchText); @@ -175,6 +182,21 @@ const ColumnProfileTable: FC = ({ setData(columns); }, [columns]); + useEffect(() => { + if (columnTests.length) { + const colResult = columnTests.reduce((acc, curr) => { + const fqn = curr.entityFQN || ''; + if (isUndefined(acc[fqn])) { + acc[fqn] = { ...INITIAL_TEST_RESULT_SUMMARY }; + } + updateTestResults(acc[fqn], curr.testCaseResult?.testCaseStatus || ''); + + return acc; + }, {} as columnTestResultType); + setColumnTestSummary(colResult); + } + }, [columnTests]); + return (
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts index 073a1417638..ea098e07838 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.interface.ts @@ -12,7 +12,7 @@ */ import { Column, Table } from '../../generated/entity/data/table'; -import { TestCaseStatus } from '../../generated/tests/tableTest'; +import { TestCase } from '../../generated/tests/testCase'; import { DatasetTestModeType } from '../../interface/dataQuality.interface'; export interface TableProfilerProps { @@ -24,8 +24,20 @@ export interface TableProfilerProps { table: Table; } +export type TableTestsType = { + tests: TestCase[]; + results: { + success: number; + aborted: number; + failed: number; + }; +}; + +export type columnTestResultType = { [key: string]: TableTestsType['results'] }; + export interface ColumnProfileTableProps { columns: Column[]; + columnTests: TestCase[]; onAddTestClick: ( tabValue: number, testMode?: DatasetTestModeType, @@ -47,7 +59,7 @@ export interface ProfilerSettingsModalProps { export interface TestIndicatorProps { value: number | string; - type: TestCaseStatus; + type: string; } export type OverallTableSummeryType = { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.test.tsx index 36377d49beb..3295f636fb3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.test.tsx @@ -20,7 +20,7 @@ import { screen, } from '@testing-library/react'; import React from 'react'; -import { MOCK_TABLE } from '../../mocks/TableData.mock'; +import { MOCK_TABLE, TEST_CASE } from '../../mocks/TableData.mock'; import { getCurrentDatasetTab } from '../../utils/DatasetDetailsUtils'; import { TableProfilerProps } from './TableProfiler.interface'; // internal imports @@ -69,6 +69,12 @@ jest.mock('../../utils/CommonUtils', () => ({ })); const mockGetCurrentDatasetTab = getCurrentDatasetTab as jest.Mock; +jest.mock('../../axiosAPIs/testAPI', () => ({ + getListTestCase: jest + .fn() + .mockImplementation(() => Promise.resolve(TEST_CASE)), +})); + const mockProps: TableProfilerProps = { table: MOCK_TABLE, onAddTestClick: jest.fn(), @@ -128,9 +134,6 @@ describe('Test TableProfiler component', () => { }); it('CTA: Setting button should work properly', async () => { - const setSettingModalVisible = jest.fn(); - const handleClick = jest.spyOn(React, 'useState'); - handleClick.mockImplementation(() => [false, setSettingModalVisible]); render(); const settingBtn = await screen.findByTestId('profiler-setting-btn'); @@ -141,6 +144,8 @@ describe('Test TableProfiler component', () => { fireEvent.click(settingBtn); }); - expect(setSettingModalVisible).toHaveBeenCalledTimes(1); + expect( + await screen.findByText('ProfilerSettingsModal.component') + ).toBeInTheDocument(); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.tsx index 61e0ce843c6..dde41ba0bae 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfilerV1.tsx @@ -12,27 +12,43 @@ */ import { Button, Col, Row } from 'antd'; +import { AxiosError } from 'axios'; import classNames from 'classnames'; -import { isUndefined } from 'lodash'; -import React, { FC, useMemo, useState } from 'react'; +import { isEmpty, isUndefined } from 'lodash'; +import React, { FC, useEffect, useMemo, useState } from 'react'; import { Link } from 'react-router-dom'; +import { getListTestCase } from '../../axiosAPIs/testAPI'; +import { API_RES_MAX_SIZE } from '../../constants/constants'; +import { INITIAL_TEST_RESULT_SUMMARY } from '../../constants/profiler.constant'; +import { ProfilerDashboardType } from '../../enums/table.enum'; +import { TestCase } from '../../generated/tests/testCase'; import { formatNumberWithComma, formTwoDigitNmber, } from '../../utils/CommonUtils'; +import { updateTestResults } from '../../utils/DataQualityAndProfilerUtils'; import { getCurrentDatasetTab } from '../../utils/DatasetDetailsUtils'; +import { getProfilerDashboardWithFqnPath } from '../../utils/RouterUtils'; import SVGIcons, { Icons } from '../../utils/SvgUtils'; +import { generateEntityLink } from '../../utils/TableUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; import ColumnProfileTable from './Component/ColumnProfileTable'; import ProfilerSettingsModal from './Component/ProfilerSettingsModal'; import { OverallTableSummeryType, TableProfilerProps, + TableTestsType, } from './TableProfiler.interface'; import './tableProfiler.less'; const TableProfilerV1: FC = ({ table, onAddTestClick }) => { const { profile, columns } = table; const [settingModalVisible, setSettingModalVisible] = useState(false); + const [columnTests, setColumnTests] = useState([]); + const [tableTests, setTableTests] = useState({ + tests: [], + results: INITIAL_TEST_RESULT_SUMMARY, + }); const handleSettingModal = (value: boolean) => { setSettingModalVisible(value); @@ -53,21 +69,59 @@ const TableProfilerV1: FC = ({ table, onAddTestClick }) => { }, { title: 'Success', - value: formTwoDigitNmber(0), + value: formTwoDigitNmber(tableTests.results.success), className: 'success', }, { title: 'Aborted', - value: formTwoDigitNmber(0), + value: formTwoDigitNmber(tableTests.results.aborted), className: 'aborted', }, { title: 'Failed', - value: formTwoDigitNmber(0), + value: formTwoDigitNmber(tableTests.results.failed), className: 'failed', }, ]; - }, [profile]); + }, [profile, tableTests]); + + const fetchAllTests = async () => { + try { + const { data } = await getListTestCase({ + fields: 'testCaseResult', + entityLink: generateEntityLink(table.fullyQualifiedName || ''), + includeAllTests: true, + limit: API_RES_MAX_SIZE, + }); + const columnTestsCase: TestCase[] = []; + const tableTests: TableTestsType = { + tests: [], + results: { ...INITIAL_TEST_RESULT_SUMMARY }, + }; + data.forEach((test) => { + if (test.entityFQN === table.fullyQualifiedName) { + tableTests.tests.push(test); + + updateTestResults( + tableTests.results, + test.testCaseResult?.testCaseStatus || '' + ); + + return; + } + columnTestsCase.push(test); + }); + setTableTests(tableTests); + setColumnTests(columnTestsCase); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + useEffect(() => { + if (isEmpty(table)) return; + fetchAllTests(); + }, [table]); if (isUndefined(profile)) { return ( @@ -134,9 +188,19 @@ const TableProfilerV1: FC = ({ table, onAddTestClick }) => {

))} + + + View more detail + + ({ ...col, key: col.name, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidget.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidget.interface.ts index 271be816d67..54288b1a2e3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidget.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidget.interface.ts @@ -20,6 +20,7 @@ export interface DeleteWidgetModalProps { entityType: string; isAdminUser?: boolean; entityId?: string; + prepareType?: boolean; isRecursiveDelete?: boolean; afterDeleteAction?: () => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx index 5fac0747975..3125c3bb8ce 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx @@ -35,6 +35,7 @@ const DeleteWidgetModal = ({ entityType, onCancel, entityId, + prepareType = true, isRecursiveDelete, afterDeleteAction, }: DeleteWidgetModalProps) => { @@ -109,7 +110,7 @@ const DeleteWidgetModal = ({ const handleOnEntityDeleteConfirm = () => { setEntityDeleteState((prev) => ({ ...prev, loading: 'waiting' })); deleteEntity( - prepareEntityType(), + prepareType ? prepareEntityType() : entityType, entityId ?? '', Boolean(isRecursiveDelete), entityDeleteState.softDelete diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts index 3c61ceceecf..60a05b043d5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts @@ -93,6 +93,7 @@ export const PLACEHOLDER_SETTING_CATEGORY = ':settingCategory'; export const PLACEHOLDER_USER_BOT = ':bot'; export const PLACEHOLDER_WEBHOOK_TYPE = ':webhookType'; export const PLACEHOLDER_RULE_NAME = ':ruleName'; +export const PLACEHOLDER_DASHBOARD_TYPE = ':dashboardType'; export const pagingObject = { after: '', before: '', total: 0 }; @@ -220,7 +221,7 @@ export const ROUTES = { MLMODEL_DETAILS_WITH_TAB: `/mlmodel/${PLACEHOLDER_ROUTE_MLMODEL_FQN}/${PLACEHOLDER_ROUTE_TAB}`, CUSTOM_ENTITY_DETAIL: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}`, ADD_CUSTOM_PROPERTY: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}/add-field`, - PROFILER_DASHBOARD: `/profiler-dashboard/${PLACEHOLDER_ENTITY_TYPE_FQN}`, + PROFILER_DASHBOARD: `/profiler-dashboard/${PLACEHOLDER_DASHBOARD_TYPE}/${PLACEHOLDER_ENTITY_TYPE_FQN}`, // Tasks Routes REQUEST_DESCRIPTION: `/request-description/${PLACEHOLDER_ROUTE_ENTITY_TYPE}/${PLACEHOLDER_ROUTE_ENTITY_FQN}`, diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts index 77dbffeb8db..a1a4b2306ee 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts @@ -13,6 +13,7 @@ import { CSMode } from '../enums/codemirror.enum'; import { ColumnProfilerConfig } from '../generated/entity/data/table'; +import { TestCaseStatus } from '../generated/tests/tableTest'; import { JSON_TAB_SIZE } from './constants'; export const excludedMetrics = [ @@ -63,6 +64,8 @@ export const PROFILER_FILTER_RANGE = { last60days: { days: 60, title: 'Last 60 days' }, }; +export const COLORS = ['#7147E8', '#B02AAC', '#B02AAC', '#1890FF', '#008376']; + export const DEFAULT_CHART_COLLECTION_VALUE = { distinctCount: { data: [], color: '#1890FF' }, uniqueCount: { data: [], color: '#008376' }, @@ -161,6 +164,27 @@ export const DEFAULT_INCLUDE_PROFILE: ColumnProfilerConfig[] = [ }, ]; +export const INITIAL_TEST_RESULT_SUMMARY = { + success: 0, + aborted: 0, + failed: 0, +}; + +export const DEFAULT_TEST_VALUE = [ + { + value: 0, + type: TestCaseStatus.Success, + }, + { + value: 0, + type: TestCaseStatus.Aborted, + }, + { + value: 0, + type: TestCaseStatus.Failed, + }, +]; + export const codeMirrorOption = { tabSize: JSON_TAB_SIZE, indentUnit: JSON_TAB_SIZE, diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/table.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/table.enum.ts index 0c778df88c6..00683f6a9ab 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/table.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/table.enum.ts @@ -26,3 +26,8 @@ export enum PrimaryTableDataTypes { NUMERIC = 'numeric', BOOLEAN = 'boolean', } + +export enum ProfilerDashboardType { + TABLE = 'table', + COLUMN = 'column', +} diff --git a/openmetadata-ui/src/main/resources/ui/src/jsons/en.ts b/openmetadata-ui/src/main/resources/ui/src/jsons/en.ts index eb7cc3f5013..45e81533917 100644 --- a/openmetadata-ui/src/main/resources/ui/src/jsons/en.ts +++ b/openmetadata-ui/src/main/resources/ui/src/jsons/en.ts @@ -111,6 +111,7 @@ const jsonData = { 'fetch-users-error': 'Error while fetching users!', 'fetch-table-profiler-config-error': 'Error while fetching table profiler config!', + 'fetch-column-test-error': 'Error while fetching column test case!', 'test-connection-error': 'Error while testing connection!', diff --git a/openmetadata-ui/src/main/resources/ui/src/mocks/TableData.mock.ts b/openmetadata-ui/src/main/resources/ui/src/mocks/TableData.mock.ts index 9a5a69aa60d..29181307baf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/mocks/TableData.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/mocks/TableData.mock.ts @@ -201,3 +201,71 @@ export const MOCK_TABLE = { ], deleted: false, } as unknown as Table; + +export const TEST_CASE = { + data: [ + { + id: 'b9d059d8-b968-42ad-9f89-2b40b92a6659', + name: 'column_value_max_to_be_between', + fullyQualifiedName: + 'sample_data.ecommerce_db.shopify.dim_address.shop_id.column_value_max_to_be_between', + description: 'test the value of a column is between x and z, new value', + testDefinition: { + id: '16b32e12-21c5-491c-919e-88748d9d5d67', + type: 'testDefinition', + name: 'columnValueMaxToBeBetween', + fullyQualifiedName: 'columnValueMaxToBeBetween', + description: + 'This schema defines the test ColumnValueMaxToBeBetween. Test the maximum value in a col is within a range.', + displayName: 'columnValueMaxToBeBetween', + deleted: false, + href: 'http://localhost:8585/api/v1/testDefinition/16b32e12-21c5-491c-919e-88748d9d5d67', + }, + entityLink: + '<#E::table::sample_data.ecommerce_db.shopify.dim_address::columns::shop_id>', + entityFQN: 'sample_data.ecommerce_db.shopify.dim_address.shop_id', + parameterValues: [ + { + name: 'minValueForMaxInCol', + value: '40', + }, + { + name: 'maxValueForMaxInCol', + value: '100', + }, + ], + testCaseResult: { + timestamp: 1661416859, + testCaseStatus: 'Success', + result: 'Found max=65 vs. the expected min=50, max=100.', + testResultValue: [ + { + name: 'max', + value: '65', + }, + ], + }, + version: 0.3, + updatedAt: 1661425991294, + updatedBy: 'anonymous', + href: 'http://localhost:8585/api/v1/testCase/b9d059d8-b968-42ad-9f89-2b40b92a6659', + changeDescription: { + fieldsAdded: [], + fieldsUpdated: [ + { + name: 'description', + oldValue: 'test the value of a column is between x and y', + newValue: + 'test the value of a column is between x and z, new value', + }, + ], + fieldsDeleted: [], + previousVersion: 0.2, + }, + deleted: false, + }, + ], + paging: { + total: 1, + }, +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx index d7987ef5692..efe16306d3f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx @@ -21,32 +21,40 @@ import { getTableDetailsByFQN, patchTableDetails, } from '../../axiosAPIs/tableAPI'; +import { getListTestCase } from '../../axiosAPIs/testAPI'; import ErrorPlaceHolder from '../../components/common/error-with-placeholder/ErrorPlaceHolder'; import PageContainerV1 from '../../components/containers/PageContainerV1'; import Loader from '../../components/Loader/Loader'; import ProfilerDashboard from '../../components/ProfilerDashboard/ProfilerDashboard'; import { API_RES_MAX_SIZE } from '../../constants/constants'; +import { ProfilerDashboardType } from '../../enums/table.enum'; import { ColumnProfile, Table } from '../../generated/entity/data/table'; +import { TestCase } from '../../generated/tests/testCase'; import jsonData from '../../jsons/en'; import { getNameFromFQN, getTableFQNFromColumnFQN, } from '../../utils/CommonUtils'; +import { generateEntityLink } from '../../utils/TableUtils'; import { showErrorToast } from '../../utils/ToastUtils'; const ProfilerDashboardPage = () => { - const { entityTypeFQN } = useParams>(); + const { entityTypeFQN, dashboardType } = useParams>(); + const isColumnView = dashboardType === ProfilerDashboardType.COLUMN; const [table, setTable] = useState({} as Table); const [profilerData, setProfilerData] = useState([]); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(false); + const [testCases, setTestCases] = useState([]); const fetchProfilerData = async (fqn: string, days = 3) => { try { const startTs = moment().subtract(days, 'days').unix(); + const endTs = moment().unix(); const { data } = await getColumnProfilerList(fqn, { - startTs: startTs, + startTs, + endTs, limit: API_RES_MAX_SIZE, }); setProfilerData(data || []); @@ -57,15 +65,39 @@ const ProfilerDashboardPage = () => { } }; - const fetchTableEntity = async (fqn: string) => { + const fetchTestCases = async (fqn: string) => { try { - getTableFQNFromColumnFQN(fqn); - const data = await getTableDetailsByFQN( - getTableFQNFromColumnFQN(fqn), - 'tags, usageSummary, owner, followers, profile' + const { data } = await getListTestCase({ + fields: 'testDefinition,testCaseResult', + entityLink: fqn, + limit: API_RES_MAX_SIZE, + }); + setTestCases(data); + } catch (error) { + showErrorToast( + error as AxiosError, + jsonData['api-error-messages']['fetch-column-test-error'] ); + } finally { + setIsLoading(false); + } + }; + + const fetchTableEntity = async () => { + try { + const fqn = isColumnView + ? getTableFQNFromColumnFQN(entityTypeFQN) + : entityTypeFQN; + const field = `tags, usageSummary, owner, followers${ + isColumnView ? ', profile' : '' + }`; + const data = await getTableDetailsByFQN(fqn, field); setTable(data ?? ({} as Table)); - fetchProfilerData(entityTypeFQN); + if (isColumnView) { + fetchProfilerData(entityTypeFQN); + } else { + fetchTestCases(generateEntityLink(entityTypeFQN)); + } } catch (error) { showErrorToast( error as AxiosError, @@ -92,7 +124,7 @@ const ProfilerDashboardPage = () => { useEffect(() => { if (entityTypeFQN) { - fetchTableEntity(entityTypeFQN); + fetchTableEntity(); } else { setIsLoading(false); setError(true); @@ -118,8 +150,10 @@ const ProfilerDashboardPage = () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DataQualityAndProfilerUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DataQualityAndProfilerUtils.ts new file mode 100644 index 00000000000..34401aa29b5 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DataQualityAndProfilerUtils.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TableTestsType } from '../components/TableProfiler/TableProfiler.interface'; +import { TestCaseStatus } from '../generated/tests/tableTest'; + +export const updateTestResults = ( + results: TableTestsType['results'], + testCaseStatus: string +) => { + switch (testCaseStatus) { + case TestCaseStatus.Success: + results.success += 1; + + break; + case TestCaseStatus.Failed: + results.failed += 1; + + break; + case TestCaseStatus.Aborted: + results.aborted += 1; + + break; + } +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts index f519104d4c4..7730702482a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts @@ -14,6 +14,7 @@ import { FQN_SEPARATOR_CHAR } from '../constants/char.constants'; import { IN_PAGE_SEARCH_ROUTES, + PLACEHOLDER_DASHBOARD_TYPE, PLACEHOLDER_ENTITY_TYPE_FQN, PLACEHOLDER_GLOSSARY_NAME, PLACEHOLDER_GLOSSARY_TERMS_FQN, @@ -289,10 +290,15 @@ export const getPath = (pathName: string) => { } }; -export const getProfilerDashboardWithFqnPath = (entityTypeFQN: string) => { +export const getProfilerDashboardWithFqnPath = ( + dashboardType: string, + entityTypeFQN: string +) => { let path = ROUTES.PROFILER_DASHBOARD; - path = path.replace(PLACEHOLDER_ENTITY_TYPE_FQN, entityTypeFQN); + path = path + .replace(PLACEHOLDER_DASHBOARD_TYPE, dashboardType) + .replace(PLACEHOLDER_ENTITY_TYPE_FQN, entityTypeFQN); return path; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx index 37cd3bc5ad5..294350ce0c7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx @@ -37,6 +37,7 @@ import { Column, DataType } from '../generated/entity/data/table'; import { TableTest, TestCaseStatus } from '../generated/tests/tableTest'; import { TagLabel } from '../generated/type/tagLabel'; import { ModifiedTableColumn } from '../interface/dataQuality.interface'; +import { getNameFromFQN, getTableFQNFromColumnFQN } from './CommonUtils'; import { getGlossaryPath } from './RouterUtils'; import { ordinalize } from './StringsUtils'; import SVGIcons, { Icons } from './SvgUtils'; @@ -306,6 +307,38 @@ export const getDataTypeString = (dataType: string): string => { } }; +export const generateEntityLink = (fqn: string, includeColumn = false) => { + const columnLink = '<#E::table::ENTITY_FQN::columns::COLUMN>'; + const tableLink = '<#E::table::ENTITY_FQN>'; + + if (includeColumn) { + const tableFqn = getTableFQNFromColumnFQN(fqn); + const columnName = getNameFromFQN(fqn); + + return columnLink + .replace('ENTITY_FQN', tableFqn) + .replace('COLUMN', columnName); + } else { + return tableLink.replace('ENTITY_FQN', fqn); + } +}; + +export const getTestResultBadgeIcon = (status?: TestCaseStatus) => { + switch (status) { + case TestCaseStatus.Success: + return Icons.SUCCESS_BADGE; + + case TestCaseStatus.Failed: + return Icons.FAIL_BADGE; + + case TestCaseStatus.Aborted: + return Icons.PENDING_BADGE; + + default: + return ''; + } +}; + export const getTableTestsValue = (tableTestCase: TableTest[]) => { const tableTestLength = tableTestCase.length;