mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-11-09 07:23:39 +00:00
feat(ui): Refactor Table Profiler to enhance test case summary handling and improve data quality metrics display
This commit is contained in:
parent
77065638a8
commit
f062908e56
@ -11,8 +11,8 @@
|
|||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Grid, Stack } from '@mui/material';
|
import { Grid, Stack, Typography, useTheme } from '@mui/material';
|
||||||
import { Button, Col, Row, Typography } from 'antd';
|
import { Button, Col, Row } from 'antd';
|
||||||
import { ColumnsType } from 'antd/lib/table';
|
import { ColumnsType } from 'antd/lib/table';
|
||||||
import classNames from 'classnames';
|
import classNames from 'classnames';
|
||||||
import {
|
import {
|
||||||
@ -20,7 +20,6 @@ import {
|
|||||||
find,
|
find,
|
||||||
groupBy,
|
groupBy,
|
||||||
isEmpty,
|
isEmpty,
|
||||||
isEqual,
|
|
||||||
isUndefined,
|
isUndefined,
|
||||||
map,
|
map,
|
||||||
round,
|
round,
|
||||||
@ -56,7 +55,6 @@ import {
|
|||||||
getTableFQNFromColumnFQN,
|
getTableFQNFromColumnFQN,
|
||||||
} from '../../../../../utils/CommonUtils';
|
} from '../../../../../utils/CommonUtils';
|
||||||
import { getEntityName } from '../../../../../utils/EntityUtils';
|
import { getEntityName } from '../../../../../utils/EntityUtils';
|
||||||
import { getEntityColumnFQN } from '../../../../../utils/FeedUtils';
|
|
||||||
import {
|
import {
|
||||||
generateEntityLink,
|
generateEntityLink,
|
||||||
getTableExpandableConfig,
|
getTableExpandableConfig,
|
||||||
@ -69,7 +67,6 @@ import { SummaryCard } from '../../../../common/SummaryCard/SummaryCard.componen
|
|||||||
import { SummaryCardProps } from '../../../../common/SummaryCard/SummaryCard.interface';
|
import { SummaryCardProps } from '../../../../common/SummaryCard/SummaryCard.interface';
|
||||||
import SummaryCardV1 from '../../../../common/SummaryCard/SummaryCardV1';
|
import SummaryCardV1 from '../../../../common/SummaryCard/SummaryCardV1';
|
||||||
import Table from '../../../../common/Table/Table';
|
import Table from '../../../../common/Table/Table';
|
||||||
import TestCaseStatusSummaryIndicator from '../../../../common/TestCaseStatusSummaryIndicator/TestCaseStatusSummaryIndicator.component';
|
|
||||||
import { TableProfilerTab } from '../../ProfilerDashboard/profilerDashboard.interface';
|
import { TableProfilerTab } from '../../ProfilerDashboard/profilerDashboard.interface';
|
||||||
import ColumnSummary from '../ColumnSummary';
|
import ColumnSummary from '../ColumnSummary';
|
||||||
import NoProfilerBanner from '../NoProfilerBanner/NoProfilerBanner.component';
|
import NoProfilerBanner from '../NoProfilerBanner/NoProfilerBanner.component';
|
||||||
@ -79,6 +76,7 @@ import { useTableProfiler } from '../TableProfilerProvider';
|
|||||||
|
|
||||||
const ColumnProfileTable = () => {
|
const ColumnProfileTable = () => {
|
||||||
const location = useCustomLocation();
|
const location = useCustomLocation();
|
||||||
|
const theme = useTheme();
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
const { fqn } = useFqn();
|
const { fqn } = useFqn();
|
||||||
@ -94,10 +92,6 @@ const ColumnProfileTable = () => {
|
|||||||
testCaseSummary,
|
testCaseSummary,
|
||||||
} = useTableProfiler();
|
} = useTableProfiler();
|
||||||
|
|
||||||
const testCaseCounts = useMemo(
|
|
||||||
() => testCaseSummary?.columnTestSummary ?? [],
|
|
||||||
[testCaseSummary]
|
|
||||||
);
|
|
||||||
const isLoading = isTestsLoading || isProfilerDataLoading;
|
const isLoading = isTestsLoading || isProfilerDataLoading;
|
||||||
const [searchText, setSearchText] = useState<string>('');
|
const [searchText, setSearchText] = useState<string>('');
|
||||||
const [data, setData] = useState<ModifiedColumn[]>([]);
|
const [data, setData] = useState<ModifiedColumn[]>([]);
|
||||||
@ -163,9 +157,9 @@ const ColumnProfileTable = () => {
|
|||||||
width: 150,
|
width: 150,
|
||||||
render: (dataTypeDisplay: string) => {
|
render: (dataTypeDisplay: string) => {
|
||||||
return (
|
return (
|
||||||
<Typography.Text className="break-word">
|
<Typography className="break-word">
|
||||||
{dataTypeDisplay || 'N/A'}
|
{dataTypeDisplay || 'N/A'}
|
||||||
</Typography.Text>
|
</Typography>
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
sorter: (col1, col2) => col1.dataType.localeCompare(col2.dataType),
|
sorter: (col1, col2) => col1.dataType.localeCompare(col2.dataType),
|
||||||
@ -221,51 +215,76 @@ const ColumnProfileTable = () => {
|
|||||||
(col1.profile?.valuesCount || 0) - (col2.profile?.valuesCount || 0),
|
(col1.profile?.valuesCount || 0) - (col2.profile?.valuesCount || 0),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: t('label.test-plural'),
|
title: t('label.success'),
|
||||||
dataIndex: 'testCount',
|
dataIndex: 'success',
|
||||||
key: 'Tests',
|
key: 'success',
|
||||||
width: 100,
|
width: 100,
|
||||||
render: (_, record) => {
|
render: (_, record) => {
|
||||||
const testCounts = testCaseCounts.find((column) => {
|
const testCounts = testCaseSummary?.[record.fullyQualifiedName ?? ''];
|
||||||
return isEqual(
|
|
||||||
getEntityColumnFQN(column.entityLink ?? ''),
|
|
||||||
record.fullyQualifiedName
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Link
|
<Link
|
||||||
data-testid={`${record.name}-test-count`}
|
data-testid={`${record.name}-test-success-count`}
|
||||||
to={{
|
to={{
|
||||||
search: Qs.stringify({
|
search: Qs.stringify({
|
||||||
activeTab: TableProfilerTab.DATA_QUALITY,
|
activeTab: TableProfilerTab.DATA_QUALITY,
|
||||||
}),
|
}),
|
||||||
}}>
|
}}>
|
||||||
{testCounts?.total ?? 0}
|
<Typography sx={{ color: theme.palette.success.main }}>
|
||||||
|
{testCounts?.success ?? 0}
|
||||||
|
</Typography>
|
||||||
</Link>
|
</Link>
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: t('label.status'),
|
title: t('label.failed'),
|
||||||
dataIndex: 'dataQualityTest',
|
dataIndex: 'failed',
|
||||||
key: 'dataQualityTest',
|
key: 'failed',
|
||||||
width: 150,
|
width: 100,
|
||||||
render: (_, record) => {
|
render: (_, record) => {
|
||||||
const testCounts = testCaseCounts.find((column) => {
|
const testCounts = testCaseSummary?.[record.fullyQualifiedName ?? ''];
|
||||||
return isEqual(
|
|
||||||
getEntityColumnFQN(column.entityLink ?? ''),
|
|
||||||
record.fullyQualifiedName
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<TestCaseStatusSummaryIndicator testCaseStatusCounts={testCounts} />
|
<Link
|
||||||
|
data-testid={`${record.name}-test-failed-count`}
|
||||||
|
to={{
|
||||||
|
search: Qs.stringify({
|
||||||
|
activeTab: TableProfilerTab.DATA_QUALITY,
|
||||||
|
}),
|
||||||
|
}}>
|
||||||
|
<Typography sx={{ color: theme.palette.error.main }}>
|
||||||
|
{testCounts?.failed ?? 0}
|
||||||
|
</Typography>
|
||||||
|
</Link>
|
||||||
|
);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: t('label.aborted'),
|
||||||
|
dataIndex: 'aborted',
|
||||||
|
key: 'aborted',
|
||||||
|
width: 100,
|
||||||
|
render: (_, record) => {
|
||||||
|
const testCounts = testCaseSummary?.[record.fullyQualifiedName ?? ''];
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Link
|
||||||
|
data-testid={`${record.name}-test-aborted-count`}
|
||||||
|
to={{
|
||||||
|
search: Qs.stringify({
|
||||||
|
activeTab: TableProfilerTab.DATA_QUALITY,
|
||||||
|
}),
|
||||||
|
}}>
|
||||||
|
<Typography sx={{ color: theme.palette.warning.main }}>
|
||||||
|
{testCounts?.aborted ?? 0}
|
||||||
|
</Typography>
|
||||||
|
</Link>
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}, [testCaseCounts]);
|
}, [testCaseSummary]);
|
||||||
|
|
||||||
const selectedColumn = useMemo(() => {
|
const selectedColumn = useMemo(() => {
|
||||||
return find(
|
return find(
|
||||||
|
|||||||
@ -22,9 +22,10 @@ import {
|
|||||||
Table,
|
Table,
|
||||||
TableProfilerConfig,
|
TableProfilerConfig,
|
||||||
} from '../../../../generated/entity/data/table';
|
} from '../../../../generated/entity/data/table';
|
||||||
import { TestCase, TestSummary } from '../../../../generated/tests/testCase';
|
import { TestCase } from '../../../../generated/tests/testCase';
|
||||||
import { UsePagingInterface } from '../../../../hooks/paging/usePaging';
|
import { UsePagingInterface } from '../../../../hooks/paging/usePaging';
|
||||||
import { ListTestCaseParamsBySearch } from '../../../../rest/testAPI';
|
import { ListTestCaseParamsBySearch } from '../../../../rest/testAPI';
|
||||||
|
import { TestCaseCountByStatus } from '../../../../utils/DataQuality/DataQualityUtils';
|
||||||
import { TestLevel } from '../../../DataQuality/AddDataQualityTest/components/TestCaseFormV1.interface';
|
import { TestLevel } from '../../../DataQuality/AddDataQualityTest/components/TestCaseFormV1.interface';
|
||||||
|
|
||||||
export interface TableProfilerProps {
|
export interface TableProfilerProps {
|
||||||
@ -38,7 +39,7 @@ export interface TableProfilerProviderProps extends TableProfilerProps {
|
|||||||
|
|
||||||
export interface TableProfilerContextInterface {
|
export interface TableProfilerContextInterface {
|
||||||
isTableDeleted?: boolean;
|
isTableDeleted?: boolean;
|
||||||
testCaseSummary?: TestSummary;
|
testCaseSummary?: Record<string, TestCaseCountByStatus>;
|
||||||
permissions: OperationPermission;
|
permissions: OperationPermission;
|
||||||
isTestsLoading: boolean;
|
isTestsLoading: boolean;
|
||||||
isProfilerDataLoading: boolean;
|
isProfilerDataLoading: boolean;
|
||||||
|
|||||||
@ -37,7 +37,7 @@ import { useTourProvider } from '../../../../context/TourProvider/TourProvider';
|
|||||||
import { TabSpecificField } from '../../../../enums/entity.enum';
|
import { TabSpecificField } from '../../../../enums/entity.enum';
|
||||||
import { Table } from '../../../../generated/entity/data/table';
|
import { Table } from '../../../../generated/entity/data/table';
|
||||||
import { ProfileSampleType } from '../../../../generated/metadataIngestion/databaseServiceProfilerPipeline';
|
import { ProfileSampleType } from '../../../../generated/metadataIngestion/databaseServiceProfilerPipeline';
|
||||||
import { TestCase, TestSummary } from '../../../../generated/tests/testCase';
|
import { TestCase } from '../../../../generated/tests/testCase';
|
||||||
import { Include } from '../../../../generated/type/include';
|
import { Include } from '../../../../generated/type/include';
|
||||||
import { usePaging } from '../../../../hooks/paging/usePaging';
|
import { usePaging } from '../../../../hooks/paging/usePaging';
|
||||||
import useCustomLocation from '../../../../hooks/useCustomLocation/useCustomLocation';
|
import useCustomLocation from '../../../../hooks/useCustomLocation/useCustomLocation';
|
||||||
@ -49,10 +49,12 @@ import {
|
|||||||
} from '../../../../rest/tableAPI';
|
} from '../../../../rest/tableAPI';
|
||||||
import {
|
import {
|
||||||
getListTestCaseBySearch,
|
getListTestCaseBySearch,
|
||||||
getTestCaseExecutionSummary,
|
|
||||||
ListTestCaseParamsBySearch,
|
ListTestCaseParamsBySearch,
|
||||||
} from '../../../../rest/testAPI';
|
} from '../../../../rest/testAPI';
|
||||||
import { aggregateTestResultsByEntity } from '../../../../utils/DataQuality/DataQualityUtils';
|
import {
|
||||||
|
aggregateTestResultsByEntity,
|
||||||
|
TestCaseCountByStatus,
|
||||||
|
} from '../../../../utils/DataQuality/DataQualityUtils';
|
||||||
import { bytesToSize } from '../../../../utils/StringsUtils';
|
import { bytesToSize } from '../../../../utils/StringsUtils';
|
||||||
import { generateEntityLink } from '../../../../utils/TableUtils';
|
import { generateEntityLink } from '../../../../utils/TableUtils';
|
||||||
import { showErrorToast } from '../../../../utils/ToastUtils';
|
import { showErrorToast } from '../../../../utils/ToastUtils';
|
||||||
@ -94,7 +96,8 @@ export const TableProfilerProvider = ({
|
|||||||
const [isTestCaseDrawerOpen, setIsTestCaseDrawerOpen] = useState(false);
|
const [isTestCaseDrawerOpen, setIsTestCaseDrawerOpen] = useState(false);
|
||||||
const [testLevel, setTestLevel] = useState<TestLevel>();
|
const [testLevel, setTestLevel] = useState<TestLevel>();
|
||||||
const [table, setTable] = useState<Table | undefined>(tableEntity);
|
const [table, setTable] = useState<Table | undefined>(tableEntity);
|
||||||
const [testCaseSummary, setTestCaseSummary] = useState<TestSummary>();
|
const [testCaseSummary, setTestCaseSummary] =
|
||||||
|
useState<Record<string, TestCaseCountByStatus>>();
|
||||||
|
|
||||||
const isTableDeleted = useMemo(() => table?.deleted, [table]);
|
const isTableDeleted = useMemo(() => table?.deleted, [table]);
|
||||||
|
|
||||||
@ -331,7 +334,6 @@ export const TableProfilerProvider = ({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const response = await getTestCaseExecutionSummary(table.testSuite.id);
|
|
||||||
const { data } = await fetchTestCaseResultByTestSuiteId(
|
const { data } = await fetchTestCaseResultByTestSuiteId(
|
||||||
table.testSuite.id
|
table.testSuite.id
|
||||||
);
|
);
|
||||||
@ -342,8 +344,7 @@ export const TableProfilerProvider = ({
|
|||||||
'testCaseResult.testCaseStatus': string;
|
'testCaseResult.testCaseStatus': string;
|
||||||
}>
|
}>
|
||||||
);
|
);
|
||||||
console.log(testCaseResults);
|
setTestCaseSummary(testCaseResults);
|
||||||
setTestCaseSummary(response);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
setTestCaseSummary(undefined);
|
setTestCaseSummary(undefined);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -440,13 +440,20 @@ export const CustomDQTooltip = (props: DataInsightChartTooltipProps) => {
|
|||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type TestCaseCountByStatus = {
|
||||||
|
success: number;
|
||||||
|
failed: number;
|
||||||
|
aborted: number;
|
||||||
|
total: number;
|
||||||
|
};
|
||||||
|
|
||||||
export const aggregateTestResultsByEntity = (
|
export const aggregateTestResultsByEntity = (
|
||||||
data: Array<{
|
data: Array<{
|
||||||
document_count: string;
|
document_count: string;
|
||||||
entityFQN: string;
|
entityFQN: string;
|
||||||
'testCaseResult.testCaseStatus': string;
|
'testCaseResult.testCaseStatus': string;
|
||||||
}>
|
}>
|
||||||
) => {
|
): Record<string, TestCaseCountByStatus> => {
|
||||||
const overallTotal = {
|
const overallTotal = {
|
||||||
failed: 0,
|
failed: 0,
|
||||||
success: 0,
|
success: 0,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user