mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-24 08:58:06 +00:00
added data-quality tab and filters for test, with basic table (#6869)
* added data-quality tab and filters for test, with basic table * added pass fail status with icon * added test summary component * render the chart for column test * revers the time stamp * fix right side data render layout * added legend and colors * added filter by time * updated style and move filter to chart level * updated column profiler route * added table dashboard * updated test result summary on profiler tab * fixed failing unit test * added loader
This commit is contained in:
Shailesh Parmar
GitHub
parent
4c7fd03b5a
commit
ef5de94a8f
@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { TestCase, TestCaseResult } from '../generated/tests/testCase';
|
||||
import { Include } from '../generated/type/include';
|
||||
import { Paging } from '../generated/type/paging';
|
||||
import APIClient from './index';
|
||||
|
||||
export type ListTestCaseParams = {
|
||||
fields?: string;
|
||||
limit?: number;
|
||||
before?: string;
|
||||
after?: string;
|
||||
entityLink?: string;
|
||||
testSuiteId?: string;
|
||||
includeAllTests?: boolean;
|
||||
include?: Include;
|
||||
};
|
||||
|
||||
export type ListTestCaseResultsParams = {
|
||||
startTs?: number;
|
||||
endTs?: number;
|
||||
before?: string;
|
||||
after?: string;
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
const baseUrl = '/testCase';
|
||||
|
||||
export const getListTestCase = async (params?: ListTestCaseParams) => {
|
||||
const response = await APIClient.get<{ data: TestCase[]; paging: Paging }>(
|
||||
baseUrl,
|
||||
{
|
||||
params,
|
||||
}
|
||||
);
|
||||
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const getListTestCaseResults = async (
|
||||
fqn: string,
|
||||
params?: ListTestCaseResultsParams
|
||||
) => {
|
||||
const url = `${baseUrl}/${fqn}/testCaseResult`;
|
||||
const response = await APIClient.get<{
|
||||
data: TestCaseResult[];
|
||||
paging: Paging;
|
||||
}>(url, {
|
||||
params,
|
||||
});
|
||||
|
||||
return response.data;
|
||||
};
|
||||
@ -10,7 +10,6 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Button, Col, Radio, Row, Select, Space } from 'antd';
|
||||
import { RadioChangeEvent } from 'antd/lib/radio';
|
||||
import { AxiosError } from 'axios';
|
||||
@ -29,8 +28,13 @@ import {
|
||||
import { PROFILER_FILTER_RANGE } from '../../constants/profiler.constant';
|
||||
import { EntityType, FqnPart } from '../../enums/entity.enum';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import { ProfilerDashboardType } from '../../enums/table.enum';
|
||||
import { OwnerType } from '../../enums/user.enum';
|
||||
import { Column, Table } from '../../generated/entity/data/table';
|
||||
import {
|
||||
Column,
|
||||
Table,
|
||||
TestCaseStatus,
|
||||
} from '../../generated/entity/data/table';
|
||||
import { EntityReference } from '../../generated/type/entityReference';
|
||||
import { LabelType, State } from '../../generated/type/tagLabel';
|
||||
import jsonData from '../../jsons/en';
|
||||
@ -44,6 +48,7 @@ import {
|
||||
} from '../../utils/CommonUtils';
|
||||
import { serviceTypeLogo } from '../../utils/ServiceUtils';
|
||||
import {
|
||||
generateEntityLink,
|
||||
getTagsWithoutTier,
|
||||
getTierTags,
|
||||
getUsagePercentile,
|
||||
@ -51,6 +56,7 @@ import {
|
||||
import { showErrorToast } from '../../utils/ToastUtils';
|
||||
import EntityPageInfo from '../common/entityPageInfo/EntityPageInfo';
|
||||
import PageLayout from '../containers/PageLayout';
|
||||
import DataQualityTab from './component/DataQualityTab';
|
||||
import ProfilerTab from './component/ProfilerTab';
|
||||
import {
|
||||
ProfilerDashboardProps,
|
||||
@ -60,23 +66,40 @@ import './profilerDashboard.less';
|
||||
|
||||
const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
||||
table,
|
||||
testCases,
|
||||
fetchProfilerData,
|
||||
fetchTestCases,
|
||||
profilerData,
|
||||
onTableChange,
|
||||
}) => {
|
||||
const history = useHistory();
|
||||
const { entityTypeFQN } = useParams<Record<string, string>>();
|
||||
const { entityTypeFQN, dashboardType } = useParams<Record<string, string>>();
|
||||
const isColumnView = dashboardType === ProfilerDashboardType.COLUMN;
|
||||
const [follower, setFollower] = useState<EntityReference[]>([]);
|
||||
const [isFollowing, setIsFollowing] = useState<boolean>(false);
|
||||
const [activeTab, setActiveTab] = useState<ProfilerDashboardTab>(
|
||||
ProfilerDashboardTab.PROFILER
|
||||
isColumnView
|
||||
? ProfilerDashboardTab.PROFILER
|
||||
: ProfilerDashboardTab.DATA_QUALITY
|
||||
);
|
||||
const [selectedTestCaseStatus, setSelectedTestCaseStatus] =
|
||||
useState<string>('');
|
||||
const [selectedTimeRange, setSelectedTimeRange] =
|
||||
useState<keyof typeof PROFILER_FILTER_RANGE>('last3days');
|
||||
const [activeColumnDetails, setActiveColumnDetails] = useState<Column>(
|
||||
{} as Column
|
||||
);
|
||||
|
||||
const tabOptions = useMemo(() => {
|
||||
return Object.values(ProfilerDashboardTab).filter((value) => {
|
||||
if (value === ProfilerDashboardTab.PROFILER) {
|
||||
return isColumnView;
|
||||
}
|
||||
|
||||
return value;
|
||||
});
|
||||
}, [dashboardType]);
|
||||
|
||||
const timeRangeOption = useMemo(() => {
|
||||
return Object.entries(PROFILER_FILTER_RANGE).map(([key, value]) => ({
|
||||
label: value.title,
|
||||
@ -84,6 +107,21 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const testCaseStatusOption = useMemo(() => {
|
||||
const testCaseStatus: Record<string, string>[] = Object.values(
|
||||
TestCaseStatus
|
||||
).map((value) => ({
|
||||
label: value,
|
||||
value: value,
|
||||
}));
|
||||
testCaseStatus.unshift({
|
||||
label: 'All Test',
|
||||
value: '',
|
||||
});
|
||||
|
||||
return testCaseStatus;
|
||||
}, []);
|
||||
|
||||
const tier = useMemo(() => getTierTags(table.tags ?? []), [table]);
|
||||
const breadcrumb = useMemo(() => {
|
||||
const serviceName = getEntityName(table.service);
|
||||
@ -251,7 +289,10 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
||||
const value = e.target.value as ProfilerDashboardTab;
|
||||
if (ProfilerDashboardTab.SUMMARY === value) {
|
||||
history.push(getTableTabPath(table.fullyQualifiedName || '', 'profiler'));
|
||||
} else if (ProfilerDashboardTab.DATA_QUALITY === value) {
|
||||
fetchTestCases(generateEntityLink(entityTypeFQN, true));
|
||||
}
|
||||
setSelectedTestCaseStatus('');
|
||||
setActiveTab(value);
|
||||
};
|
||||
|
||||
@ -264,17 +305,35 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
||||
const handleTimeRangeChange = (value: keyof typeof PROFILER_FILTER_RANGE) => {
|
||||
if (value !== selectedTimeRange) {
|
||||
setSelectedTimeRange(value);
|
||||
fetchProfilerData(entityTypeFQN, PROFILER_FILTER_RANGE[value].days);
|
||||
if (activeTab === ProfilerDashboardTab.PROFILER) {
|
||||
fetchProfilerData(entityTypeFQN, PROFILER_FILTER_RANGE[value].days);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const handleTestCaseStatusChange = (value: string) => {
|
||||
if (value !== selectedTestCaseStatus) {
|
||||
setSelectedTestCaseStatus(value);
|
||||
}
|
||||
};
|
||||
|
||||
const getFilterTestCase = () => {
|
||||
return testCases.filter(
|
||||
(data) =>
|
||||
selectedTestCaseStatus === '' ||
|
||||
data.testCaseResult?.testCaseStatus === selectedTestCaseStatus
|
||||
);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (table) {
|
||||
const columnName = getNameFromFQN(entityTypeFQN);
|
||||
const selectedColumn = table.columns.find(
|
||||
(col) => col.name === columnName
|
||||
);
|
||||
setActiveColumnDetails(selectedColumn || ({} as Column));
|
||||
if (isColumnView) {
|
||||
const columnName = getNameFromFQN(entityTypeFQN);
|
||||
const selectedColumn = table.columns.find(
|
||||
(col) => col.name === columnName
|
||||
);
|
||||
setActiveColumnDetails(selectedColumn || ({} as Column));
|
||||
}
|
||||
setFollower(table?.followers || []);
|
||||
setIsFollowing(
|
||||
follower.some(({ id }: { id: string }) => id === getCurrentUserId())
|
||||
@ -315,18 +374,28 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
||||
<Radio.Group
|
||||
buttonStyle="solid"
|
||||
optionType="button"
|
||||
options={Object.values(ProfilerDashboardTab)}
|
||||
options={tabOptions}
|
||||
value={activeTab}
|
||||
onChange={handleTabChange}
|
||||
/>
|
||||
|
||||
<Space size={16}>
|
||||
<Select
|
||||
className="tw-w-32"
|
||||
options={timeRangeOption}
|
||||
value={selectedTimeRange}
|
||||
onChange={handleTimeRangeChange}
|
||||
/>
|
||||
{activeTab === ProfilerDashboardTab.DATA_QUALITY && (
|
||||
<Select
|
||||
className="tw-w-32"
|
||||
options={testCaseStatusOption}
|
||||
value={selectedTestCaseStatus}
|
||||
onChange={handleTestCaseStatusChange}
|
||||
/>
|
||||
)}
|
||||
{activeTab === ProfilerDashboardTab.PROFILER && (
|
||||
<Select
|
||||
className="tw-w-32"
|
||||
options={timeRangeOption}
|
||||
value={selectedTimeRange}
|
||||
onChange={handleTimeRangeChange}
|
||||
/>
|
||||
)}
|
||||
<Button type="primary" onClick={handleAddTestClick}>
|
||||
Add Test
|
||||
</Button>
|
||||
@ -344,7 +413,9 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
||||
)}
|
||||
|
||||
{activeTab === ProfilerDashboardTab.DATA_QUALITY && (
|
||||
<Col span={24}>Data Quality</Col>
|
||||
<Col span={24}>
|
||||
<DataQualityTab testCases={getFilterTestCase()} />
|
||||
</Col>
|
||||
)}
|
||||
</Row>
|
||||
</PageLayout>
|
||||
|
||||
@ -0,0 +1,120 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Button, Space, Table, Tooltip } from 'antd';
|
||||
import { ColumnsType } from 'antd/lib/table';
|
||||
import { isUndefined } from 'lodash';
|
||||
import moment from 'moment';
|
||||
import React, { useMemo, useState } from 'react';
|
||||
import { TestCase, TestCaseResult } from '../../../generated/tests/testCase';
|
||||
import SVGIcons, { Icons } from '../../../utils/SvgUtils';
|
||||
import { getTestResultBadgeIcon } from '../../../utils/TableUtils';
|
||||
import DeleteWidgetModal from '../../common/DeleteWidget/DeleteWidgetModal';
|
||||
import { DataQualityTabProps } from '../profilerDashboard.interface';
|
||||
import TestSummary from './TestSummary';
|
||||
|
||||
const DataQualityTab: React.FC<DataQualityTabProps> = ({ testCases }) => {
|
||||
const [selectedTestCase, setSelectedTestCase] = useState<TestCase>();
|
||||
const columns: ColumnsType<TestCase> = useMemo(() => {
|
||||
return [
|
||||
{
|
||||
title: 'Last Run Result',
|
||||
dataIndex: 'testCaseResult',
|
||||
key: 'testCaseResult',
|
||||
render: (result: TestCaseResult) => (
|
||||
<Space size={8}>
|
||||
{result?.testCaseStatus && (
|
||||
<SVGIcons
|
||||
alt="result"
|
||||
className="tw-w-4"
|
||||
icon={getTestResultBadgeIcon(result.testCaseStatus)}
|
||||
/>
|
||||
)}
|
||||
<span>{result?.testCaseStatus || '--'}</span>
|
||||
</Space>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: 'Last Run',
|
||||
dataIndex: 'testCaseResult',
|
||||
key: 'lastRun',
|
||||
render: (result: TestCaseResult) =>
|
||||
result?.timestamp
|
||||
? moment.unix(result.timestamp || 0).format('DD/MMM HH:mm')
|
||||
: '--',
|
||||
},
|
||||
{
|
||||
title: 'Name',
|
||||
dataIndex: 'name',
|
||||
key: 'name',
|
||||
},
|
||||
{
|
||||
title: 'Description',
|
||||
dataIndex: 'description',
|
||||
key: 'description',
|
||||
},
|
||||
{
|
||||
title: 'Actions',
|
||||
dataIndex: 'actions',
|
||||
key: 'actions',
|
||||
render: (_, record) => (
|
||||
<Space>
|
||||
<Tooltip placement="bottom" title="Delete">
|
||||
<Button
|
||||
icon={
|
||||
<SVGIcons
|
||||
alt="Delete"
|
||||
className="tw-w-4"
|
||||
icon={Icons.DELETE}
|
||||
/>
|
||||
}
|
||||
type="text"
|
||||
onClick={() => {
|
||||
setSelectedTestCase(record);
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
</Space>
|
||||
),
|
||||
},
|
||||
];
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Table
|
||||
columns={columns}
|
||||
dataSource={testCases.map((test) => ({ ...test, key: test.name }))}
|
||||
expandable={{
|
||||
rowExpandable: () => true,
|
||||
expandedRowRender: (recode) => <TestSummary data={recode} />,
|
||||
}}
|
||||
pagination={false}
|
||||
size="small"
|
||||
/>
|
||||
|
||||
<DeleteWidgetModal
|
||||
entityId={selectedTestCase?.id || ''}
|
||||
entityName={selectedTestCase?.name || ''}
|
||||
entityType="testCase"
|
||||
prepareType={false}
|
||||
visible={!isUndefined(selectedTestCase)}
|
||||
onCancel={() => {
|
||||
setSelectedTestCase(undefined);
|
||||
}}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default DataQualityTab;
|
||||
@ -12,6 +12,7 @@
|
||||
*/
|
||||
|
||||
import { Card, Col, Row, Statistic } from 'antd';
|
||||
import { sortBy } from 'lodash';
|
||||
import moment from 'moment';
|
||||
import React, { useEffect, useMemo, useState } from 'react';
|
||||
import {
|
||||
@ -82,11 +83,12 @@ const ProfilerTab: React.FC<ProfilerTabProps> = ({
|
||||
);
|
||||
|
||||
const createMetricsChartData = () => {
|
||||
const updateProfilerData = sortBy(profilerData, 'timestamp');
|
||||
const countMetricData: MetricChartType['data'] = [];
|
||||
const proportionMetricData: MetricChartType['data'] = [];
|
||||
const mathMetricData: MetricChartType['data'] = [];
|
||||
const sumMetricData: MetricChartType['data'] = [];
|
||||
profilerData.forEach((col) => {
|
||||
updateProfilerData.forEach((col) => {
|
||||
const x = moment.unix(col.timestamp || 0).format('DD/MMM HH:mm');
|
||||
|
||||
countMetricData.push({
|
||||
@ -124,21 +126,27 @@ const ProfilerTab: React.FC<ProfilerTabProps> = ({
|
||||
|
||||
const countMetricInfo = countMetrics.information.map((item) => ({
|
||||
...item,
|
||||
latestValue: countMetricData[0]?.[item.dataKey] || 0,
|
||||
latestValue:
|
||||
countMetricData[countMetricData.length - 1]?.[item.dataKey] || 0,
|
||||
}));
|
||||
const proportionMetricInfo = proportionMetrics.information.map((item) => ({
|
||||
...item,
|
||||
latestValue: parseFloat(
|
||||
`${proportionMetricData[0]?.[item.dataKey] || 0}`
|
||||
`${
|
||||
proportionMetricData[proportionMetricData.length - 1]?.[
|
||||
item.dataKey
|
||||
] || 0
|
||||
}`
|
||||
).toFixed(2),
|
||||
}));
|
||||
const mathMetricInfo = mathMetrics.information.map((item) => ({
|
||||
...item,
|
||||
latestValue: mathMetricData[0]?.[item.dataKey] || 0,
|
||||
latestValue:
|
||||
mathMetricData[mathMetricData.length - 1]?.[item.dataKey] || 0,
|
||||
}));
|
||||
const sumMetricInfo = sumMetrics.information.map((item) => ({
|
||||
...item,
|
||||
latestValue: sumMetricData[0]?.[item.dataKey] || 0,
|
||||
latestValue: sumMetricData[sumMetricData.length - 1]?.[item.dataKey] || 0,
|
||||
}));
|
||||
|
||||
setCountMetrics((pre) => ({
|
||||
@ -172,13 +180,13 @@ const ProfilerTab: React.FC<ProfilerTabProps> = ({
|
||||
<Col span={8}>
|
||||
<Card className="tw-rounded-md tw-border tw-h-full">
|
||||
<Row gutter={16}>
|
||||
<Col span={18}>
|
||||
<Col span={16}>
|
||||
<p className="tw-font-medium tw-text-base">Column summary</p>
|
||||
<Ellipses className="tw-text-grey-muted" rows={4}>
|
||||
{activeColumnDetails.description}
|
||||
{activeColumnDetails.description || 'No Description'}
|
||||
</Ellipses>
|
||||
</Col>
|
||||
<Col span={6}>
|
||||
<Col span={8}>
|
||||
<Statistic
|
||||
title="Data type"
|
||||
value={activeColumnDetails.dataTypeDisplay || ''}
|
||||
|
||||
@ -0,0 +1,237 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Col, Empty, Row, Select, Space, Typography } from 'antd';
|
||||
import { AxiosError } from 'axios';
|
||||
import { isEmpty } from 'lodash';
|
||||
import moment from 'moment';
|
||||
import React, { useEffect, useMemo, useState } from 'react';
|
||||
import {
|
||||
Legend,
|
||||
Line,
|
||||
LineChart,
|
||||
LineProps,
|
||||
ReferenceArea,
|
||||
ResponsiveContainer,
|
||||
Tooltip,
|
||||
XAxis,
|
||||
YAxis,
|
||||
} from 'recharts';
|
||||
import { getListTestCaseResults } from '../../../axiosAPIs/testAPI';
|
||||
import { API_RES_MAX_SIZE } from '../../../constants/constants';
|
||||
import {
|
||||
COLORS,
|
||||
PROFILER_FILTER_RANGE,
|
||||
} from '../../../constants/profiler.constant';
|
||||
import {
|
||||
TestCaseResult,
|
||||
TestCaseStatus,
|
||||
} from '../../../generated/tests/tableTest';
|
||||
import { showErrorToast } from '../../../utils/ToastUtils';
|
||||
import RichTextEditorPreviewer from '../../common/rich-text-editor/RichTextEditorPreviewer';
|
||||
import Loader from '../../Loader/Loader';
|
||||
import { TestSummaryProps } from '../profilerDashboard.interface';
|
||||
|
||||
type ChartDataType = {
|
||||
information: { label: string; color: string }[];
|
||||
data: { [key: string]: string }[];
|
||||
};
|
||||
|
||||
const TestSummary: React.FC<TestSummaryProps> = ({ data }) => {
|
||||
const [chartData, setChartData] = useState<ChartDataType>(
|
||||
{} as ChartDataType
|
||||
);
|
||||
const [results, setResults] = useState<TestCaseResult[]>([]);
|
||||
const [selectedTimeRange, setSelectedTimeRange] =
|
||||
useState<keyof typeof PROFILER_FILTER_RANGE>('last3days');
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
const timeRangeOption = useMemo(() => {
|
||||
return Object.entries(PROFILER_FILTER_RANGE).map(([key, value]) => ({
|
||||
label: value.title,
|
||||
value: key,
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const handleTimeRangeChange = (value: keyof typeof PROFILER_FILTER_RANGE) => {
|
||||
if (value !== selectedTimeRange) {
|
||||
setSelectedTimeRange(value);
|
||||
}
|
||||
};
|
||||
|
||||
const generateChartData = (currentData: TestCaseResult[]) => {
|
||||
const chartData: { [key: string]: string }[] = [];
|
||||
currentData.forEach((result) => {
|
||||
const values = result.testResultValue?.reduce((acc, curr) => {
|
||||
return {
|
||||
...acc,
|
||||
[curr.name || 'value']: parseInt(curr.value || '') || 0,
|
||||
};
|
||||
}, {});
|
||||
|
||||
chartData.push({
|
||||
name: moment.unix(result.timestamp || 0).format('DD/MMM HH:mm'),
|
||||
status: result.testCaseStatus || '',
|
||||
...values,
|
||||
});
|
||||
});
|
||||
setChartData({
|
||||
information:
|
||||
currentData[0]?.testResultValue?.map((info, i) => ({
|
||||
label: info.name || '',
|
||||
color: COLORS[i],
|
||||
})) || [],
|
||||
data: chartData.reverse(),
|
||||
});
|
||||
};
|
||||
|
||||
const updatedDot: LineProps['dot'] = (props) => {
|
||||
const { cx = 0, cy = 0, payload } = props;
|
||||
const fill =
|
||||
payload.status === TestCaseStatus.Success
|
||||
? '#28A745'
|
||||
: payload.status === TestCaseStatus.Failed
|
||||
? '#CB2431'
|
||||
: '#EFAE2F';
|
||||
|
||||
return (
|
||||
<svg
|
||||
fill="none"
|
||||
height={8}
|
||||
width={8}
|
||||
x={cx - 4}
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
y={cy - 4}>
|
||||
<circle cx={4} cy={4} fill={fill} r={4} />
|
||||
</svg>
|
||||
);
|
||||
};
|
||||
|
||||
const fetchTestResults = async () => {
|
||||
if (isEmpty(data)) return;
|
||||
|
||||
try {
|
||||
const startTs = moment()
|
||||
.subtract(PROFILER_FILTER_RANGE[selectedTimeRange].days, 'days')
|
||||
.unix();
|
||||
const endTs = moment().unix();
|
||||
const { data: chartData } = await getListTestCaseResults(
|
||||
data.fullyQualifiedName || '',
|
||||
{
|
||||
startTs,
|
||||
endTs,
|
||||
limit: API_RES_MAX_SIZE,
|
||||
}
|
||||
);
|
||||
setResults(chartData);
|
||||
generateChartData(chartData);
|
||||
} catch (error) {
|
||||
showErrorToast(error as AxiosError);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchTestResults();
|
||||
}, [selectedTimeRange]);
|
||||
|
||||
const referenceArea = () => {
|
||||
const yValues = data.parameterValues?.reduce((acc, curr, i) => {
|
||||
return { ...acc, [`y${i + 1}`]: parseInt(curr.value || '') };
|
||||
}, {});
|
||||
|
||||
return (
|
||||
<ReferenceArea
|
||||
fill="#28A74530"
|
||||
ifOverflow="extendDomain"
|
||||
stroke="#28A745"
|
||||
strokeDasharray="4"
|
||||
{...yValues}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<Row gutter={16}>
|
||||
<Col span={14}>
|
||||
{isLoading ? (
|
||||
<Loader />
|
||||
) : results.length ? (
|
||||
<div>
|
||||
<Space align="end" className="tw-w-full" direction="vertical">
|
||||
<Select
|
||||
className="tw-w-32 tw-mb-2"
|
||||
options={timeRangeOption}
|
||||
value={selectedTimeRange}
|
||||
onChange={handleTimeRangeChange}
|
||||
/>
|
||||
</Space>
|
||||
<ResponsiveContainer className="tw-bg-white" minHeight={300}>
|
||||
<LineChart
|
||||
data={chartData.data}
|
||||
margin={{
|
||||
top: 8,
|
||||
bottom: 8,
|
||||
right: 8,
|
||||
}}>
|
||||
<XAxis dataKey="name" padding={{ left: 8, right: 8 }} />
|
||||
<YAxis allowDataOverflow padding={{ top: 8, bottom: 8 }} />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
{data.parameterValues?.length === 2 && referenceArea()}
|
||||
{chartData?.information?.map((info, i) => (
|
||||
<Line
|
||||
dataKey={info.label}
|
||||
dot={updatedDot}
|
||||
key={i}
|
||||
stroke={info.color}
|
||||
type="monotone"
|
||||
/>
|
||||
))}
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
) : (
|
||||
<Empty description="No Result Available" />
|
||||
)}
|
||||
</Col>
|
||||
<Col span={10}>
|
||||
<Row gutter={[8, 8]}>
|
||||
<Col span={24}>
|
||||
<Typography.Text type="secondary">Name: </Typography.Text>
|
||||
<Typography.Text>{data.displayName || data.name}</Typography.Text>
|
||||
</Col>
|
||||
<Col span={24}>
|
||||
<Typography.Text type="secondary">Parameter: </Typography.Text>
|
||||
</Col>
|
||||
<Col offset={2} span={24}>
|
||||
{data.parameterValues?.map((param) => (
|
||||
<Typography key={param.name}>
|
||||
<Typography.Text>{param.name}: </Typography.Text>
|
||||
<Typography.Text>{param.value}</Typography.Text>
|
||||
</Typography>
|
||||
))}
|
||||
</Col>
|
||||
|
||||
<Col className="tw-flex tw-gap-2" span={24}>
|
||||
<Typography.Text type="secondary">Description: </Typography.Text>
|
||||
<RichTextEditorPreviewer markdown={data.description || ''} />
|
||||
</Col>
|
||||
</Row>
|
||||
</Col>
|
||||
</Row>
|
||||
);
|
||||
};
|
||||
|
||||
export default TestSummary;
|
||||
@ -16,12 +16,15 @@ import {
|
||||
ColumnProfile,
|
||||
Table,
|
||||
} from '../../generated/entity/data/table';
|
||||
import { TestCase } from '../../generated/tests/testCase';
|
||||
|
||||
export interface ProfilerDashboardProps {
|
||||
onTableChange: (table: Table) => void;
|
||||
table: Table;
|
||||
testCases: TestCase[];
|
||||
profilerData: ColumnProfile[];
|
||||
fetchProfilerData: (tableId: string, days?: number) => void;
|
||||
fetchTestCases: (fqn: string) => void;
|
||||
}
|
||||
|
||||
export type MetricChartType = {
|
||||
@ -73,3 +76,11 @@ export interface ProfilerSummaryCardProps {
|
||||
}[];
|
||||
showIndicator?: boolean;
|
||||
}
|
||||
|
||||
export interface DataQualityTabProps {
|
||||
testCases: TestCase[];
|
||||
}
|
||||
|
||||
export interface TestSummaryProps {
|
||||
data: TestCase;
|
||||
}
|
||||
|
||||
@ -101,6 +101,7 @@ jest.mock('../../../utils/DatasetDetailsUtils');
|
||||
const mockProps: ColumnProfileTableProps = {
|
||||
columns: MOCK_TABLE.columns,
|
||||
onAddTestClick: jest.fn,
|
||||
columnTests: [],
|
||||
};
|
||||
|
||||
describe('Test ColumnProfileTable component', () => {
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
|
||||
import { Button, Space, Table } from 'antd';
|
||||
import { ColumnsType } from 'antd/lib/table';
|
||||
import { isUndefined } from 'lodash';
|
||||
import React, { FC, useEffect, useMemo, useState } from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import {
|
||||
@ -20,38 +21,35 @@ import {
|
||||
SECONDARY_COLOR,
|
||||
SUCCESS_COLOR,
|
||||
} from '../../../constants/constants';
|
||||
import {
|
||||
DEFAULT_TEST_VALUE,
|
||||
INITIAL_TEST_RESULT_SUMMARY,
|
||||
} from '../../../constants/profiler.constant';
|
||||
import { ProfilerDashboardType } from '../../../enums/table.enum';
|
||||
import { Column, ColumnProfile } from '../../../generated/entity/data/table';
|
||||
import { TestCaseStatus } from '../../../generated/tests/tableTest';
|
||||
import { formatNumberWithComma } from '../../../utils/CommonUtils';
|
||||
import { updateTestResults } from '../../../utils/DataQualityAndProfilerUtils';
|
||||
import { getCurrentDatasetTab } from '../../../utils/DatasetDetailsUtils';
|
||||
import { getProfilerDashboardWithFqnPath } from '../../../utils/RouterUtils';
|
||||
import Ellipses from '../../common/Ellipses/Ellipses';
|
||||
import Searchbar from '../../common/searchbar/Searchbar';
|
||||
import TestIndicator from '../../common/TestIndicator/TestIndicator';
|
||||
import { ColumnProfileTableProps } from '../TableProfiler.interface';
|
||||
import {
|
||||
ColumnProfileTableProps,
|
||||
columnTestResultType,
|
||||
} from '../TableProfiler.interface';
|
||||
import ProfilerProgressWidget from './ProfilerProgressWidget';
|
||||
|
||||
const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
||||
columnTests,
|
||||
onAddTestClick,
|
||||
columns = [],
|
||||
}) => {
|
||||
const [searchText, setSearchText] = useState<string>('');
|
||||
const [data, setData] = useState<Column[]>(columns);
|
||||
// TODO:- Once column level test filter is implemented in test case API, remove this hardcoded value
|
||||
const testDetails = [
|
||||
{
|
||||
value: 0,
|
||||
type: TestCaseStatus.Success,
|
||||
},
|
||||
{
|
||||
value: 0,
|
||||
type: TestCaseStatus.Aborted,
|
||||
},
|
||||
{
|
||||
value: 0,
|
||||
type: TestCaseStatus.Failed,
|
||||
},
|
||||
];
|
||||
const [columnTestSummary, setColumnTestSummary] =
|
||||
useState<columnTestResultType>();
|
||||
|
||||
const tableColumn: ColumnsType<Column> = useMemo(() => {
|
||||
return [
|
||||
{
|
||||
@ -62,6 +60,7 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
||||
return (
|
||||
<Link
|
||||
to={getProfilerDashboardWithFqnPath(
|
||||
ProfilerDashboardType.COLUMN,
|
||||
record.fullyQualifiedName || ''
|
||||
)}>
|
||||
{name}
|
||||
@ -130,10 +129,18 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
||||
title: 'Test',
|
||||
dataIndex: 'dataQualityTest',
|
||||
key: 'dataQualityTest',
|
||||
render: () => {
|
||||
render: (_, record) => {
|
||||
const summary = columnTestSummary?.[record.fullyQualifiedName || ''];
|
||||
const currentResult = summary
|
||||
? Object.entries(summary).map(([key, value]) => ({
|
||||
value,
|
||||
type: key,
|
||||
}))
|
||||
: DEFAULT_TEST_VALUE;
|
||||
|
||||
return (
|
||||
<Space size={16}>
|
||||
{testDetails.map((test, i) => (
|
||||
{currentResult.map((test, i) => (
|
||||
<TestIndicator key={i} type={test.type} value={test.value} />
|
||||
))}
|
||||
</Space>
|
||||
@ -160,7 +167,7 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
||||
),
|
||||
},
|
||||
];
|
||||
}, [columns]);
|
||||
}, [columns, columnTestSummary]);
|
||||
|
||||
const handleSearchAction = (searchText: string) => {
|
||||
setSearchText(searchText);
|
||||
@ -175,6 +182,21 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
||||
setData(columns);
|
||||
}, [columns]);
|
||||
|
||||
useEffect(() => {
|
||||
if (columnTests.length) {
|
||||
const colResult = columnTests.reduce((acc, curr) => {
|
||||
const fqn = curr.entityFQN || '';
|
||||
if (isUndefined(acc[fqn])) {
|
||||
acc[fqn] = { ...INITIAL_TEST_RESULT_SUMMARY };
|
||||
}
|
||||
updateTestResults(acc[fqn], curr.testCaseResult?.testCaseStatus || '');
|
||||
|
||||
return acc;
|
||||
}, {} as columnTestResultType);
|
||||
setColumnTestSummary(colResult);
|
||||
}
|
||||
}, [columnTests]);
|
||||
|
||||
return (
|
||||
<div data-testid="column-profile-table-container">
|
||||
<div className="tw-w-2/6">
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
*/
|
||||
|
||||
import { Column, Table } from '../../generated/entity/data/table';
|
||||
import { TestCaseStatus } from '../../generated/tests/tableTest';
|
||||
import { TestCase } from '../../generated/tests/testCase';
|
||||
import { DatasetTestModeType } from '../../interface/dataQuality.interface';
|
||||
|
||||
export interface TableProfilerProps {
|
||||
@ -24,8 +24,20 @@ export interface TableProfilerProps {
|
||||
table: Table;
|
||||
}
|
||||
|
||||
export type TableTestsType = {
|
||||
tests: TestCase[];
|
||||
results: {
|
||||
success: number;
|
||||
aborted: number;
|
||||
failed: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type columnTestResultType = { [key: string]: TableTestsType['results'] };
|
||||
|
||||
export interface ColumnProfileTableProps {
|
||||
columns: Column[];
|
||||
columnTests: TestCase[];
|
||||
onAddTestClick: (
|
||||
tabValue: number,
|
||||
testMode?: DatasetTestModeType,
|
||||
@ -47,7 +59,7 @@ export interface ProfilerSettingsModalProps {
|
||||
|
||||
export interface TestIndicatorProps {
|
||||
value: number | string;
|
||||
type: TestCaseStatus;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export type OverallTableSummeryType = {
|
||||
|
||||
@ -20,7 +20,7 @@ import {
|
||||
screen,
|
||||
} from '@testing-library/react';
|
||||
import React from 'react';
|
||||
import { MOCK_TABLE } from '../../mocks/TableData.mock';
|
||||
import { MOCK_TABLE, TEST_CASE } from '../../mocks/TableData.mock';
|
||||
import { getCurrentDatasetTab } from '../../utils/DatasetDetailsUtils';
|
||||
import { TableProfilerProps } from './TableProfiler.interface';
|
||||
// internal imports
|
||||
@ -69,6 +69,12 @@ jest.mock('../../utils/CommonUtils', () => ({
|
||||
}));
|
||||
const mockGetCurrentDatasetTab = getCurrentDatasetTab as jest.Mock;
|
||||
|
||||
jest.mock('../../axiosAPIs/testAPI', () => ({
|
||||
getListTestCase: jest
|
||||
.fn()
|
||||
.mockImplementation(() => Promise.resolve(TEST_CASE)),
|
||||
}));
|
||||
|
||||
const mockProps: TableProfilerProps = {
|
||||
table: MOCK_TABLE,
|
||||
onAddTestClick: jest.fn(),
|
||||
@ -128,9 +134,6 @@ describe('Test TableProfiler component', () => {
|
||||
});
|
||||
|
||||
it('CTA: Setting button should work properly', async () => {
|
||||
const setSettingModalVisible = jest.fn();
|
||||
const handleClick = jest.spyOn(React, 'useState');
|
||||
handleClick.mockImplementation(() => [false, setSettingModalVisible]);
|
||||
render(<TableProfilerV1 {...mockProps} />);
|
||||
|
||||
const settingBtn = await screen.findByTestId('profiler-setting-btn');
|
||||
@ -141,6 +144,8 @@ describe('Test TableProfiler component', () => {
|
||||
fireEvent.click(settingBtn);
|
||||
});
|
||||
|
||||
expect(setSettingModalVisible).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
await screen.findByText('ProfilerSettingsModal.component')
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
@ -12,27 +12,43 @@
|
||||
*/
|
||||
|
||||
import { Button, Col, Row } from 'antd';
|
||||
import { AxiosError } from 'axios';
|
||||
import classNames from 'classnames';
|
||||
import { isUndefined } from 'lodash';
|
||||
import React, { FC, useMemo, useState } from 'react';
|
||||
import { isEmpty, isUndefined } from 'lodash';
|
||||
import React, { FC, useEffect, useMemo, useState } from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { getListTestCase } from '../../axiosAPIs/testAPI';
|
||||
import { API_RES_MAX_SIZE } from '../../constants/constants';
|
||||
import { INITIAL_TEST_RESULT_SUMMARY } from '../../constants/profiler.constant';
|
||||
import { ProfilerDashboardType } from '../../enums/table.enum';
|
||||
import { TestCase } from '../../generated/tests/testCase';
|
||||
import {
|
||||
formatNumberWithComma,
|
||||
formTwoDigitNmber,
|
||||
} from '../../utils/CommonUtils';
|
||||
import { updateTestResults } from '../../utils/DataQualityAndProfilerUtils';
|
||||
import { getCurrentDatasetTab } from '../../utils/DatasetDetailsUtils';
|
||||
import { getProfilerDashboardWithFqnPath } from '../../utils/RouterUtils';
|
||||
import SVGIcons, { Icons } from '../../utils/SvgUtils';
|
||||
import { generateEntityLink } from '../../utils/TableUtils';
|
||||
import { showErrorToast } from '../../utils/ToastUtils';
|
||||
import ColumnProfileTable from './Component/ColumnProfileTable';
|
||||
import ProfilerSettingsModal from './Component/ProfilerSettingsModal';
|
||||
import {
|
||||
OverallTableSummeryType,
|
||||
TableProfilerProps,
|
||||
TableTestsType,
|
||||
} from './TableProfiler.interface';
|
||||
import './tableProfiler.less';
|
||||
|
||||
const TableProfilerV1: FC<TableProfilerProps> = ({ table, onAddTestClick }) => {
|
||||
const { profile, columns } = table;
|
||||
const [settingModalVisible, setSettingModalVisible] = useState(false);
|
||||
const [columnTests, setColumnTests] = useState<TestCase[]>([]);
|
||||
const [tableTests, setTableTests] = useState<TableTestsType>({
|
||||
tests: [],
|
||||
results: INITIAL_TEST_RESULT_SUMMARY,
|
||||
});
|
||||
|
||||
const handleSettingModal = (value: boolean) => {
|
||||
setSettingModalVisible(value);
|
||||
@ -53,21 +69,59 @@ const TableProfilerV1: FC<TableProfilerProps> = ({ table, onAddTestClick }) => {
|
||||
},
|
||||
{
|
||||
title: 'Success',
|
||||
value: formTwoDigitNmber(0),
|
||||
value: formTwoDigitNmber(tableTests.results.success),
|
||||
className: 'success',
|
||||
},
|
||||
{
|
||||
title: 'Aborted',
|
||||
value: formTwoDigitNmber(0),
|
||||
value: formTwoDigitNmber(tableTests.results.aborted),
|
||||
className: 'aborted',
|
||||
},
|
||||
{
|
||||
title: 'Failed',
|
||||
value: formTwoDigitNmber(0),
|
||||
value: formTwoDigitNmber(tableTests.results.failed),
|
||||
className: 'failed',
|
||||
},
|
||||
];
|
||||
}, [profile]);
|
||||
}, [profile, tableTests]);
|
||||
|
||||
const fetchAllTests = async () => {
|
||||
try {
|
||||
const { data } = await getListTestCase({
|
||||
fields: 'testCaseResult',
|
||||
entityLink: generateEntityLink(table.fullyQualifiedName || ''),
|
||||
includeAllTests: true,
|
||||
limit: API_RES_MAX_SIZE,
|
||||
});
|
||||
const columnTestsCase: TestCase[] = [];
|
||||
const tableTests: TableTestsType = {
|
||||
tests: [],
|
||||
results: { ...INITIAL_TEST_RESULT_SUMMARY },
|
||||
};
|
||||
data.forEach((test) => {
|
||||
if (test.entityFQN === table.fullyQualifiedName) {
|
||||
tableTests.tests.push(test);
|
||||
|
||||
updateTestResults(
|
||||
tableTests.results,
|
||||
test.testCaseResult?.testCaseStatus || ''
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
columnTestsCase.push(test);
|
||||
});
|
||||
setTableTests(tableTests);
|
||||
setColumnTests(columnTestsCase);
|
||||
} catch (error) {
|
||||
showErrorToast(error as AxiosError);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (isEmpty(table)) return;
|
||||
fetchAllTests();
|
||||
}, [table]);
|
||||
|
||||
if (isUndefined(profile)) {
|
||||
return (
|
||||
@ -134,9 +188,19 @@ const TableProfilerV1: FC<TableProfilerProps> = ({ table, onAddTestClick }) => {
|
||||
</p>
|
||||
</Col>
|
||||
))}
|
||||
<Col className="tw-flex tw-justify-end" span={24}>
|
||||
<Link
|
||||
to={getProfilerDashboardWithFqnPath(
|
||||
ProfilerDashboardType.TABLE,
|
||||
table.fullyQualifiedName || ''
|
||||
)}>
|
||||
View more detail
|
||||
</Link>
|
||||
</Col>
|
||||
</Row>
|
||||
|
||||
<ColumnProfileTable
|
||||
columnTests={columnTests}
|
||||
columns={columns.map((col) => ({
|
||||
...col,
|
||||
key: col.name,
|
||||
|
||||
@ -20,6 +20,7 @@ export interface DeleteWidgetModalProps {
|
||||
entityType: string;
|
||||
isAdminUser?: boolean;
|
||||
entityId?: string;
|
||||
prepareType?: boolean;
|
||||
isRecursiveDelete?: boolean;
|
||||
afterDeleteAction?: () => void;
|
||||
}
|
||||
|
||||
@ -35,6 +35,7 @@ const DeleteWidgetModal = ({
|
||||
entityType,
|
||||
onCancel,
|
||||
entityId,
|
||||
prepareType = true,
|
||||
isRecursiveDelete,
|
||||
afterDeleteAction,
|
||||
}: DeleteWidgetModalProps) => {
|
||||
@ -109,7 +110,7 @@ const DeleteWidgetModal = ({
|
||||
const handleOnEntityDeleteConfirm = () => {
|
||||
setEntityDeleteState((prev) => ({ ...prev, loading: 'waiting' }));
|
||||
deleteEntity(
|
||||
prepareEntityType(),
|
||||
prepareType ? prepareEntityType() : entityType,
|
||||
entityId ?? '',
|
||||
Boolean(isRecursiveDelete),
|
||||
entityDeleteState.softDelete
|
||||
|
||||
@ -93,6 +93,7 @@ export const PLACEHOLDER_SETTING_CATEGORY = ':settingCategory';
|
||||
export const PLACEHOLDER_USER_BOT = ':bot';
|
||||
export const PLACEHOLDER_WEBHOOK_TYPE = ':webhookType';
|
||||
export const PLACEHOLDER_RULE_NAME = ':ruleName';
|
||||
export const PLACEHOLDER_DASHBOARD_TYPE = ':dashboardType';
|
||||
|
||||
export const pagingObject = { after: '', before: '', total: 0 };
|
||||
|
||||
@ -220,7 +221,7 @@ export const ROUTES = {
|
||||
MLMODEL_DETAILS_WITH_TAB: `/mlmodel/${PLACEHOLDER_ROUTE_MLMODEL_FQN}/${PLACEHOLDER_ROUTE_TAB}`,
|
||||
CUSTOM_ENTITY_DETAIL: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
||||
ADD_CUSTOM_PROPERTY: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}/add-field`,
|
||||
PROFILER_DASHBOARD: `/profiler-dashboard/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
||||
PROFILER_DASHBOARD: `/profiler-dashboard/${PLACEHOLDER_DASHBOARD_TYPE}/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
||||
|
||||
// Tasks Routes
|
||||
REQUEST_DESCRIPTION: `/request-description/${PLACEHOLDER_ROUTE_ENTITY_TYPE}/${PLACEHOLDER_ROUTE_ENTITY_FQN}`,
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
|
||||
import { CSMode } from '../enums/codemirror.enum';
|
||||
import { ColumnProfilerConfig } from '../generated/entity/data/table';
|
||||
import { TestCaseStatus } from '../generated/tests/tableTest';
|
||||
import { JSON_TAB_SIZE } from './constants';
|
||||
|
||||
export const excludedMetrics = [
|
||||
@ -63,6 +64,8 @@ export const PROFILER_FILTER_RANGE = {
|
||||
last60days: { days: 60, title: 'Last 60 days' },
|
||||
};
|
||||
|
||||
export const COLORS = ['#7147E8', '#B02AAC', '#B02AAC', '#1890FF', '#008376'];
|
||||
|
||||
export const DEFAULT_CHART_COLLECTION_VALUE = {
|
||||
distinctCount: { data: [], color: '#1890FF' },
|
||||
uniqueCount: { data: [], color: '#008376' },
|
||||
@ -161,6 +164,27 @@ export const DEFAULT_INCLUDE_PROFILE: ColumnProfilerConfig[] = [
|
||||
},
|
||||
];
|
||||
|
||||
export const INITIAL_TEST_RESULT_SUMMARY = {
|
||||
success: 0,
|
||||
aborted: 0,
|
||||
failed: 0,
|
||||
};
|
||||
|
||||
export const DEFAULT_TEST_VALUE = [
|
||||
{
|
||||
value: 0,
|
||||
type: TestCaseStatus.Success,
|
||||
},
|
||||
{
|
||||
value: 0,
|
||||
type: TestCaseStatus.Aborted,
|
||||
},
|
||||
{
|
||||
value: 0,
|
||||
type: TestCaseStatus.Failed,
|
||||
},
|
||||
];
|
||||
|
||||
export const codeMirrorOption = {
|
||||
tabSize: JSON_TAB_SIZE,
|
||||
indentUnit: JSON_TAB_SIZE,
|
||||
|
||||
@ -26,3 +26,8 @@ export enum PrimaryTableDataTypes {
|
||||
NUMERIC = 'numeric',
|
||||
BOOLEAN = 'boolean',
|
||||
}
|
||||
|
||||
export enum ProfilerDashboardType {
|
||||
TABLE = 'table',
|
||||
COLUMN = 'column',
|
||||
}
|
||||
|
||||
@ -111,6 +111,7 @@ const jsonData = {
|
||||
'fetch-users-error': 'Error while fetching users!',
|
||||
'fetch-table-profiler-config-error':
|
||||
'Error while fetching table profiler config!',
|
||||
'fetch-column-test-error': 'Error while fetching column test case!',
|
||||
|
||||
'test-connection-error': 'Error while testing connection!',
|
||||
|
||||
|
||||
@ -201,3 +201,71 @@ export const MOCK_TABLE = {
|
||||
],
|
||||
deleted: false,
|
||||
} as unknown as Table;
|
||||
|
||||
export const TEST_CASE = {
|
||||
data: [
|
||||
{
|
||||
id: 'b9d059d8-b968-42ad-9f89-2b40b92a6659',
|
||||
name: 'column_value_max_to_be_between',
|
||||
fullyQualifiedName:
|
||||
'sample_data.ecommerce_db.shopify.dim_address.shop_id.column_value_max_to_be_between',
|
||||
description: 'test the value of a column is between x and z, new value',
|
||||
testDefinition: {
|
||||
id: '16b32e12-21c5-491c-919e-88748d9d5d67',
|
||||
type: 'testDefinition',
|
||||
name: 'columnValueMaxToBeBetween',
|
||||
fullyQualifiedName: 'columnValueMaxToBeBetween',
|
||||
description:
|
||||
'This schema defines the test ColumnValueMaxToBeBetween. Test the maximum value in a col is within a range.',
|
||||
displayName: 'columnValueMaxToBeBetween',
|
||||
deleted: false,
|
||||
href: 'http://localhost:8585/api/v1/testDefinition/16b32e12-21c5-491c-919e-88748d9d5d67',
|
||||
},
|
||||
entityLink:
|
||||
'<#E::table::sample_data.ecommerce_db.shopify.dim_address::columns::shop_id>',
|
||||
entityFQN: 'sample_data.ecommerce_db.shopify.dim_address.shop_id',
|
||||
parameterValues: [
|
||||
{
|
||||
name: 'minValueForMaxInCol',
|
||||
value: '40',
|
||||
},
|
||||
{
|
||||
name: 'maxValueForMaxInCol',
|
||||
value: '100',
|
||||
},
|
||||
],
|
||||
testCaseResult: {
|
||||
timestamp: 1661416859,
|
||||
testCaseStatus: 'Success',
|
||||
result: 'Found max=65 vs. the expected min=50, max=100.',
|
||||
testResultValue: [
|
||||
{
|
||||
name: 'max',
|
||||
value: '65',
|
||||
},
|
||||
],
|
||||
},
|
||||
version: 0.3,
|
||||
updatedAt: 1661425991294,
|
||||
updatedBy: 'anonymous',
|
||||
href: 'http://localhost:8585/api/v1/testCase/b9d059d8-b968-42ad-9f89-2b40b92a6659',
|
||||
changeDescription: {
|
||||
fieldsAdded: [],
|
||||
fieldsUpdated: [
|
||||
{
|
||||
name: 'description',
|
||||
oldValue: 'test the value of a column is between x and y',
|
||||
newValue:
|
||||
'test the value of a column is between x and z, new value',
|
||||
},
|
||||
],
|
||||
fieldsDeleted: [],
|
||||
previousVersion: 0.2,
|
||||
},
|
||||
deleted: false,
|
||||
},
|
||||
],
|
||||
paging: {
|
||||
total: 1,
|
||||
},
|
||||
};
|
||||
|
||||
@ -21,32 +21,40 @@ import {
|
||||
getTableDetailsByFQN,
|
||||
patchTableDetails,
|
||||
} from '../../axiosAPIs/tableAPI';
|
||||
import { getListTestCase } from '../../axiosAPIs/testAPI';
|
||||
import ErrorPlaceHolder from '../../components/common/error-with-placeholder/ErrorPlaceHolder';
|
||||
import PageContainerV1 from '../../components/containers/PageContainerV1';
|
||||
import Loader from '../../components/Loader/Loader';
|
||||
import ProfilerDashboard from '../../components/ProfilerDashboard/ProfilerDashboard';
|
||||
import { API_RES_MAX_SIZE } from '../../constants/constants';
|
||||
import { ProfilerDashboardType } from '../../enums/table.enum';
|
||||
import { ColumnProfile, Table } from '../../generated/entity/data/table';
|
||||
import { TestCase } from '../../generated/tests/testCase';
|
||||
import jsonData from '../../jsons/en';
|
||||
import {
|
||||
getNameFromFQN,
|
||||
getTableFQNFromColumnFQN,
|
||||
} from '../../utils/CommonUtils';
|
||||
import { generateEntityLink } from '../../utils/TableUtils';
|
||||
import { showErrorToast } from '../../utils/ToastUtils';
|
||||
|
||||
const ProfilerDashboardPage = () => {
|
||||
const { entityTypeFQN } = useParams<Record<string, string>>();
|
||||
const { entityTypeFQN, dashboardType } = useParams<Record<string, string>>();
|
||||
const isColumnView = dashboardType === ProfilerDashboardType.COLUMN;
|
||||
const [table, setTable] = useState<Table>({} as Table);
|
||||
const [profilerData, setProfilerData] = useState<ColumnProfile[]>([]);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [error, setError] = useState(false);
|
||||
const [testCases, setTestCases] = useState<TestCase[]>([]);
|
||||
|
||||
const fetchProfilerData = async (fqn: string, days = 3) => {
|
||||
try {
|
||||
const startTs = moment().subtract(days, 'days').unix();
|
||||
const endTs = moment().unix();
|
||||
|
||||
const { data } = await getColumnProfilerList(fqn, {
|
||||
startTs: startTs,
|
||||
startTs,
|
||||
endTs,
|
||||
limit: API_RES_MAX_SIZE,
|
||||
});
|
||||
setProfilerData(data || []);
|
||||
@ -57,15 +65,39 @@ const ProfilerDashboardPage = () => {
|
||||
}
|
||||
};
|
||||
|
||||
const fetchTableEntity = async (fqn: string) => {
|
||||
const fetchTestCases = async (fqn: string) => {
|
||||
try {
|
||||
getTableFQNFromColumnFQN(fqn);
|
||||
const data = await getTableDetailsByFQN(
|
||||
getTableFQNFromColumnFQN(fqn),
|
||||
'tags, usageSummary, owner, followers, profile'
|
||||
const { data } = await getListTestCase({
|
||||
fields: 'testDefinition,testCaseResult',
|
||||
entityLink: fqn,
|
||||
limit: API_RES_MAX_SIZE,
|
||||
});
|
||||
setTestCases(data);
|
||||
} catch (error) {
|
||||
showErrorToast(
|
||||
error as AxiosError,
|
||||
jsonData['api-error-messages']['fetch-column-test-error']
|
||||
);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const fetchTableEntity = async () => {
|
||||
try {
|
||||
const fqn = isColumnView
|
||||
? getTableFQNFromColumnFQN(entityTypeFQN)
|
||||
: entityTypeFQN;
|
||||
const field = `tags, usageSummary, owner, followers${
|
||||
isColumnView ? ', profile' : ''
|
||||
}`;
|
||||
const data = await getTableDetailsByFQN(fqn, field);
|
||||
setTable(data ?? ({} as Table));
|
||||
fetchProfilerData(entityTypeFQN);
|
||||
if (isColumnView) {
|
||||
fetchProfilerData(entityTypeFQN);
|
||||
} else {
|
||||
fetchTestCases(generateEntityLink(entityTypeFQN));
|
||||
}
|
||||
} catch (error) {
|
||||
showErrorToast(
|
||||
error as AxiosError,
|
||||
@ -92,7 +124,7 @@ const ProfilerDashboardPage = () => {
|
||||
|
||||
useEffect(() => {
|
||||
if (entityTypeFQN) {
|
||||
fetchTableEntity(entityTypeFQN);
|
||||
fetchTableEntity();
|
||||
} else {
|
||||
setIsLoading(false);
|
||||
setError(true);
|
||||
@ -118,8 +150,10 @@ const ProfilerDashboardPage = () => {
|
||||
<PageContainerV1 className="tw-py-4">
|
||||
<ProfilerDashboard
|
||||
fetchProfilerData={fetchProfilerData}
|
||||
fetchTestCases={fetchTestCases}
|
||||
profilerData={profilerData}
|
||||
table={table}
|
||||
testCases={testCases}
|
||||
onTableChange={updateTableHandler}
|
||||
/>
|
||||
</PageContainerV1>
|
||||
|
||||
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { TableTestsType } from '../components/TableProfiler/TableProfiler.interface';
|
||||
import { TestCaseStatus } from '../generated/tests/tableTest';
|
||||
|
||||
export const updateTestResults = (
|
||||
results: TableTestsType['results'],
|
||||
testCaseStatus: string
|
||||
) => {
|
||||
switch (testCaseStatus) {
|
||||
case TestCaseStatus.Success:
|
||||
results.success += 1;
|
||||
|
||||
break;
|
||||
case TestCaseStatus.Failed:
|
||||
results.failed += 1;
|
||||
|
||||
break;
|
||||
case TestCaseStatus.Aborted:
|
||||
results.aborted += 1;
|
||||
|
||||
break;
|
||||
}
|
||||
};
|
||||
@ -14,6 +14,7 @@
|
||||
import { FQN_SEPARATOR_CHAR } from '../constants/char.constants';
|
||||
import {
|
||||
IN_PAGE_SEARCH_ROUTES,
|
||||
PLACEHOLDER_DASHBOARD_TYPE,
|
||||
PLACEHOLDER_ENTITY_TYPE_FQN,
|
||||
PLACEHOLDER_GLOSSARY_NAME,
|
||||
PLACEHOLDER_GLOSSARY_TERMS_FQN,
|
||||
@ -289,10 +290,15 @@ export const getPath = (pathName: string) => {
|
||||
}
|
||||
};
|
||||
|
||||
export const getProfilerDashboardWithFqnPath = (entityTypeFQN: string) => {
|
||||
export const getProfilerDashboardWithFqnPath = (
|
||||
dashboardType: string,
|
||||
entityTypeFQN: string
|
||||
) => {
|
||||
let path = ROUTES.PROFILER_DASHBOARD;
|
||||
|
||||
path = path.replace(PLACEHOLDER_ENTITY_TYPE_FQN, entityTypeFQN);
|
||||
path = path
|
||||
.replace(PLACEHOLDER_DASHBOARD_TYPE, dashboardType)
|
||||
.replace(PLACEHOLDER_ENTITY_TYPE_FQN, entityTypeFQN);
|
||||
|
||||
return path;
|
||||
};
|
||||
|
||||
@ -37,6 +37,7 @@ import { Column, DataType } from '../generated/entity/data/table';
|
||||
import { TableTest, TestCaseStatus } from '../generated/tests/tableTest';
|
||||
import { TagLabel } from '../generated/type/tagLabel';
|
||||
import { ModifiedTableColumn } from '../interface/dataQuality.interface';
|
||||
import { getNameFromFQN, getTableFQNFromColumnFQN } from './CommonUtils';
|
||||
import { getGlossaryPath } from './RouterUtils';
|
||||
import { ordinalize } from './StringsUtils';
|
||||
import SVGIcons, { Icons } from './SvgUtils';
|
||||
@ -306,6 +307,38 @@ export const getDataTypeString = (dataType: string): string => {
|
||||
}
|
||||
};
|
||||
|
||||
export const generateEntityLink = (fqn: string, includeColumn = false) => {
|
||||
const columnLink = '<#E::table::ENTITY_FQN::columns::COLUMN>';
|
||||
const tableLink = '<#E::table::ENTITY_FQN>';
|
||||
|
||||
if (includeColumn) {
|
||||
const tableFqn = getTableFQNFromColumnFQN(fqn);
|
||||
const columnName = getNameFromFQN(fqn);
|
||||
|
||||
return columnLink
|
||||
.replace('ENTITY_FQN', tableFqn)
|
||||
.replace('COLUMN', columnName);
|
||||
} else {
|
||||
return tableLink.replace('ENTITY_FQN', fqn);
|
||||
}
|
||||
};
|
||||
|
||||
export const getTestResultBadgeIcon = (status?: TestCaseStatus) => {
|
||||
switch (status) {
|
||||
case TestCaseStatus.Success:
|
||||
return Icons.SUCCESS_BADGE;
|
||||
|
||||
case TestCaseStatus.Failed:
|
||||
return Icons.FAIL_BADGE;
|
||||
|
||||
case TestCaseStatus.Aborted:
|
||||
return Icons.PENDING_BADGE;
|
||||
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
export const getTableTestsValue = (tableTestCase: TableTest[]) => {
|
||||
const tableTestLength = tableTestCase.length;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user