mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-16 03:08:13 +00:00
added data-quality tab and filters for test, with basic table (#6869)
* added data-quality tab and filters for test, with basic table * added pass fail status with icon * added test summary component * render the chart for column test * revers the time stamp * fix right side data render layout * added legend and colors * added filter by time * updated style and move filter to chart level * updated column profiler route * added table dashboard * updated test result summary on profiler tab * fixed failing unit test * added loader
This commit is contained in:
parent
4c7fd03b5a
commit
ef5de94a8f
@ -0,0 +1,64 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Collate
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { TestCase, TestCaseResult } from '../generated/tests/testCase';
|
||||||
|
import { Include } from '../generated/type/include';
|
||||||
|
import { Paging } from '../generated/type/paging';
|
||||||
|
import APIClient from './index';
|
||||||
|
|
||||||
|
export type ListTestCaseParams = {
|
||||||
|
fields?: string;
|
||||||
|
limit?: number;
|
||||||
|
before?: string;
|
||||||
|
after?: string;
|
||||||
|
entityLink?: string;
|
||||||
|
testSuiteId?: string;
|
||||||
|
includeAllTests?: boolean;
|
||||||
|
include?: Include;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ListTestCaseResultsParams = {
|
||||||
|
startTs?: number;
|
||||||
|
endTs?: number;
|
||||||
|
before?: string;
|
||||||
|
after?: string;
|
||||||
|
limit?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
const baseUrl = '/testCase';
|
||||||
|
|
||||||
|
export const getListTestCase = async (params?: ListTestCaseParams) => {
|
||||||
|
const response = await APIClient.get<{ data: TestCase[]; paging: Paging }>(
|
||||||
|
baseUrl,
|
||||||
|
{
|
||||||
|
params,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return response.data;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getListTestCaseResults = async (
|
||||||
|
fqn: string,
|
||||||
|
params?: ListTestCaseResultsParams
|
||||||
|
) => {
|
||||||
|
const url = `${baseUrl}/${fqn}/testCaseResult`;
|
||||||
|
const response = await APIClient.get<{
|
||||||
|
data: TestCaseResult[];
|
||||||
|
paging: Paging;
|
||||||
|
}>(url, {
|
||||||
|
params,
|
||||||
|
});
|
||||||
|
|
||||||
|
return response.data;
|
||||||
|
};
|
@ -10,7 +10,6 @@
|
|||||||
* See the License for the specific language governing permissions and
|
* See the License for the specific language governing permissions and
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Button, Col, Radio, Row, Select, Space } from 'antd';
|
import { Button, Col, Radio, Row, Select, Space } from 'antd';
|
||||||
import { RadioChangeEvent } from 'antd/lib/radio';
|
import { RadioChangeEvent } from 'antd/lib/radio';
|
||||||
import { AxiosError } from 'axios';
|
import { AxiosError } from 'axios';
|
||||||
@ -29,8 +28,13 @@ import {
|
|||||||
import { PROFILER_FILTER_RANGE } from '../../constants/profiler.constant';
|
import { PROFILER_FILTER_RANGE } from '../../constants/profiler.constant';
|
||||||
import { EntityType, FqnPart } from '../../enums/entity.enum';
|
import { EntityType, FqnPart } from '../../enums/entity.enum';
|
||||||
import { ServiceCategory } from '../../enums/service.enum';
|
import { ServiceCategory } from '../../enums/service.enum';
|
||||||
|
import { ProfilerDashboardType } from '../../enums/table.enum';
|
||||||
import { OwnerType } from '../../enums/user.enum';
|
import { OwnerType } from '../../enums/user.enum';
|
||||||
import { Column, Table } from '../../generated/entity/data/table';
|
import {
|
||||||
|
Column,
|
||||||
|
Table,
|
||||||
|
TestCaseStatus,
|
||||||
|
} from '../../generated/entity/data/table';
|
||||||
import { EntityReference } from '../../generated/type/entityReference';
|
import { EntityReference } from '../../generated/type/entityReference';
|
||||||
import { LabelType, State } from '../../generated/type/tagLabel';
|
import { LabelType, State } from '../../generated/type/tagLabel';
|
||||||
import jsonData from '../../jsons/en';
|
import jsonData from '../../jsons/en';
|
||||||
@ -44,6 +48,7 @@ import {
|
|||||||
} from '../../utils/CommonUtils';
|
} from '../../utils/CommonUtils';
|
||||||
import { serviceTypeLogo } from '../../utils/ServiceUtils';
|
import { serviceTypeLogo } from '../../utils/ServiceUtils';
|
||||||
import {
|
import {
|
||||||
|
generateEntityLink,
|
||||||
getTagsWithoutTier,
|
getTagsWithoutTier,
|
||||||
getTierTags,
|
getTierTags,
|
||||||
getUsagePercentile,
|
getUsagePercentile,
|
||||||
@ -51,6 +56,7 @@ import {
|
|||||||
import { showErrorToast } from '../../utils/ToastUtils';
|
import { showErrorToast } from '../../utils/ToastUtils';
|
||||||
import EntityPageInfo from '../common/entityPageInfo/EntityPageInfo';
|
import EntityPageInfo from '../common/entityPageInfo/EntityPageInfo';
|
||||||
import PageLayout from '../containers/PageLayout';
|
import PageLayout from '../containers/PageLayout';
|
||||||
|
import DataQualityTab from './component/DataQualityTab';
|
||||||
import ProfilerTab from './component/ProfilerTab';
|
import ProfilerTab from './component/ProfilerTab';
|
||||||
import {
|
import {
|
||||||
ProfilerDashboardProps,
|
ProfilerDashboardProps,
|
||||||
@ -60,23 +66,40 @@ import './profilerDashboard.less';
|
|||||||
|
|
||||||
const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
||||||
table,
|
table,
|
||||||
|
testCases,
|
||||||
fetchProfilerData,
|
fetchProfilerData,
|
||||||
|
fetchTestCases,
|
||||||
profilerData,
|
profilerData,
|
||||||
onTableChange,
|
onTableChange,
|
||||||
}) => {
|
}) => {
|
||||||
const history = useHistory();
|
const history = useHistory();
|
||||||
const { entityTypeFQN } = useParams<Record<string, string>>();
|
const { entityTypeFQN, dashboardType } = useParams<Record<string, string>>();
|
||||||
|
const isColumnView = dashboardType === ProfilerDashboardType.COLUMN;
|
||||||
const [follower, setFollower] = useState<EntityReference[]>([]);
|
const [follower, setFollower] = useState<EntityReference[]>([]);
|
||||||
const [isFollowing, setIsFollowing] = useState<boolean>(false);
|
const [isFollowing, setIsFollowing] = useState<boolean>(false);
|
||||||
const [activeTab, setActiveTab] = useState<ProfilerDashboardTab>(
|
const [activeTab, setActiveTab] = useState<ProfilerDashboardTab>(
|
||||||
ProfilerDashboardTab.PROFILER
|
isColumnView
|
||||||
|
? ProfilerDashboardTab.PROFILER
|
||||||
|
: ProfilerDashboardTab.DATA_QUALITY
|
||||||
);
|
);
|
||||||
|
const [selectedTestCaseStatus, setSelectedTestCaseStatus] =
|
||||||
|
useState<string>('');
|
||||||
const [selectedTimeRange, setSelectedTimeRange] =
|
const [selectedTimeRange, setSelectedTimeRange] =
|
||||||
useState<keyof typeof PROFILER_FILTER_RANGE>('last3days');
|
useState<keyof typeof PROFILER_FILTER_RANGE>('last3days');
|
||||||
const [activeColumnDetails, setActiveColumnDetails] = useState<Column>(
|
const [activeColumnDetails, setActiveColumnDetails] = useState<Column>(
|
||||||
{} as Column
|
{} as Column
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const tabOptions = useMemo(() => {
|
||||||
|
return Object.values(ProfilerDashboardTab).filter((value) => {
|
||||||
|
if (value === ProfilerDashboardTab.PROFILER) {
|
||||||
|
return isColumnView;
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
});
|
||||||
|
}, [dashboardType]);
|
||||||
|
|
||||||
const timeRangeOption = useMemo(() => {
|
const timeRangeOption = useMemo(() => {
|
||||||
return Object.entries(PROFILER_FILTER_RANGE).map(([key, value]) => ({
|
return Object.entries(PROFILER_FILTER_RANGE).map(([key, value]) => ({
|
||||||
label: value.title,
|
label: value.title,
|
||||||
@ -84,6 +107,21 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
|||||||
}));
|
}));
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const testCaseStatusOption = useMemo(() => {
|
||||||
|
const testCaseStatus: Record<string, string>[] = Object.values(
|
||||||
|
TestCaseStatus
|
||||||
|
).map((value) => ({
|
||||||
|
label: value,
|
||||||
|
value: value,
|
||||||
|
}));
|
||||||
|
testCaseStatus.unshift({
|
||||||
|
label: 'All Test',
|
||||||
|
value: '',
|
||||||
|
});
|
||||||
|
|
||||||
|
return testCaseStatus;
|
||||||
|
}, []);
|
||||||
|
|
||||||
const tier = useMemo(() => getTierTags(table.tags ?? []), [table]);
|
const tier = useMemo(() => getTierTags(table.tags ?? []), [table]);
|
||||||
const breadcrumb = useMemo(() => {
|
const breadcrumb = useMemo(() => {
|
||||||
const serviceName = getEntityName(table.service);
|
const serviceName = getEntityName(table.service);
|
||||||
@ -251,7 +289,10 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
|||||||
const value = e.target.value as ProfilerDashboardTab;
|
const value = e.target.value as ProfilerDashboardTab;
|
||||||
if (ProfilerDashboardTab.SUMMARY === value) {
|
if (ProfilerDashboardTab.SUMMARY === value) {
|
||||||
history.push(getTableTabPath(table.fullyQualifiedName || '', 'profiler'));
|
history.push(getTableTabPath(table.fullyQualifiedName || '', 'profiler'));
|
||||||
|
} else if (ProfilerDashboardTab.DATA_QUALITY === value) {
|
||||||
|
fetchTestCases(generateEntityLink(entityTypeFQN, true));
|
||||||
}
|
}
|
||||||
|
setSelectedTestCaseStatus('');
|
||||||
setActiveTab(value);
|
setActiveTab(value);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -264,17 +305,35 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
|||||||
const handleTimeRangeChange = (value: keyof typeof PROFILER_FILTER_RANGE) => {
|
const handleTimeRangeChange = (value: keyof typeof PROFILER_FILTER_RANGE) => {
|
||||||
if (value !== selectedTimeRange) {
|
if (value !== selectedTimeRange) {
|
||||||
setSelectedTimeRange(value);
|
setSelectedTimeRange(value);
|
||||||
|
if (activeTab === ProfilerDashboardTab.PROFILER) {
|
||||||
fetchProfilerData(entityTypeFQN, PROFILER_FILTER_RANGE[value].days);
|
fetchProfilerData(entityTypeFQN, PROFILER_FILTER_RANGE[value].days);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleTestCaseStatusChange = (value: string) => {
|
||||||
|
if (value !== selectedTestCaseStatus) {
|
||||||
|
setSelectedTestCaseStatus(value);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const getFilterTestCase = () => {
|
||||||
|
return testCases.filter(
|
||||||
|
(data) =>
|
||||||
|
selectedTestCaseStatus === '' ||
|
||||||
|
data.testCaseResult?.testCaseStatus === selectedTestCaseStatus
|
||||||
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (table) {
|
if (table) {
|
||||||
|
if (isColumnView) {
|
||||||
const columnName = getNameFromFQN(entityTypeFQN);
|
const columnName = getNameFromFQN(entityTypeFQN);
|
||||||
const selectedColumn = table.columns.find(
|
const selectedColumn = table.columns.find(
|
||||||
(col) => col.name === columnName
|
(col) => col.name === columnName
|
||||||
);
|
);
|
||||||
setActiveColumnDetails(selectedColumn || ({} as Column));
|
setActiveColumnDetails(selectedColumn || ({} as Column));
|
||||||
|
}
|
||||||
setFollower(table?.followers || []);
|
setFollower(table?.followers || []);
|
||||||
setIsFollowing(
|
setIsFollowing(
|
||||||
follower.some(({ id }: { id: string }) => id === getCurrentUserId())
|
follower.some(({ id }: { id: string }) => id === getCurrentUserId())
|
||||||
@ -315,18 +374,28 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
|||||||
<Radio.Group
|
<Radio.Group
|
||||||
buttonStyle="solid"
|
buttonStyle="solid"
|
||||||
optionType="button"
|
optionType="button"
|
||||||
options={Object.values(ProfilerDashboardTab)}
|
options={tabOptions}
|
||||||
value={activeTab}
|
value={activeTab}
|
||||||
onChange={handleTabChange}
|
onChange={handleTabChange}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<Space size={16}>
|
<Space size={16}>
|
||||||
|
{activeTab === ProfilerDashboardTab.DATA_QUALITY && (
|
||||||
|
<Select
|
||||||
|
className="tw-w-32"
|
||||||
|
options={testCaseStatusOption}
|
||||||
|
value={selectedTestCaseStatus}
|
||||||
|
onChange={handleTestCaseStatusChange}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
{activeTab === ProfilerDashboardTab.PROFILER && (
|
||||||
<Select
|
<Select
|
||||||
className="tw-w-32"
|
className="tw-w-32"
|
||||||
options={timeRangeOption}
|
options={timeRangeOption}
|
||||||
value={selectedTimeRange}
|
value={selectedTimeRange}
|
||||||
onChange={handleTimeRangeChange}
|
onChange={handleTimeRangeChange}
|
||||||
/>
|
/>
|
||||||
|
)}
|
||||||
<Button type="primary" onClick={handleAddTestClick}>
|
<Button type="primary" onClick={handleAddTestClick}>
|
||||||
Add Test
|
Add Test
|
||||||
</Button>
|
</Button>
|
||||||
@ -344,7 +413,9 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
|||||||
)}
|
)}
|
||||||
|
|
||||||
{activeTab === ProfilerDashboardTab.DATA_QUALITY && (
|
{activeTab === ProfilerDashboardTab.DATA_QUALITY && (
|
||||||
<Col span={24}>Data Quality</Col>
|
<Col span={24}>
|
||||||
|
<DataQualityTab testCases={getFilterTestCase()} />
|
||||||
|
</Col>
|
||||||
)}
|
)}
|
||||||
</Row>
|
</Row>
|
||||||
</PageLayout>
|
</PageLayout>
|
||||||
|
@ -0,0 +1,120 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Collate
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Button, Space, Table, Tooltip } from 'antd';
|
||||||
|
import { ColumnsType } from 'antd/lib/table';
|
||||||
|
import { isUndefined } from 'lodash';
|
||||||
|
import moment from 'moment';
|
||||||
|
import React, { useMemo, useState } from 'react';
|
||||||
|
import { TestCase, TestCaseResult } from '../../../generated/tests/testCase';
|
||||||
|
import SVGIcons, { Icons } from '../../../utils/SvgUtils';
|
||||||
|
import { getTestResultBadgeIcon } from '../../../utils/TableUtils';
|
||||||
|
import DeleteWidgetModal from '../../common/DeleteWidget/DeleteWidgetModal';
|
||||||
|
import { DataQualityTabProps } from '../profilerDashboard.interface';
|
||||||
|
import TestSummary from './TestSummary';
|
||||||
|
|
||||||
|
const DataQualityTab: React.FC<DataQualityTabProps> = ({ testCases }) => {
|
||||||
|
const [selectedTestCase, setSelectedTestCase] = useState<TestCase>();
|
||||||
|
const columns: ColumnsType<TestCase> = useMemo(() => {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
title: 'Last Run Result',
|
||||||
|
dataIndex: 'testCaseResult',
|
||||||
|
key: 'testCaseResult',
|
||||||
|
render: (result: TestCaseResult) => (
|
||||||
|
<Space size={8}>
|
||||||
|
{result?.testCaseStatus && (
|
||||||
|
<SVGIcons
|
||||||
|
alt="result"
|
||||||
|
className="tw-w-4"
|
||||||
|
icon={getTestResultBadgeIcon(result.testCaseStatus)}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
<span>{result?.testCaseStatus || '--'}</span>
|
||||||
|
</Space>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Last Run',
|
||||||
|
dataIndex: 'testCaseResult',
|
||||||
|
key: 'lastRun',
|
||||||
|
render: (result: TestCaseResult) =>
|
||||||
|
result?.timestamp
|
||||||
|
? moment.unix(result.timestamp || 0).format('DD/MMM HH:mm')
|
||||||
|
: '--',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Name',
|
||||||
|
dataIndex: 'name',
|
||||||
|
key: 'name',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Description',
|
||||||
|
dataIndex: 'description',
|
||||||
|
key: 'description',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Actions',
|
||||||
|
dataIndex: 'actions',
|
||||||
|
key: 'actions',
|
||||||
|
render: (_, record) => (
|
||||||
|
<Space>
|
||||||
|
<Tooltip placement="bottom" title="Delete">
|
||||||
|
<Button
|
||||||
|
icon={
|
||||||
|
<SVGIcons
|
||||||
|
alt="Delete"
|
||||||
|
className="tw-w-4"
|
||||||
|
icon={Icons.DELETE}
|
||||||
|
/>
|
||||||
|
}
|
||||||
|
type="text"
|
||||||
|
onClick={() => {
|
||||||
|
setSelectedTestCase(record);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</Tooltip>
|
||||||
|
</Space>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Table
|
||||||
|
columns={columns}
|
||||||
|
dataSource={testCases.map((test) => ({ ...test, key: test.name }))}
|
||||||
|
expandable={{
|
||||||
|
rowExpandable: () => true,
|
||||||
|
expandedRowRender: (recode) => <TestSummary data={recode} />,
|
||||||
|
}}
|
||||||
|
pagination={false}
|
||||||
|
size="small"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<DeleteWidgetModal
|
||||||
|
entityId={selectedTestCase?.id || ''}
|
||||||
|
entityName={selectedTestCase?.name || ''}
|
||||||
|
entityType="testCase"
|
||||||
|
prepareType={false}
|
||||||
|
visible={!isUndefined(selectedTestCase)}
|
||||||
|
onCancel={() => {
|
||||||
|
setSelectedTestCase(undefined);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default DataQualityTab;
|
@ -12,6 +12,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { Card, Col, Row, Statistic } from 'antd';
|
import { Card, Col, Row, Statistic } from 'antd';
|
||||||
|
import { sortBy } from 'lodash';
|
||||||
import moment from 'moment';
|
import moment from 'moment';
|
||||||
import React, { useEffect, useMemo, useState } from 'react';
|
import React, { useEffect, useMemo, useState } from 'react';
|
||||||
import {
|
import {
|
||||||
@ -82,11 +83,12 @@ const ProfilerTab: React.FC<ProfilerTabProps> = ({
|
|||||||
);
|
);
|
||||||
|
|
||||||
const createMetricsChartData = () => {
|
const createMetricsChartData = () => {
|
||||||
|
const updateProfilerData = sortBy(profilerData, 'timestamp');
|
||||||
const countMetricData: MetricChartType['data'] = [];
|
const countMetricData: MetricChartType['data'] = [];
|
||||||
const proportionMetricData: MetricChartType['data'] = [];
|
const proportionMetricData: MetricChartType['data'] = [];
|
||||||
const mathMetricData: MetricChartType['data'] = [];
|
const mathMetricData: MetricChartType['data'] = [];
|
||||||
const sumMetricData: MetricChartType['data'] = [];
|
const sumMetricData: MetricChartType['data'] = [];
|
||||||
profilerData.forEach((col) => {
|
updateProfilerData.forEach((col) => {
|
||||||
const x = moment.unix(col.timestamp || 0).format('DD/MMM HH:mm');
|
const x = moment.unix(col.timestamp || 0).format('DD/MMM HH:mm');
|
||||||
|
|
||||||
countMetricData.push({
|
countMetricData.push({
|
||||||
@ -124,21 +126,27 @@ const ProfilerTab: React.FC<ProfilerTabProps> = ({
|
|||||||
|
|
||||||
const countMetricInfo = countMetrics.information.map((item) => ({
|
const countMetricInfo = countMetrics.information.map((item) => ({
|
||||||
...item,
|
...item,
|
||||||
latestValue: countMetricData[0]?.[item.dataKey] || 0,
|
latestValue:
|
||||||
|
countMetricData[countMetricData.length - 1]?.[item.dataKey] || 0,
|
||||||
}));
|
}));
|
||||||
const proportionMetricInfo = proportionMetrics.information.map((item) => ({
|
const proportionMetricInfo = proportionMetrics.information.map((item) => ({
|
||||||
...item,
|
...item,
|
||||||
latestValue: parseFloat(
|
latestValue: parseFloat(
|
||||||
`${proportionMetricData[0]?.[item.dataKey] || 0}`
|
`${
|
||||||
|
proportionMetricData[proportionMetricData.length - 1]?.[
|
||||||
|
item.dataKey
|
||||||
|
] || 0
|
||||||
|
}`
|
||||||
).toFixed(2),
|
).toFixed(2),
|
||||||
}));
|
}));
|
||||||
const mathMetricInfo = mathMetrics.information.map((item) => ({
|
const mathMetricInfo = mathMetrics.information.map((item) => ({
|
||||||
...item,
|
...item,
|
||||||
latestValue: mathMetricData[0]?.[item.dataKey] || 0,
|
latestValue:
|
||||||
|
mathMetricData[mathMetricData.length - 1]?.[item.dataKey] || 0,
|
||||||
}));
|
}));
|
||||||
const sumMetricInfo = sumMetrics.information.map((item) => ({
|
const sumMetricInfo = sumMetrics.information.map((item) => ({
|
||||||
...item,
|
...item,
|
||||||
latestValue: sumMetricData[0]?.[item.dataKey] || 0,
|
latestValue: sumMetricData[sumMetricData.length - 1]?.[item.dataKey] || 0,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
setCountMetrics((pre) => ({
|
setCountMetrics((pre) => ({
|
||||||
@ -172,13 +180,13 @@ const ProfilerTab: React.FC<ProfilerTabProps> = ({
|
|||||||
<Col span={8}>
|
<Col span={8}>
|
||||||
<Card className="tw-rounded-md tw-border tw-h-full">
|
<Card className="tw-rounded-md tw-border tw-h-full">
|
||||||
<Row gutter={16}>
|
<Row gutter={16}>
|
||||||
<Col span={18}>
|
<Col span={16}>
|
||||||
<p className="tw-font-medium tw-text-base">Column summary</p>
|
<p className="tw-font-medium tw-text-base">Column summary</p>
|
||||||
<Ellipses className="tw-text-grey-muted" rows={4}>
|
<Ellipses className="tw-text-grey-muted" rows={4}>
|
||||||
{activeColumnDetails.description}
|
{activeColumnDetails.description || 'No Description'}
|
||||||
</Ellipses>
|
</Ellipses>
|
||||||
</Col>
|
</Col>
|
||||||
<Col span={6}>
|
<Col span={8}>
|
||||||
<Statistic
|
<Statistic
|
||||||
title="Data type"
|
title="Data type"
|
||||||
value={activeColumnDetails.dataTypeDisplay || ''}
|
value={activeColumnDetails.dataTypeDisplay || ''}
|
||||||
|
@ -0,0 +1,237 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Collate
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Col, Empty, Row, Select, Space, Typography } from 'antd';
|
||||||
|
import { AxiosError } from 'axios';
|
||||||
|
import { isEmpty } from 'lodash';
|
||||||
|
import moment from 'moment';
|
||||||
|
import React, { useEffect, useMemo, useState } from 'react';
|
||||||
|
import {
|
||||||
|
Legend,
|
||||||
|
Line,
|
||||||
|
LineChart,
|
||||||
|
LineProps,
|
||||||
|
ReferenceArea,
|
||||||
|
ResponsiveContainer,
|
||||||
|
Tooltip,
|
||||||
|
XAxis,
|
||||||
|
YAxis,
|
||||||
|
} from 'recharts';
|
||||||
|
import { getListTestCaseResults } from '../../../axiosAPIs/testAPI';
|
||||||
|
import { API_RES_MAX_SIZE } from '../../../constants/constants';
|
||||||
|
import {
|
||||||
|
COLORS,
|
||||||
|
PROFILER_FILTER_RANGE,
|
||||||
|
} from '../../../constants/profiler.constant';
|
||||||
|
import {
|
||||||
|
TestCaseResult,
|
||||||
|
TestCaseStatus,
|
||||||
|
} from '../../../generated/tests/tableTest';
|
||||||
|
import { showErrorToast } from '../../../utils/ToastUtils';
|
||||||
|
import RichTextEditorPreviewer from '../../common/rich-text-editor/RichTextEditorPreviewer';
|
||||||
|
import Loader from '../../Loader/Loader';
|
||||||
|
import { TestSummaryProps } from '../profilerDashboard.interface';
|
||||||
|
|
||||||
|
type ChartDataType = {
|
||||||
|
information: { label: string; color: string }[];
|
||||||
|
data: { [key: string]: string }[];
|
||||||
|
};
|
||||||
|
|
||||||
|
const TestSummary: React.FC<TestSummaryProps> = ({ data }) => {
|
||||||
|
const [chartData, setChartData] = useState<ChartDataType>(
|
||||||
|
{} as ChartDataType
|
||||||
|
);
|
||||||
|
const [results, setResults] = useState<TestCaseResult[]>([]);
|
||||||
|
const [selectedTimeRange, setSelectedTimeRange] =
|
||||||
|
useState<keyof typeof PROFILER_FILTER_RANGE>('last3days');
|
||||||
|
const [isLoading, setIsLoading] = useState(true);
|
||||||
|
|
||||||
|
const timeRangeOption = useMemo(() => {
|
||||||
|
return Object.entries(PROFILER_FILTER_RANGE).map(([key, value]) => ({
|
||||||
|
label: value.title,
|
||||||
|
value: key,
|
||||||
|
}));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const handleTimeRangeChange = (value: keyof typeof PROFILER_FILTER_RANGE) => {
|
||||||
|
if (value !== selectedTimeRange) {
|
||||||
|
setSelectedTimeRange(value);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const generateChartData = (currentData: TestCaseResult[]) => {
|
||||||
|
const chartData: { [key: string]: string }[] = [];
|
||||||
|
currentData.forEach((result) => {
|
||||||
|
const values = result.testResultValue?.reduce((acc, curr) => {
|
||||||
|
return {
|
||||||
|
...acc,
|
||||||
|
[curr.name || 'value']: parseInt(curr.value || '') || 0,
|
||||||
|
};
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
chartData.push({
|
||||||
|
name: moment.unix(result.timestamp || 0).format('DD/MMM HH:mm'),
|
||||||
|
status: result.testCaseStatus || '',
|
||||||
|
...values,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
setChartData({
|
||||||
|
information:
|
||||||
|
currentData[0]?.testResultValue?.map((info, i) => ({
|
||||||
|
label: info.name || '',
|
||||||
|
color: COLORS[i],
|
||||||
|
})) || [],
|
||||||
|
data: chartData.reverse(),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const updatedDot: LineProps['dot'] = (props) => {
|
||||||
|
const { cx = 0, cy = 0, payload } = props;
|
||||||
|
const fill =
|
||||||
|
payload.status === TestCaseStatus.Success
|
||||||
|
? '#28A745'
|
||||||
|
: payload.status === TestCaseStatus.Failed
|
||||||
|
? '#CB2431'
|
||||||
|
: '#EFAE2F';
|
||||||
|
|
||||||
|
return (
|
||||||
|
<svg
|
||||||
|
fill="none"
|
||||||
|
height={8}
|
||||||
|
width={8}
|
||||||
|
x={cx - 4}
|
||||||
|
xmlns="http://www.w3.org/2000/svg"
|
||||||
|
y={cy - 4}>
|
||||||
|
<circle cx={4} cy={4} fill={fill} r={4} />
|
||||||
|
</svg>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const fetchTestResults = async () => {
|
||||||
|
if (isEmpty(data)) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const startTs = moment()
|
||||||
|
.subtract(PROFILER_FILTER_RANGE[selectedTimeRange].days, 'days')
|
||||||
|
.unix();
|
||||||
|
const endTs = moment().unix();
|
||||||
|
const { data: chartData } = await getListTestCaseResults(
|
||||||
|
data.fullyQualifiedName || '',
|
||||||
|
{
|
||||||
|
startTs,
|
||||||
|
endTs,
|
||||||
|
limit: API_RES_MAX_SIZE,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
setResults(chartData);
|
||||||
|
generateChartData(chartData);
|
||||||
|
} catch (error) {
|
||||||
|
showErrorToast(error as AxiosError);
|
||||||
|
} finally {
|
||||||
|
setIsLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
fetchTestResults();
|
||||||
|
}, [selectedTimeRange]);
|
||||||
|
|
||||||
|
const referenceArea = () => {
|
||||||
|
const yValues = data.parameterValues?.reduce((acc, curr, i) => {
|
||||||
|
return { ...acc, [`y${i + 1}`]: parseInt(curr.value || '') };
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ReferenceArea
|
||||||
|
fill="#28A74530"
|
||||||
|
ifOverflow="extendDomain"
|
||||||
|
stroke="#28A745"
|
||||||
|
strokeDasharray="4"
|
||||||
|
{...yValues}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Row gutter={16}>
|
||||||
|
<Col span={14}>
|
||||||
|
{isLoading ? (
|
||||||
|
<Loader />
|
||||||
|
) : results.length ? (
|
||||||
|
<div>
|
||||||
|
<Space align="end" className="tw-w-full" direction="vertical">
|
||||||
|
<Select
|
||||||
|
className="tw-w-32 tw-mb-2"
|
||||||
|
options={timeRangeOption}
|
||||||
|
value={selectedTimeRange}
|
||||||
|
onChange={handleTimeRangeChange}
|
||||||
|
/>
|
||||||
|
</Space>
|
||||||
|
<ResponsiveContainer className="tw-bg-white" minHeight={300}>
|
||||||
|
<LineChart
|
||||||
|
data={chartData.data}
|
||||||
|
margin={{
|
||||||
|
top: 8,
|
||||||
|
bottom: 8,
|
||||||
|
right: 8,
|
||||||
|
}}>
|
||||||
|
<XAxis dataKey="name" padding={{ left: 8, right: 8 }} />
|
||||||
|
<YAxis allowDataOverflow padding={{ top: 8, bottom: 8 }} />
|
||||||
|
<Tooltip />
|
||||||
|
<Legend />
|
||||||
|
{data.parameterValues?.length === 2 && referenceArea()}
|
||||||
|
{chartData?.information?.map((info, i) => (
|
||||||
|
<Line
|
||||||
|
dataKey={info.label}
|
||||||
|
dot={updatedDot}
|
||||||
|
key={i}
|
||||||
|
stroke={info.color}
|
||||||
|
type="monotone"
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</LineChart>
|
||||||
|
</ResponsiveContainer>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<Empty description="No Result Available" />
|
||||||
|
)}
|
||||||
|
</Col>
|
||||||
|
<Col span={10}>
|
||||||
|
<Row gutter={[8, 8]}>
|
||||||
|
<Col span={24}>
|
||||||
|
<Typography.Text type="secondary">Name: </Typography.Text>
|
||||||
|
<Typography.Text>{data.displayName || data.name}</Typography.Text>
|
||||||
|
</Col>
|
||||||
|
<Col span={24}>
|
||||||
|
<Typography.Text type="secondary">Parameter: </Typography.Text>
|
||||||
|
</Col>
|
||||||
|
<Col offset={2} span={24}>
|
||||||
|
{data.parameterValues?.map((param) => (
|
||||||
|
<Typography key={param.name}>
|
||||||
|
<Typography.Text>{param.name}: </Typography.Text>
|
||||||
|
<Typography.Text>{param.value}</Typography.Text>
|
||||||
|
</Typography>
|
||||||
|
))}
|
||||||
|
</Col>
|
||||||
|
|
||||||
|
<Col className="tw-flex tw-gap-2" span={24}>
|
||||||
|
<Typography.Text type="secondary">Description: </Typography.Text>
|
||||||
|
<RichTextEditorPreviewer markdown={data.description || ''} />
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default TestSummary;
|
@ -16,12 +16,15 @@ import {
|
|||||||
ColumnProfile,
|
ColumnProfile,
|
||||||
Table,
|
Table,
|
||||||
} from '../../generated/entity/data/table';
|
} from '../../generated/entity/data/table';
|
||||||
|
import { TestCase } from '../../generated/tests/testCase';
|
||||||
|
|
||||||
export interface ProfilerDashboardProps {
|
export interface ProfilerDashboardProps {
|
||||||
onTableChange: (table: Table) => void;
|
onTableChange: (table: Table) => void;
|
||||||
table: Table;
|
table: Table;
|
||||||
|
testCases: TestCase[];
|
||||||
profilerData: ColumnProfile[];
|
profilerData: ColumnProfile[];
|
||||||
fetchProfilerData: (tableId: string, days?: number) => void;
|
fetchProfilerData: (tableId: string, days?: number) => void;
|
||||||
|
fetchTestCases: (fqn: string) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type MetricChartType = {
|
export type MetricChartType = {
|
||||||
@ -73,3 +76,11 @@ export interface ProfilerSummaryCardProps {
|
|||||||
}[];
|
}[];
|
||||||
showIndicator?: boolean;
|
showIndicator?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DataQualityTabProps {
|
||||||
|
testCases: TestCase[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TestSummaryProps {
|
||||||
|
data: TestCase;
|
||||||
|
}
|
||||||
|
@ -101,6 +101,7 @@ jest.mock('../../../utils/DatasetDetailsUtils');
|
|||||||
const mockProps: ColumnProfileTableProps = {
|
const mockProps: ColumnProfileTableProps = {
|
||||||
columns: MOCK_TABLE.columns,
|
columns: MOCK_TABLE.columns,
|
||||||
onAddTestClick: jest.fn,
|
onAddTestClick: jest.fn,
|
||||||
|
columnTests: [],
|
||||||
};
|
};
|
||||||
|
|
||||||
describe('Test ColumnProfileTable component', () => {
|
describe('Test ColumnProfileTable component', () => {
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
|
|
||||||
import { Button, Space, Table } from 'antd';
|
import { Button, Space, Table } from 'antd';
|
||||||
import { ColumnsType } from 'antd/lib/table';
|
import { ColumnsType } from 'antd/lib/table';
|
||||||
|
import { isUndefined } from 'lodash';
|
||||||
import React, { FC, useEffect, useMemo, useState } from 'react';
|
import React, { FC, useEffect, useMemo, useState } from 'react';
|
||||||
import { Link } from 'react-router-dom';
|
import { Link } from 'react-router-dom';
|
||||||
import {
|
import {
|
||||||
@ -20,38 +21,35 @@ import {
|
|||||||
SECONDARY_COLOR,
|
SECONDARY_COLOR,
|
||||||
SUCCESS_COLOR,
|
SUCCESS_COLOR,
|
||||||
} from '../../../constants/constants';
|
} from '../../../constants/constants';
|
||||||
|
import {
|
||||||
|
DEFAULT_TEST_VALUE,
|
||||||
|
INITIAL_TEST_RESULT_SUMMARY,
|
||||||
|
} from '../../../constants/profiler.constant';
|
||||||
|
import { ProfilerDashboardType } from '../../../enums/table.enum';
|
||||||
import { Column, ColumnProfile } from '../../../generated/entity/data/table';
|
import { Column, ColumnProfile } from '../../../generated/entity/data/table';
|
||||||
import { TestCaseStatus } from '../../../generated/tests/tableTest';
|
|
||||||
import { formatNumberWithComma } from '../../../utils/CommonUtils';
|
import { formatNumberWithComma } from '../../../utils/CommonUtils';
|
||||||
|
import { updateTestResults } from '../../../utils/DataQualityAndProfilerUtils';
|
||||||
import { getCurrentDatasetTab } from '../../../utils/DatasetDetailsUtils';
|
import { getCurrentDatasetTab } from '../../../utils/DatasetDetailsUtils';
|
||||||
import { getProfilerDashboardWithFqnPath } from '../../../utils/RouterUtils';
|
import { getProfilerDashboardWithFqnPath } from '../../../utils/RouterUtils';
|
||||||
import Ellipses from '../../common/Ellipses/Ellipses';
|
import Ellipses from '../../common/Ellipses/Ellipses';
|
||||||
import Searchbar from '../../common/searchbar/Searchbar';
|
import Searchbar from '../../common/searchbar/Searchbar';
|
||||||
import TestIndicator from '../../common/TestIndicator/TestIndicator';
|
import TestIndicator from '../../common/TestIndicator/TestIndicator';
|
||||||
import { ColumnProfileTableProps } from '../TableProfiler.interface';
|
import {
|
||||||
|
ColumnProfileTableProps,
|
||||||
|
columnTestResultType,
|
||||||
|
} from '../TableProfiler.interface';
|
||||||
import ProfilerProgressWidget from './ProfilerProgressWidget';
|
import ProfilerProgressWidget from './ProfilerProgressWidget';
|
||||||
|
|
||||||
const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
||||||
|
columnTests,
|
||||||
onAddTestClick,
|
onAddTestClick,
|
||||||
columns = [],
|
columns = [],
|
||||||
}) => {
|
}) => {
|
||||||
const [searchText, setSearchText] = useState<string>('');
|
const [searchText, setSearchText] = useState<string>('');
|
||||||
const [data, setData] = useState<Column[]>(columns);
|
const [data, setData] = useState<Column[]>(columns);
|
||||||
// TODO:- Once column level test filter is implemented in test case API, remove this hardcoded value
|
const [columnTestSummary, setColumnTestSummary] =
|
||||||
const testDetails = [
|
useState<columnTestResultType>();
|
||||||
{
|
|
||||||
value: 0,
|
|
||||||
type: TestCaseStatus.Success,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
value: 0,
|
|
||||||
type: TestCaseStatus.Aborted,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
value: 0,
|
|
||||||
type: TestCaseStatus.Failed,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
const tableColumn: ColumnsType<Column> = useMemo(() => {
|
const tableColumn: ColumnsType<Column> = useMemo(() => {
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
@ -62,6 +60,7 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
|||||||
return (
|
return (
|
||||||
<Link
|
<Link
|
||||||
to={getProfilerDashboardWithFqnPath(
|
to={getProfilerDashboardWithFqnPath(
|
||||||
|
ProfilerDashboardType.COLUMN,
|
||||||
record.fullyQualifiedName || ''
|
record.fullyQualifiedName || ''
|
||||||
)}>
|
)}>
|
||||||
{name}
|
{name}
|
||||||
@ -130,10 +129,18 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
|||||||
title: 'Test',
|
title: 'Test',
|
||||||
dataIndex: 'dataQualityTest',
|
dataIndex: 'dataQualityTest',
|
||||||
key: 'dataQualityTest',
|
key: 'dataQualityTest',
|
||||||
render: () => {
|
render: (_, record) => {
|
||||||
|
const summary = columnTestSummary?.[record.fullyQualifiedName || ''];
|
||||||
|
const currentResult = summary
|
||||||
|
? Object.entries(summary).map(([key, value]) => ({
|
||||||
|
value,
|
||||||
|
type: key,
|
||||||
|
}))
|
||||||
|
: DEFAULT_TEST_VALUE;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Space size={16}>
|
<Space size={16}>
|
||||||
{testDetails.map((test, i) => (
|
{currentResult.map((test, i) => (
|
||||||
<TestIndicator key={i} type={test.type} value={test.value} />
|
<TestIndicator key={i} type={test.type} value={test.value} />
|
||||||
))}
|
))}
|
||||||
</Space>
|
</Space>
|
||||||
@ -160,7 +167,7 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}, [columns]);
|
}, [columns, columnTestSummary]);
|
||||||
|
|
||||||
const handleSearchAction = (searchText: string) => {
|
const handleSearchAction = (searchText: string) => {
|
||||||
setSearchText(searchText);
|
setSearchText(searchText);
|
||||||
@ -175,6 +182,21 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
|||||||
setData(columns);
|
setData(columns);
|
||||||
}, [columns]);
|
}, [columns]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (columnTests.length) {
|
||||||
|
const colResult = columnTests.reduce((acc, curr) => {
|
||||||
|
const fqn = curr.entityFQN || '';
|
||||||
|
if (isUndefined(acc[fqn])) {
|
||||||
|
acc[fqn] = { ...INITIAL_TEST_RESULT_SUMMARY };
|
||||||
|
}
|
||||||
|
updateTestResults(acc[fqn], curr.testCaseResult?.testCaseStatus || '');
|
||||||
|
|
||||||
|
return acc;
|
||||||
|
}, {} as columnTestResultType);
|
||||||
|
setColumnTestSummary(colResult);
|
||||||
|
}
|
||||||
|
}, [columnTests]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div data-testid="column-profile-table-container">
|
<div data-testid="column-profile-table-container">
|
||||||
<div className="tw-w-2/6">
|
<div className="tw-w-2/6">
|
||||||
|
@ -12,7 +12,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { Column, Table } from '../../generated/entity/data/table';
|
import { Column, Table } from '../../generated/entity/data/table';
|
||||||
import { TestCaseStatus } from '../../generated/tests/tableTest';
|
import { TestCase } from '../../generated/tests/testCase';
|
||||||
import { DatasetTestModeType } from '../../interface/dataQuality.interface';
|
import { DatasetTestModeType } from '../../interface/dataQuality.interface';
|
||||||
|
|
||||||
export interface TableProfilerProps {
|
export interface TableProfilerProps {
|
||||||
@ -24,8 +24,20 @@ export interface TableProfilerProps {
|
|||||||
table: Table;
|
table: Table;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type TableTestsType = {
|
||||||
|
tests: TestCase[];
|
||||||
|
results: {
|
||||||
|
success: number;
|
||||||
|
aborted: number;
|
||||||
|
failed: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export type columnTestResultType = { [key: string]: TableTestsType['results'] };
|
||||||
|
|
||||||
export interface ColumnProfileTableProps {
|
export interface ColumnProfileTableProps {
|
||||||
columns: Column[];
|
columns: Column[];
|
||||||
|
columnTests: TestCase[];
|
||||||
onAddTestClick: (
|
onAddTestClick: (
|
||||||
tabValue: number,
|
tabValue: number,
|
||||||
testMode?: DatasetTestModeType,
|
testMode?: DatasetTestModeType,
|
||||||
@ -47,7 +59,7 @@ export interface ProfilerSettingsModalProps {
|
|||||||
|
|
||||||
export interface TestIndicatorProps {
|
export interface TestIndicatorProps {
|
||||||
value: number | string;
|
value: number | string;
|
||||||
type: TestCaseStatus;
|
type: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type OverallTableSummeryType = {
|
export type OverallTableSummeryType = {
|
||||||
|
@ -20,7 +20,7 @@ import {
|
|||||||
screen,
|
screen,
|
||||||
} from '@testing-library/react';
|
} from '@testing-library/react';
|
||||||
import React from 'react';
|
import React from 'react';
|
||||||
import { MOCK_TABLE } from '../../mocks/TableData.mock';
|
import { MOCK_TABLE, TEST_CASE } from '../../mocks/TableData.mock';
|
||||||
import { getCurrentDatasetTab } from '../../utils/DatasetDetailsUtils';
|
import { getCurrentDatasetTab } from '../../utils/DatasetDetailsUtils';
|
||||||
import { TableProfilerProps } from './TableProfiler.interface';
|
import { TableProfilerProps } from './TableProfiler.interface';
|
||||||
// internal imports
|
// internal imports
|
||||||
@ -69,6 +69,12 @@ jest.mock('../../utils/CommonUtils', () => ({
|
|||||||
}));
|
}));
|
||||||
const mockGetCurrentDatasetTab = getCurrentDatasetTab as jest.Mock;
|
const mockGetCurrentDatasetTab = getCurrentDatasetTab as jest.Mock;
|
||||||
|
|
||||||
|
jest.mock('../../axiosAPIs/testAPI', () => ({
|
||||||
|
getListTestCase: jest
|
||||||
|
.fn()
|
||||||
|
.mockImplementation(() => Promise.resolve(TEST_CASE)),
|
||||||
|
}));
|
||||||
|
|
||||||
const mockProps: TableProfilerProps = {
|
const mockProps: TableProfilerProps = {
|
||||||
table: MOCK_TABLE,
|
table: MOCK_TABLE,
|
||||||
onAddTestClick: jest.fn(),
|
onAddTestClick: jest.fn(),
|
||||||
@ -128,9 +134,6 @@ describe('Test TableProfiler component', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('CTA: Setting button should work properly', async () => {
|
it('CTA: Setting button should work properly', async () => {
|
||||||
const setSettingModalVisible = jest.fn();
|
|
||||||
const handleClick = jest.spyOn(React, 'useState');
|
|
||||||
handleClick.mockImplementation(() => [false, setSettingModalVisible]);
|
|
||||||
render(<TableProfilerV1 {...mockProps} />);
|
render(<TableProfilerV1 {...mockProps} />);
|
||||||
|
|
||||||
const settingBtn = await screen.findByTestId('profiler-setting-btn');
|
const settingBtn = await screen.findByTestId('profiler-setting-btn');
|
||||||
@ -141,6 +144,8 @@ describe('Test TableProfiler component', () => {
|
|||||||
fireEvent.click(settingBtn);
|
fireEvent.click(settingBtn);
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(setSettingModalVisible).toHaveBeenCalledTimes(1);
|
expect(
|
||||||
|
await screen.findByText('ProfilerSettingsModal.component')
|
||||||
|
).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -12,27 +12,43 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { Button, Col, Row } from 'antd';
|
import { Button, Col, Row } from 'antd';
|
||||||
|
import { AxiosError } from 'axios';
|
||||||
import classNames from 'classnames';
|
import classNames from 'classnames';
|
||||||
import { isUndefined } from 'lodash';
|
import { isEmpty, isUndefined } from 'lodash';
|
||||||
import React, { FC, useMemo, useState } from 'react';
|
import React, { FC, useEffect, useMemo, useState } from 'react';
|
||||||
import { Link } from 'react-router-dom';
|
import { Link } from 'react-router-dom';
|
||||||
|
import { getListTestCase } from '../../axiosAPIs/testAPI';
|
||||||
|
import { API_RES_MAX_SIZE } from '../../constants/constants';
|
||||||
|
import { INITIAL_TEST_RESULT_SUMMARY } from '../../constants/profiler.constant';
|
||||||
|
import { ProfilerDashboardType } from '../../enums/table.enum';
|
||||||
|
import { TestCase } from '../../generated/tests/testCase';
|
||||||
import {
|
import {
|
||||||
formatNumberWithComma,
|
formatNumberWithComma,
|
||||||
formTwoDigitNmber,
|
formTwoDigitNmber,
|
||||||
} from '../../utils/CommonUtils';
|
} from '../../utils/CommonUtils';
|
||||||
|
import { updateTestResults } from '../../utils/DataQualityAndProfilerUtils';
|
||||||
import { getCurrentDatasetTab } from '../../utils/DatasetDetailsUtils';
|
import { getCurrentDatasetTab } from '../../utils/DatasetDetailsUtils';
|
||||||
|
import { getProfilerDashboardWithFqnPath } from '../../utils/RouterUtils';
|
||||||
import SVGIcons, { Icons } from '../../utils/SvgUtils';
|
import SVGIcons, { Icons } from '../../utils/SvgUtils';
|
||||||
|
import { generateEntityLink } from '../../utils/TableUtils';
|
||||||
|
import { showErrorToast } from '../../utils/ToastUtils';
|
||||||
import ColumnProfileTable from './Component/ColumnProfileTable';
|
import ColumnProfileTable from './Component/ColumnProfileTable';
|
||||||
import ProfilerSettingsModal from './Component/ProfilerSettingsModal';
|
import ProfilerSettingsModal from './Component/ProfilerSettingsModal';
|
||||||
import {
|
import {
|
||||||
OverallTableSummeryType,
|
OverallTableSummeryType,
|
||||||
TableProfilerProps,
|
TableProfilerProps,
|
||||||
|
TableTestsType,
|
||||||
} from './TableProfiler.interface';
|
} from './TableProfiler.interface';
|
||||||
import './tableProfiler.less';
|
import './tableProfiler.less';
|
||||||
|
|
||||||
const TableProfilerV1: FC<TableProfilerProps> = ({ table, onAddTestClick }) => {
|
const TableProfilerV1: FC<TableProfilerProps> = ({ table, onAddTestClick }) => {
|
||||||
const { profile, columns } = table;
|
const { profile, columns } = table;
|
||||||
const [settingModalVisible, setSettingModalVisible] = useState(false);
|
const [settingModalVisible, setSettingModalVisible] = useState(false);
|
||||||
|
const [columnTests, setColumnTests] = useState<TestCase[]>([]);
|
||||||
|
const [tableTests, setTableTests] = useState<TableTestsType>({
|
||||||
|
tests: [],
|
||||||
|
results: INITIAL_TEST_RESULT_SUMMARY,
|
||||||
|
});
|
||||||
|
|
||||||
const handleSettingModal = (value: boolean) => {
|
const handleSettingModal = (value: boolean) => {
|
||||||
setSettingModalVisible(value);
|
setSettingModalVisible(value);
|
||||||
@ -53,21 +69,59 @@ const TableProfilerV1: FC<TableProfilerProps> = ({ table, onAddTestClick }) => {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: 'Success',
|
title: 'Success',
|
||||||
value: formTwoDigitNmber(0),
|
value: formTwoDigitNmber(tableTests.results.success),
|
||||||
className: 'success',
|
className: 'success',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: 'Aborted',
|
title: 'Aborted',
|
||||||
value: formTwoDigitNmber(0),
|
value: formTwoDigitNmber(tableTests.results.aborted),
|
||||||
className: 'aborted',
|
className: 'aborted',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: 'Failed',
|
title: 'Failed',
|
||||||
value: formTwoDigitNmber(0),
|
value: formTwoDigitNmber(tableTests.results.failed),
|
||||||
className: 'failed',
|
className: 'failed',
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}, [profile]);
|
}, [profile, tableTests]);
|
||||||
|
|
||||||
|
const fetchAllTests = async () => {
|
||||||
|
try {
|
||||||
|
const { data } = await getListTestCase({
|
||||||
|
fields: 'testCaseResult',
|
||||||
|
entityLink: generateEntityLink(table.fullyQualifiedName || ''),
|
||||||
|
includeAllTests: true,
|
||||||
|
limit: API_RES_MAX_SIZE,
|
||||||
|
});
|
||||||
|
const columnTestsCase: TestCase[] = [];
|
||||||
|
const tableTests: TableTestsType = {
|
||||||
|
tests: [],
|
||||||
|
results: { ...INITIAL_TEST_RESULT_SUMMARY },
|
||||||
|
};
|
||||||
|
data.forEach((test) => {
|
||||||
|
if (test.entityFQN === table.fullyQualifiedName) {
|
||||||
|
tableTests.tests.push(test);
|
||||||
|
|
||||||
|
updateTestResults(
|
||||||
|
tableTests.results,
|
||||||
|
test.testCaseResult?.testCaseStatus || ''
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
columnTestsCase.push(test);
|
||||||
|
});
|
||||||
|
setTableTests(tableTests);
|
||||||
|
setColumnTests(columnTestsCase);
|
||||||
|
} catch (error) {
|
||||||
|
showErrorToast(error as AxiosError);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (isEmpty(table)) return;
|
||||||
|
fetchAllTests();
|
||||||
|
}, [table]);
|
||||||
|
|
||||||
if (isUndefined(profile)) {
|
if (isUndefined(profile)) {
|
||||||
return (
|
return (
|
||||||
@ -134,9 +188,19 @@ const TableProfilerV1: FC<TableProfilerProps> = ({ table, onAddTestClick }) => {
|
|||||||
</p>
|
</p>
|
||||||
</Col>
|
</Col>
|
||||||
))}
|
))}
|
||||||
|
<Col className="tw-flex tw-justify-end" span={24}>
|
||||||
|
<Link
|
||||||
|
to={getProfilerDashboardWithFqnPath(
|
||||||
|
ProfilerDashboardType.TABLE,
|
||||||
|
table.fullyQualifiedName || ''
|
||||||
|
)}>
|
||||||
|
View more detail
|
||||||
|
</Link>
|
||||||
|
</Col>
|
||||||
</Row>
|
</Row>
|
||||||
|
|
||||||
<ColumnProfileTable
|
<ColumnProfileTable
|
||||||
|
columnTests={columnTests}
|
||||||
columns={columns.map((col) => ({
|
columns={columns.map((col) => ({
|
||||||
...col,
|
...col,
|
||||||
key: col.name,
|
key: col.name,
|
||||||
|
@ -20,6 +20,7 @@ export interface DeleteWidgetModalProps {
|
|||||||
entityType: string;
|
entityType: string;
|
||||||
isAdminUser?: boolean;
|
isAdminUser?: boolean;
|
||||||
entityId?: string;
|
entityId?: string;
|
||||||
|
prepareType?: boolean;
|
||||||
isRecursiveDelete?: boolean;
|
isRecursiveDelete?: boolean;
|
||||||
afterDeleteAction?: () => void;
|
afterDeleteAction?: () => void;
|
||||||
}
|
}
|
||||||
|
@ -35,6 +35,7 @@ const DeleteWidgetModal = ({
|
|||||||
entityType,
|
entityType,
|
||||||
onCancel,
|
onCancel,
|
||||||
entityId,
|
entityId,
|
||||||
|
prepareType = true,
|
||||||
isRecursiveDelete,
|
isRecursiveDelete,
|
||||||
afterDeleteAction,
|
afterDeleteAction,
|
||||||
}: DeleteWidgetModalProps) => {
|
}: DeleteWidgetModalProps) => {
|
||||||
@ -109,7 +110,7 @@ const DeleteWidgetModal = ({
|
|||||||
const handleOnEntityDeleteConfirm = () => {
|
const handleOnEntityDeleteConfirm = () => {
|
||||||
setEntityDeleteState((prev) => ({ ...prev, loading: 'waiting' }));
|
setEntityDeleteState((prev) => ({ ...prev, loading: 'waiting' }));
|
||||||
deleteEntity(
|
deleteEntity(
|
||||||
prepareEntityType(),
|
prepareType ? prepareEntityType() : entityType,
|
||||||
entityId ?? '',
|
entityId ?? '',
|
||||||
Boolean(isRecursiveDelete),
|
Boolean(isRecursiveDelete),
|
||||||
entityDeleteState.softDelete
|
entityDeleteState.softDelete
|
||||||
|
@ -93,6 +93,7 @@ export const PLACEHOLDER_SETTING_CATEGORY = ':settingCategory';
|
|||||||
export const PLACEHOLDER_USER_BOT = ':bot';
|
export const PLACEHOLDER_USER_BOT = ':bot';
|
||||||
export const PLACEHOLDER_WEBHOOK_TYPE = ':webhookType';
|
export const PLACEHOLDER_WEBHOOK_TYPE = ':webhookType';
|
||||||
export const PLACEHOLDER_RULE_NAME = ':ruleName';
|
export const PLACEHOLDER_RULE_NAME = ':ruleName';
|
||||||
|
export const PLACEHOLDER_DASHBOARD_TYPE = ':dashboardType';
|
||||||
|
|
||||||
export const pagingObject = { after: '', before: '', total: 0 };
|
export const pagingObject = { after: '', before: '', total: 0 };
|
||||||
|
|
||||||
@ -220,7 +221,7 @@ export const ROUTES = {
|
|||||||
MLMODEL_DETAILS_WITH_TAB: `/mlmodel/${PLACEHOLDER_ROUTE_MLMODEL_FQN}/${PLACEHOLDER_ROUTE_TAB}`,
|
MLMODEL_DETAILS_WITH_TAB: `/mlmodel/${PLACEHOLDER_ROUTE_MLMODEL_FQN}/${PLACEHOLDER_ROUTE_TAB}`,
|
||||||
CUSTOM_ENTITY_DETAIL: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
CUSTOM_ENTITY_DETAIL: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
||||||
ADD_CUSTOM_PROPERTY: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}/add-field`,
|
ADD_CUSTOM_PROPERTY: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}/add-field`,
|
||||||
PROFILER_DASHBOARD: `/profiler-dashboard/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
PROFILER_DASHBOARD: `/profiler-dashboard/${PLACEHOLDER_DASHBOARD_TYPE}/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
||||||
|
|
||||||
// Tasks Routes
|
// Tasks Routes
|
||||||
REQUEST_DESCRIPTION: `/request-description/${PLACEHOLDER_ROUTE_ENTITY_TYPE}/${PLACEHOLDER_ROUTE_ENTITY_FQN}`,
|
REQUEST_DESCRIPTION: `/request-description/${PLACEHOLDER_ROUTE_ENTITY_TYPE}/${PLACEHOLDER_ROUTE_ENTITY_FQN}`,
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
|
|
||||||
import { CSMode } from '../enums/codemirror.enum';
|
import { CSMode } from '../enums/codemirror.enum';
|
||||||
import { ColumnProfilerConfig } from '../generated/entity/data/table';
|
import { ColumnProfilerConfig } from '../generated/entity/data/table';
|
||||||
|
import { TestCaseStatus } from '../generated/tests/tableTest';
|
||||||
import { JSON_TAB_SIZE } from './constants';
|
import { JSON_TAB_SIZE } from './constants';
|
||||||
|
|
||||||
export const excludedMetrics = [
|
export const excludedMetrics = [
|
||||||
@ -63,6 +64,8 @@ export const PROFILER_FILTER_RANGE = {
|
|||||||
last60days: { days: 60, title: 'Last 60 days' },
|
last60days: { days: 60, title: 'Last 60 days' },
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const COLORS = ['#7147E8', '#B02AAC', '#B02AAC', '#1890FF', '#008376'];
|
||||||
|
|
||||||
export const DEFAULT_CHART_COLLECTION_VALUE = {
|
export const DEFAULT_CHART_COLLECTION_VALUE = {
|
||||||
distinctCount: { data: [], color: '#1890FF' },
|
distinctCount: { data: [], color: '#1890FF' },
|
||||||
uniqueCount: { data: [], color: '#008376' },
|
uniqueCount: { data: [], color: '#008376' },
|
||||||
@ -161,6 +164,27 @@ export const DEFAULT_INCLUDE_PROFILE: ColumnProfilerConfig[] = [
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
export const INITIAL_TEST_RESULT_SUMMARY = {
|
||||||
|
success: 0,
|
||||||
|
aborted: 0,
|
||||||
|
failed: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
export const DEFAULT_TEST_VALUE = [
|
||||||
|
{
|
||||||
|
value: 0,
|
||||||
|
type: TestCaseStatus.Success,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
value: 0,
|
||||||
|
type: TestCaseStatus.Aborted,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
value: 0,
|
||||||
|
type: TestCaseStatus.Failed,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
export const codeMirrorOption = {
|
export const codeMirrorOption = {
|
||||||
tabSize: JSON_TAB_SIZE,
|
tabSize: JSON_TAB_SIZE,
|
||||||
indentUnit: JSON_TAB_SIZE,
|
indentUnit: JSON_TAB_SIZE,
|
||||||
|
@ -26,3 +26,8 @@ export enum PrimaryTableDataTypes {
|
|||||||
NUMERIC = 'numeric',
|
NUMERIC = 'numeric',
|
||||||
BOOLEAN = 'boolean',
|
BOOLEAN = 'boolean',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export enum ProfilerDashboardType {
|
||||||
|
TABLE = 'table',
|
||||||
|
COLUMN = 'column',
|
||||||
|
}
|
||||||
|
@ -111,6 +111,7 @@ const jsonData = {
|
|||||||
'fetch-users-error': 'Error while fetching users!',
|
'fetch-users-error': 'Error while fetching users!',
|
||||||
'fetch-table-profiler-config-error':
|
'fetch-table-profiler-config-error':
|
||||||
'Error while fetching table profiler config!',
|
'Error while fetching table profiler config!',
|
||||||
|
'fetch-column-test-error': 'Error while fetching column test case!',
|
||||||
|
|
||||||
'test-connection-error': 'Error while testing connection!',
|
'test-connection-error': 'Error while testing connection!',
|
||||||
|
|
||||||
|
@ -201,3 +201,71 @@ export const MOCK_TABLE = {
|
|||||||
],
|
],
|
||||||
deleted: false,
|
deleted: false,
|
||||||
} as unknown as Table;
|
} as unknown as Table;
|
||||||
|
|
||||||
|
export const TEST_CASE = {
|
||||||
|
data: [
|
||||||
|
{
|
||||||
|
id: 'b9d059d8-b968-42ad-9f89-2b40b92a6659',
|
||||||
|
name: 'column_value_max_to_be_between',
|
||||||
|
fullyQualifiedName:
|
||||||
|
'sample_data.ecommerce_db.shopify.dim_address.shop_id.column_value_max_to_be_between',
|
||||||
|
description: 'test the value of a column is between x and z, new value',
|
||||||
|
testDefinition: {
|
||||||
|
id: '16b32e12-21c5-491c-919e-88748d9d5d67',
|
||||||
|
type: 'testDefinition',
|
||||||
|
name: 'columnValueMaxToBeBetween',
|
||||||
|
fullyQualifiedName: 'columnValueMaxToBeBetween',
|
||||||
|
description:
|
||||||
|
'This schema defines the test ColumnValueMaxToBeBetween. Test the maximum value in a col is within a range.',
|
||||||
|
displayName: 'columnValueMaxToBeBetween',
|
||||||
|
deleted: false,
|
||||||
|
href: 'http://localhost:8585/api/v1/testDefinition/16b32e12-21c5-491c-919e-88748d9d5d67',
|
||||||
|
},
|
||||||
|
entityLink:
|
||||||
|
'<#E::table::sample_data.ecommerce_db.shopify.dim_address::columns::shop_id>',
|
||||||
|
entityFQN: 'sample_data.ecommerce_db.shopify.dim_address.shop_id',
|
||||||
|
parameterValues: [
|
||||||
|
{
|
||||||
|
name: 'minValueForMaxInCol',
|
||||||
|
value: '40',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'maxValueForMaxInCol',
|
||||||
|
value: '100',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
testCaseResult: {
|
||||||
|
timestamp: 1661416859,
|
||||||
|
testCaseStatus: 'Success',
|
||||||
|
result: 'Found max=65 vs. the expected min=50, max=100.',
|
||||||
|
testResultValue: [
|
||||||
|
{
|
||||||
|
name: 'max',
|
||||||
|
value: '65',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
version: 0.3,
|
||||||
|
updatedAt: 1661425991294,
|
||||||
|
updatedBy: 'anonymous',
|
||||||
|
href: 'http://localhost:8585/api/v1/testCase/b9d059d8-b968-42ad-9f89-2b40b92a6659',
|
||||||
|
changeDescription: {
|
||||||
|
fieldsAdded: [],
|
||||||
|
fieldsUpdated: [
|
||||||
|
{
|
||||||
|
name: 'description',
|
||||||
|
oldValue: 'test the value of a column is between x and y',
|
||||||
|
newValue:
|
||||||
|
'test the value of a column is between x and z, new value',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
fieldsDeleted: [],
|
||||||
|
previousVersion: 0.2,
|
||||||
|
},
|
||||||
|
deleted: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
paging: {
|
||||||
|
total: 1,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
@ -21,32 +21,40 @@ import {
|
|||||||
getTableDetailsByFQN,
|
getTableDetailsByFQN,
|
||||||
patchTableDetails,
|
patchTableDetails,
|
||||||
} from '../../axiosAPIs/tableAPI';
|
} from '../../axiosAPIs/tableAPI';
|
||||||
|
import { getListTestCase } from '../../axiosAPIs/testAPI';
|
||||||
import ErrorPlaceHolder from '../../components/common/error-with-placeholder/ErrorPlaceHolder';
|
import ErrorPlaceHolder from '../../components/common/error-with-placeholder/ErrorPlaceHolder';
|
||||||
import PageContainerV1 from '../../components/containers/PageContainerV1';
|
import PageContainerV1 from '../../components/containers/PageContainerV1';
|
||||||
import Loader from '../../components/Loader/Loader';
|
import Loader from '../../components/Loader/Loader';
|
||||||
import ProfilerDashboard from '../../components/ProfilerDashboard/ProfilerDashboard';
|
import ProfilerDashboard from '../../components/ProfilerDashboard/ProfilerDashboard';
|
||||||
import { API_RES_MAX_SIZE } from '../../constants/constants';
|
import { API_RES_MAX_SIZE } from '../../constants/constants';
|
||||||
|
import { ProfilerDashboardType } from '../../enums/table.enum';
|
||||||
import { ColumnProfile, Table } from '../../generated/entity/data/table';
|
import { ColumnProfile, Table } from '../../generated/entity/data/table';
|
||||||
|
import { TestCase } from '../../generated/tests/testCase';
|
||||||
import jsonData from '../../jsons/en';
|
import jsonData from '../../jsons/en';
|
||||||
import {
|
import {
|
||||||
getNameFromFQN,
|
getNameFromFQN,
|
||||||
getTableFQNFromColumnFQN,
|
getTableFQNFromColumnFQN,
|
||||||
} from '../../utils/CommonUtils';
|
} from '../../utils/CommonUtils';
|
||||||
|
import { generateEntityLink } from '../../utils/TableUtils';
|
||||||
import { showErrorToast } from '../../utils/ToastUtils';
|
import { showErrorToast } from '../../utils/ToastUtils';
|
||||||
|
|
||||||
const ProfilerDashboardPage = () => {
|
const ProfilerDashboardPage = () => {
|
||||||
const { entityTypeFQN } = useParams<Record<string, string>>();
|
const { entityTypeFQN, dashboardType } = useParams<Record<string, string>>();
|
||||||
|
const isColumnView = dashboardType === ProfilerDashboardType.COLUMN;
|
||||||
const [table, setTable] = useState<Table>({} as Table);
|
const [table, setTable] = useState<Table>({} as Table);
|
||||||
const [profilerData, setProfilerData] = useState<ColumnProfile[]>([]);
|
const [profilerData, setProfilerData] = useState<ColumnProfile[]>([]);
|
||||||
const [isLoading, setIsLoading] = useState(true);
|
const [isLoading, setIsLoading] = useState(true);
|
||||||
const [error, setError] = useState(false);
|
const [error, setError] = useState(false);
|
||||||
|
const [testCases, setTestCases] = useState<TestCase[]>([]);
|
||||||
|
|
||||||
const fetchProfilerData = async (fqn: string, days = 3) => {
|
const fetchProfilerData = async (fqn: string, days = 3) => {
|
||||||
try {
|
try {
|
||||||
const startTs = moment().subtract(days, 'days').unix();
|
const startTs = moment().subtract(days, 'days').unix();
|
||||||
|
const endTs = moment().unix();
|
||||||
|
|
||||||
const { data } = await getColumnProfilerList(fqn, {
|
const { data } = await getColumnProfilerList(fqn, {
|
||||||
startTs: startTs,
|
startTs,
|
||||||
|
endTs,
|
||||||
limit: API_RES_MAX_SIZE,
|
limit: API_RES_MAX_SIZE,
|
||||||
});
|
});
|
||||||
setProfilerData(data || []);
|
setProfilerData(data || []);
|
||||||
@ -57,15 +65,39 @@ const ProfilerDashboardPage = () => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const fetchTableEntity = async (fqn: string) => {
|
const fetchTestCases = async (fqn: string) => {
|
||||||
try {
|
try {
|
||||||
getTableFQNFromColumnFQN(fqn);
|
const { data } = await getListTestCase({
|
||||||
const data = await getTableDetailsByFQN(
|
fields: 'testDefinition,testCaseResult',
|
||||||
getTableFQNFromColumnFQN(fqn),
|
entityLink: fqn,
|
||||||
'tags, usageSummary, owner, followers, profile'
|
limit: API_RES_MAX_SIZE,
|
||||||
|
});
|
||||||
|
setTestCases(data);
|
||||||
|
} catch (error) {
|
||||||
|
showErrorToast(
|
||||||
|
error as AxiosError,
|
||||||
|
jsonData['api-error-messages']['fetch-column-test-error']
|
||||||
);
|
);
|
||||||
|
} finally {
|
||||||
|
setIsLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const fetchTableEntity = async () => {
|
||||||
|
try {
|
||||||
|
const fqn = isColumnView
|
||||||
|
? getTableFQNFromColumnFQN(entityTypeFQN)
|
||||||
|
: entityTypeFQN;
|
||||||
|
const field = `tags, usageSummary, owner, followers${
|
||||||
|
isColumnView ? ', profile' : ''
|
||||||
|
}`;
|
||||||
|
const data = await getTableDetailsByFQN(fqn, field);
|
||||||
setTable(data ?? ({} as Table));
|
setTable(data ?? ({} as Table));
|
||||||
|
if (isColumnView) {
|
||||||
fetchProfilerData(entityTypeFQN);
|
fetchProfilerData(entityTypeFQN);
|
||||||
|
} else {
|
||||||
|
fetchTestCases(generateEntityLink(entityTypeFQN));
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
showErrorToast(
|
showErrorToast(
|
||||||
error as AxiosError,
|
error as AxiosError,
|
||||||
@ -92,7 +124,7 @@ const ProfilerDashboardPage = () => {
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (entityTypeFQN) {
|
if (entityTypeFQN) {
|
||||||
fetchTableEntity(entityTypeFQN);
|
fetchTableEntity();
|
||||||
} else {
|
} else {
|
||||||
setIsLoading(false);
|
setIsLoading(false);
|
||||||
setError(true);
|
setError(true);
|
||||||
@ -118,8 +150,10 @@ const ProfilerDashboardPage = () => {
|
|||||||
<PageContainerV1 className="tw-py-4">
|
<PageContainerV1 className="tw-py-4">
|
||||||
<ProfilerDashboard
|
<ProfilerDashboard
|
||||||
fetchProfilerData={fetchProfilerData}
|
fetchProfilerData={fetchProfilerData}
|
||||||
|
fetchTestCases={fetchTestCases}
|
||||||
profilerData={profilerData}
|
profilerData={profilerData}
|
||||||
table={table}
|
table={table}
|
||||||
|
testCases={testCases}
|
||||||
onTableChange={updateTableHandler}
|
onTableChange={updateTableHandler}
|
||||||
/>
|
/>
|
||||||
</PageContainerV1>
|
</PageContainerV1>
|
||||||
|
@ -0,0 +1,35 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2022 Collate
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { TableTestsType } from '../components/TableProfiler/TableProfiler.interface';
|
||||||
|
import { TestCaseStatus } from '../generated/tests/tableTest';
|
||||||
|
|
||||||
|
export const updateTestResults = (
|
||||||
|
results: TableTestsType['results'],
|
||||||
|
testCaseStatus: string
|
||||||
|
) => {
|
||||||
|
switch (testCaseStatus) {
|
||||||
|
case TestCaseStatus.Success:
|
||||||
|
results.success += 1;
|
||||||
|
|
||||||
|
break;
|
||||||
|
case TestCaseStatus.Failed:
|
||||||
|
results.failed += 1;
|
||||||
|
|
||||||
|
break;
|
||||||
|
case TestCaseStatus.Aborted:
|
||||||
|
results.aborted += 1;
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
};
|
@ -14,6 +14,7 @@
|
|||||||
import { FQN_SEPARATOR_CHAR } from '../constants/char.constants';
|
import { FQN_SEPARATOR_CHAR } from '../constants/char.constants';
|
||||||
import {
|
import {
|
||||||
IN_PAGE_SEARCH_ROUTES,
|
IN_PAGE_SEARCH_ROUTES,
|
||||||
|
PLACEHOLDER_DASHBOARD_TYPE,
|
||||||
PLACEHOLDER_ENTITY_TYPE_FQN,
|
PLACEHOLDER_ENTITY_TYPE_FQN,
|
||||||
PLACEHOLDER_GLOSSARY_NAME,
|
PLACEHOLDER_GLOSSARY_NAME,
|
||||||
PLACEHOLDER_GLOSSARY_TERMS_FQN,
|
PLACEHOLDER_GLOSSARY_TERMS_FQN,
|
||||||
@ -289,10 +290,15 @@ export const getPath = (pathName: string) => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getProfilerDashboardWithFqnPath = (entityTypeFQN: string) => {
|
export const getProfilerDashboardWithFqnPath = (
|
||||||
|
dashboardType: string,
|
||||||
|
entityTypeFQN: string
|
||||||
|
) => {
|
||||||
let path = ROUTES.PROFILER_DASHBOARD;
|
let path = ROUTES.PROFILER_DASHBOARD;
|
||||||
|
|
||||||
path = path.replace(PLACEHOLDER_ENTITY_TYPE_FQN, entityTypeFQN);
|
path = path
|
||||||
|
.replace(PLACEHOLDER_DASHBOARD_TYPE, dashboardType)
|
||||||
|
.replace(PLACEHOLDER_ENTITY_TYPE_FQN, entityTypeFQN);
|
||||||
|
|
||||||
return path;
|
return path;
|
||||||
};
|
};
|
||||||
|
@ -37,6 +37,7 @@ import { Column, DataType } from '../generated/entity/data/table';
|
|||||||
import { TableTest, TestCaseStatus } from '../generated/tests/tableTest';
|
import { TableTest, TestCaseStatus } from '../generated/tests/tableTest';
|
||||||
import { TagLabel } from '../generated/type/tagLabel';
|
import { TagLabel } from '../generated/type/tagLabel';
|
||||||
import { ModifiedTableColumn } from '../interface/dataQuality.interface';
|
import { ModifiedTableColumn } from '../interface/dataQuality.interface';
|
||||||
|
import { getNameFromFQN, getTableFQNFromColumnFQN } from './CommonUtils';
|
||||||
import { getGlossaryPath } from './RouterUtils';
|
import { getGlossaryPath } from './RouterUtils';
|
||||||
import { ordinalize } from './StringsUtils';
|
import { ordinalize } from './StringsUtils';
|
||||||
import SVGIcons, { Icons } from './SvgUtils';
|
import SVGIcons, { Icons } from './SvgUtils';
|
||||||
@ -306,6 +307,38 @@ export const getDataTypeString = (dataType: string): string => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const generateEntityLink = (fqn: string, includeColumn = false) => {
|
||||||
|
const columnLink = '<#E::table::ENTITY_FQN::columns::COLUMN>';
|
||||||
|
const tableLink = '<#E::table::ENTITY_FQN>';
|
||||||
|
|
||||||
|
if (includeColumn) {
|
||||||
|
const tableFqn = getTableFQNFromColumnFQN(fqn);
|
||||||
|
const columnName = getNameFromFQN(fqn);
|
||||||
|
|
||||||
|
return columnLink
|
||||||
|
.replace('ENTITY_FQN', tableFqn)
|
||||||
|
.replace('COLUMN', columnName);
|
||||||
|
} else {
|
||||||
|
return tableLink.replace('ENTITY_FQN', fqn);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getTestResultBadgeIcon = (status?: TestCaseStatus) => {
|
||||||
|
switch (status) {
|
||||||
|
case TestCaseStatus.Success:
|
||||||
|
return Icons.SUCCESS_BADGE;
|
||||||
|
|
||||||
|
case TestCaseStatus.Failed:
|
||||||
|
return Icons.FAIL_BADGE;
|
||||||
|
|
||||||
|
case TestCaseStatus.Aborted:
|
||||||
|
return Icons.PENDING_BADGE;
|
||||||
|
|
||||||
|
default:
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
export const getTableTestsValue = (tableTestCase: TableTest[]) => {
|
export const getTableTestsValue = (tableTestCase: TableTest[]) => {
|
||||||
const tableTestLength = tableTestCase.length;
|
const tableTestLength = tableTestCase.length;
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user