mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-24 08:58:06 +00:00
Feat: Column level profiler dashboard (#6683)
* added column dashboard page and route * added redirection from table to profiler dashboard * fetch the requierd information from api * added loader and error placeholder * added profiler dashboard component * added table summery in column profiler page * added tab section * added range dropdown and add test button * added chart for column level dashboard * added formatting for chart and data * extracted profiler tab as componene * added table level profiler card * added indicator for test summery * added description summary card * added latest profiler filter card * removed bottom border and added padding-x for table cell * show laast 7 days opption * added content on tab switch for profiler and data quality * remove loading state and updated summary spelling * removed latest option from profiler dropdown and relavent code * added more metrics in the graph and made default dropdown option as last 3 days * change style of Legend component * fixed failing test * restrict float number to show only 2 decimal * addressing comments * fixed failing unit test
This commit is contained in:
Shailesh Parmar
GitHub
parent
36338eb727
commit
bbe424dbed
@ -18,11 +18,13 @@ import { CreateTableTest } from '../generated/api/tests/createTableTest';
|
||||
import {
|
||||
ColumnTestType,
|
||||
Table,
|
||||
TableProfile,
|
||||
TableProfilerConfig,
|
||||
} from '../generated/entity/data/table';
|
||||
import { TableTestType } from '../generated/tests/tableTest';
|
||||
import { EntityHistory } from '../generated/type/entityHistory';
|
||||
import { EntityReference } from '../generated/type/entityReference';
|
||||
import { Paging } from '../generated/type/paging';
|
||||
import { getURLWithQueryFields } from '../utils/APIUtils';
|
||||
import APIClient from './index';
|
||||
|
||||
@ -212,3 +214,23 @@ export const putTableProfileConfig = async (
|
||||
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const getTableProfilesList = async (
|
||||
tableId: string,
|
||||
params?: {
|
||||
startTs?: number;
|
||||
endTs?: number;
|
||||
limit?: number;
|
||||
before?: string;
|
||||
after?: string;
|
||||
}
|
||||
) => {
|
||||
const url = `/tables/${tableId}/tableProfile`;
|
||||
|
||||
const response = await APIClient.get<{
|
||||
data: TableProfile[];
|
||||
paging: Paging;
|
||||
}>(url, { params });
|
||||
|
||||
return response.data;
|
||||
};
|
||||
|
||||
@ -0,0 +1,353 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Button, Col, Radio, Row, Select, Space } from 'antd';
|
||||
import { RadioChangeEvent } from 'antd/lib/radio';
|
||||
import { AxiosError } from 'axios';
|
||||
import { EntityTags, ExtraInfo } from 'Models';
|
||||
import React, { useEffect, useMemo, useState } from 'react';
|
||||
import { useHistory, useParams } from 'react-router-dom';
|
||||
import { addFollower, removeFollower } from '../../axiosAPIs/tableAPI';
|
||||
import { FQN_SEPARATOR_CHAR } from '../../constants/char.constants';
|
||||
import {
|
||||
getDatabaseDetailsPath,
|
||||
getDatabaseSchemaDetailsPath,
|
||||
getServiceDetailsPath,
|
||||
getTableTabPath,
|
||||
getTeamAndUserDetailsPath,
|
||||
} from '../../constants/constants';
|
||||
import { PROFILER_FILTER_RANGE } from '../../constants/profiler.constant';
|
||||
import { EntityType, FqnPart } from '../../enums/entity.enum';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import { OwnerType } from '../../enums/user.enum';
|
||||
import { Column, Table } from '../../generated/entity/data/table';
|
||||
import { EntityReference } from '../../generated/type/entityReference';
|
||||
import { LabelType, State } from '../../generated/type/tagLabel';
|
||||
import jsonData from '../../jsons/en';
|
||||
import {
|
||||
getCurrentUserId,
|
||||
getEntityName,
|
||||
getEntityPlaceHolder,
|
||||
getNameFromFQN,
|
||||
getPartialNameFromTableFQN,
|
||||
hasEditAccess,
|
||||
} from '../../utils/CommonUtils';
|
||||
import { serviceTypeLogo } from '../../utils/ServiceUtils';
|
||||
import {
|
||||
getTagsWithoutTier,
|
||||
getTierTags,
|
||||
getUsagePercentile,
|
||||
} from '../../utils/TableUtils';
|
||||
import { showErrorToast } from '../../utils/ToastUtils';
|
||||
import EntityPageInfo from '../common/entityPageInfo/EntityPageInfo';
|
||||
import PageLayout from '../containers/PageLayout';
|
||||
import ProfilerTab from './component/ProfilerTab';
|
||||
import {
|
||||
ProfilerDashboardProps,
|
||||
ProfilerDashboardTab,
|
||||
} from './profilerDashboard.interface';
|
||||
import './profilerDashboard.less';
|
||||
|
||||
const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
|
||||
table,
|
||||
fetchProfilerData,
|
||||
profilerData,
|
||||
onTableChange,
|
||||
}) => {
|
||||
const history = useHistory();
|
||||
const { entityTypeFQN } = useParams<Record<string, string>>();
|
||||
const [follower, setFollower] = useState<EntityReference[]>([]);
|
||||
const [isFollowing, setIsFollowing] = useState<boolean>(false);
|
||||
const [activeTab, setActiveTab] = useState<ProfilerDashboardTab>(
|
||||
ProfilerDashboardTab.PROFILER
|
||||
);
|
||||
const [selectedTimeRange, setSelectedTimeRange] =
|
||||
useState<keyof typeof PROFILER_FILTER_RANGE>('last3days');
|
||||
const [activeColumnDetails, setActiveColumnDetails] = useState<Column>(
|
||||
{} as Column
|
||||
);
|
||||
|
||||
const timeRangeOption = useMemo(() => {
|
||||
return Object.entries(PROFILER_FILTER_RANGE).map(([key, value]) => ({
|
||||
label: value.title,
|
||||
value: key,
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const tier = useMemo(() => getTierTags(table.tags ?? []), [table]);
|
||||
const breadcrumb = useMemo(() => {
|
||||
const serviceName = getEntityName(table.service);
|
||||
const fqn = table.fullyQualifiedName || '';
|
||||
const columnName = getNameFromFQN(entityTypeFQN);
|
||||
|
||||
return [
|
||||
{
|
||||
name: getEntityName(table.service),
|
||||
url: serviceName
|
||||
? getServiceDetailsPath(
|
||||
serviceName,
|
||||
ServiceCategory.DATABASE_SERVICES
|
||||
)
|
||||
: '',
|
||||
imgSrc: table.serviceType
|
||||
? serviceTypeLogo(table.serviceType)
|
||||
: undefined,
|
||||
},
|
||||
{
|
||||
name: getPartialNameFromTableFQN(fqn, [FqnPart.Database]),
|
||||
url: getDatabaseDetailsPath(fqn),
|
||||
},
|
||||
{
|
||||
name: getPartialNameFromTableFQN(fqn, [FqnPart.Schema]),
|
||||
url: getDatabaseSchemaDetailsPath(fqn),
|
||||
},
|
||||
{
|
||||
name: getEntityName(table),
|
||||
url: getTableTabPath(table.fullyQualifiedName || ''),
|
||||
},
|
||||
{
|
||||
name: columnName,
|
||||
url: '',
|
||||
activeTitle: true,
|
||||
},
|
||||
];
|
||||
}, [table]);
|
||||
|
||||
const extraInfo: Array<ExtraInfo> = useMemo(() => {
|
||||
return [
|
||||
{
|
||||
key: 'Owner',
|
||||
value:
|
||||
table.owner?.type === OwnerType.TEAM
|
||||
? getTeamAndUserDetailsPath(table.owner?.name || '')
|
||||
: getEntityName(table.owner),
|
||||
placeholderText: getEntityPlaceHolder(
|
||||
getEntityName(table.owner),
|
||||
table.owner?.deleted
|
||||
),
|
||||
isLink: table.owner?.type === OwnerType.TEAM,
|
||||
openInNewTab: false,
|
||||
profileName:
|
||||
table.owner?.type === OwnerType.USER ? table.owner?.name : undefined,
|
||||
},
|
||||
{
|
||||
key: 'Tier',
|
||||
value: tier?.tagFQN ? tier.tagFQN.split(FQN_SEPARATOR_CHAR)[1] : '',
|
||||
},
|
||||
{ key: 'Type', value: `${table.tableType}`, showLabel: true },
|
||||
{
|
||||
value:
|
||||
getUsagePercentile(
|
||||
table.usageSummary?.weeklyStats?.percentileRank || 0,
|
||||
true
|
||||
) || '--',
|
||||
},
|
||||
{
|
||||
value: `${
|
||||
table.usageSummary?.weeklyStats?.count.toLocaleString() || '--'
|
||||
} queries`,
|
||||
},
|
||||
];
|
||||
}, [table]);
|
||||
|
||||
const handleOwnerUpdate = (newOwner?: Table['owner']) => {
|
||||
if (newOwner) {
|
||||
const updatedTableDetails = {
|
||||
...table,
|
||||
owner: {
|
||||
...table.owner,
|
||||
...newOwner,
|
||||
},
|
||||
};
|
||||
onTableChange(updatedTableDetails);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTierUpdate = (newTier?: string) => {
|
||||
if (newTier) {
|
||||
const tierTag: Table['tags'] = newTier
|
||||
? [
|
||||
...getTagsWithoutTier(table.tags as Array<EntityTags>),
|
||||
{
|
||||
tagFQN: newTier,
|
||||
labelType: LabelType.Manual,
|
||||
state: State.Confirmed,
|
||||
},
|
||||
]
|
||||
: table.tags;
|
||||
const updatedTableDetails = {
|
||||
...table,
|
||||
tags: tierTag,
|
||||
};
|
||||
|
||||
return onTableChange(updatedTableDetails);
|
||||
} else {
|
||||
return Promise.reject();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Formulates updated tags and updates table entity data for API call
|
||||
* @param selectedTags
|
||||
*/
|
||||
const handleTagUpdate = (selectedTags?: Array<EntityTags>) => {
|
||||
if (selectedTags) {
|
||||
const updatedTags = [...(tier ? [tier] : []), ...selectedTags];
|
||||
const updatedTable = { ...table, tags: updatedTags };
|
||||
onTableChange(updatedTable);
|
||||
}
|
||||
};
|
||||
|
||||
const unfollowTable = async () => {
|
||||
try {
|
||||
const data = await removeFollower(table.id, getCurrentUserId());
|
||||
const { oldValue } = data.changeDescription.fieldsDeleted[0];
|
||||
|
||||
setFollower(
|
||||
follower.filter((follower) => follower.id !== oldValue[0].id)
|
||||
);
|
||||
} catch (error) {
|
||||
showErrorToast(
|
||||
error as AxiosError,
|
||||
jsonData['api-error-messages']['update-entity-unfollow-error']
|
||||
);
|
||||
}
|
||||
};
|
||||
const followTable = async () => {
|
||||
try {
|
||||
const data = await addFollower(table.id, getCurrentUserId());
|
||||
const { newValue } = data.changeDescription.fieldsAdded[0];
|
||||
|
||||
setFollower([...follower, ...newValue]);
|
||||
} catch (error) {
|
||||
showErrorToast(
|
||||
error as AxiosError,
|
||||
jsonData['api-error-messages']['update-entity-follow-error']
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const handleFollowClick = () => {
|
||||
if (isFollowing) {
|
||||
setIsFollowing(false);
|
||||
unfollowTable();
|
||||
} else {
|
||||
setIsFollowing(true);
|
||||
followTable();
|
||||
}
|
||||
};
|
||||
|
||||
const handleTabChange = (e: RadioChangeEvent) => {
|
||||
const value = e.target.value as ProfilerDashboardTab;
|
||||
if (ProfilerDashboardTab.SUMMARY === value) {
|
||||
history.push(getTableTabPath(table.fullyQualifiedName || '', 'profiler'));
|
||||
}
|
||||
setActiveTab(value);
|
||||
};
|
||||
|
||||
const handleAddTestClick = () => {
|
||||
history.push(
|
||||
getTableTabPath(table.fullyQualifiedName || '', 'data-quality')
|
||||
);
|
||||
};
|
||||
|
||||
const handleTimeRangeChange = (value: keyof typeof PROFILER_FILTER_RANGE) => {
|
||||
if (value !== selectedTimeRange) {
|
||||
setSelectedTimeRange(value);
|
||||
fetchProfilerData(table.id, PROFILER_FILTER_RANGE[value].days);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (table) {
|
||||
const columnName = getNameFromFQN(entityTypeFQN);
|
||||
const selectedColumn = table.columns.find(
|
||||
(col) => col.name === columnName
|
||||
);
|
||||
setActiveColumnDetails(selectedColumn || ({} as Column));
|
||||
setFollower(table?.followers || []);
|
||||
setIsFollowing(
|
||||
follower.some(({ id }: { id: string }) => id === getCurrentUserId())
|
||||
);
|
||||
}
|
||||
}, [table]);
|
||||
|
||||
return (
|
||||
<PageLayout>
|
||||
<Row gutter={[16, 16]}>
|
||||
<Col span={24}>
|
||||
<EntityPageInfo
|
||||
isTagEditable
|
||||
deleted={table.deleted}
|
||||
entityFqn={table.fullyQualifiedName}
|
||||
entityId={table.id}
|
||||
entityName={table.name}
|
||||
entityType={EntityType.TABLE}
|
||||
extraInfo={extraInfo}
|
||||
followHandler={handleFollowClick}
|
||||
followers={follower.length}
|
||||
followersList={follower}
|
||||
hasEditAccess={hasEditAccess(
|
||||
table.owner?.type || '',
|
||||
table.owner?.id || ''
|
||||
)}
|
||||
isFollowing={isFollowing}
|
||||
tags={getTagsWithoutTier(table.tags || [])}
|
||||
tagsHandler={handleTagUpdate}
|
||||
tier={tier}
|
||||
titleLinks={breadcrumb}
|
||||
updateOwner={handleOwnerUpdate}
|
||||
updateTier={handleTierUpdate}
|
||||
/>
|
||||
</Col>
|
||||
<Col span={24}>
|
||||
<Row justify="space-between">
|
||||
<Radio.Group
|
||||
buttonStyle="solid"
|
||||
optionType="button"
|
||||
options={Object.values(ProfilerDashboardTab)}
|
||||
value={activeTab}
|
||||
onChange={handleTabChange}
|
||||
/>
|
||||
|
||||
<Space size={16}>
|
||||
<Select
|
||||
className="tw-w-32"
|
||||
options={timeRangeOption}
|
||||
value={selectedTimeRange}
|
||||
onChange={handleTimeRangeChange}
|
||||
/>
|
||||
<Button type="primary" onClick={handleAddTestClick}>
|
||||
Add Test
|
||||
</Button>
|
||||
</Space>
|
||||
</Row>
|
||||
</Col>
|
||||
{activeTab === ProfilerDashboardTab.PROFILER && (
|
||||
<Col span={24}>
|
||||
<ProfilerTab
|
||||
activeColumnDetails={activeColumnDetails}
|
||||
profilerData={profilerData}
|
||||
/>
|
||||
</Col>
|
||||
)}
|
||||
|
||||
{activeTab === ProfilerDashboardTab.DATA_QUALITY && (
|
||||
<Col span={24}>Data Quality</Col>
|
||||
)}
|
||||
</Row>
|
||||
</PageLayout>
|
||||
);
|
||||
};
|
||||
|
||||
export default ProfilerDashboard;
|
||||
@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Card, Col, Empty, Row, Space, Statistic } from 'antd';
|
||||
import React from 'react';
|
||||
import {
|
||||
Legend,
|
||||
LegendValueFormatter,
|
||||
Line,
|
||||
LineChart,
|
||||
ResponsiveContainer,
|
||||
Tooltip,
|
||||
XAxis,
|
||||
YAxis,
|
||||
} from 'recharts';
|
||||
import { ProfilerDetailsCardProps } from '../profilerDashboard.interface';
|
||||
|
||||
const ProfilerDetailsCard: React.FC<ProfilerDetailsCardProps> = ({
|
||||
chartCollection,
|
||||
tickFormatter,
|
||||
}) => {
|
||||
const { data, information } = chartCollection;
|
||||
|
||||
const renderColorfulLegendText: LegendValueFormatter = (value, entry) => {
|
||||
return <span style={{ color: entry?.color }}>{value}</span>;
|
||||
};
|
||||
|
||||
return (
|
||||
<Card className="tw-rounded-md tw-border">
|
||||
<Row gutter={[16, 16]}>
|
||||
<Col span={4}>
|
||||
<Space direction="vertical" size={16}>
|
||||
{information.map((info) => (
|
||||
<Statistic
|
||||
key={info.title}
|
||||
title={<span className="tw-text-grey-body">{info.title}</span>}
|
||||
value={
|
||||
tickFormatter
|
||||
? `${info.latestValue}${tickFormatter}`
|
||||
: info.latestValue
|
||||
}
|
||||
valueStyle={{ color: info.color }}
|
||||
/>
|
||||
))}
|
||||
</Space>
|
||||
</Col>
|
||||
<Col span={20}>
|
||||
{data.length > 0 ? (
|
||||
<ResponsiveContainer minHeight={300}>
|
||||
<LineChart className="tw-w-full" data={data}>
|
||||
<XAxis dataKey="name" padding={{ left: 16, right: 16 }} />
|
||||
|
||||
<YAxis
|
||||
allowDataOverflow
|
||||
padding={{ top: 16, bottom: 16 }}
|
||||
tickFormatter={(props) =>
|
||||
tickFormatter ? `${props}${tickFormatter}` : props
|
||||
}
|
||||
/>
|
||||
<Tooltip />
|
||||
{information.map((info) => (
|
||||
<Line
|
||||
dataKey={info.dataKey}
|
||||
key={info.dataKey}
|
||||
name={info.title}
|
||||
stroke={info.color}
|
||||
type="monotone"
|
||||
/>
|
||||
))}
|
||||
<Legend formatter={renderColorfulLegendText} />
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
) : (
|
||||
<Empty description="No Data Available" />
|
||||
)}
|
||||
</Col>
|
||||
</Row>
|
||||
</Card>
|
||||
);
|
||||
};
|
||||
|
||||
export default ProfilerDetailsCard;
|
||||
@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Card, Col, Row, Statistic } from 'antd';
|
||||
import React from 'react';
|
||||
import { TestCaseStatus } from '../../../generated/tests/tableTest';
|
||||
import { formTwoDigitNmber } from '../../../utils/CommonUtils';
|
||||
import TestIndicator from '../../common/TestIndicator/TestIndicator';
|
||||
import { ProfilerSummaryCardProps } from '../profilerDashboard.interface';
|
||||
|
||||
const ProfilerSummaryCard: React.FC<ProfilerSummaryCardProps> = ({
|
||||
data,
|
||||
title,
|
||||
showIndicator = false,
|
||||
}) => {
|
||||
return (
|
||||
<Card className="tw-rounded-md tw-border">
|
||||
<p className="tw-text-base tw-font-medium tw-mb-7">{title}</p>
|
||||
<Row className="table-profiler-summary">
|
||||
{data.map((item) => (
|
||||
<Col className="overall-summary-card" key={item.title} span={8}>
|
||||
<Statistic
|
||||
title={item.title}
|
||||
value={item.value}
|
||||
valueRender={(prop) =>
|
||||
showIndicator ? (
|
||||
<TestIndicator
|
||||
type={item.title as TestCaseStatus}
|
||||
value={formTwoDigitNmber(item.value as number)}
|
||||
/>
|
||||
) : (
|
||||
prop
|
||||
)
|
||||
}
|
||||
/>
|
||||
</Col>
|
||||
))}
|
||||
</Row>
|
||||
</Card>
|
||||
);
|
||||
};
|
||||
|
||||
export default ProfilerSummaryCard;
|
||||
@ -0,0 +1,228 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Card, Col, Row, Statistic } from 'antd';
|
||||
import moment from 'moment';
|
||||
import React, { useEffect, useMemo, useState } from 'react';
|
||||
import { useParams } from 'react-router-dom';
|
||||
import {
|
||||
INITIAL_COUNT_METRIC_VALUE,
|
||||
INITIAL_MATH_METRIC_VALUE,
|
||||
INITIAL_PROPORTION_METRIC_VALUE,
|
||||
INITIAL_SUM_METRIC_VALUE,
|
||||
} from '../../../constants/profiler.constant';
|
||||
import { getNameFromFQN } from '../../../utils/CommonUtils';
|
||||
import Ellipses from '../../common/Ellipses/Ellipses';
|
||||
import {
|
||||
MetricChartType,
|
||||
ProfilerTabProps,
|
||||
} from '../profilerDashboard.interface';
|
||||
import ProfilerDetailsCard from './ProfilerDetailsCard';
|
||||
import ProfilerSummaryCard from './ProfilerSummaryCard';
|
||||
|
||||
const ProfilerTab: React.FC<ProfilerTabProps> = ({
|
||||
activeColumnDetails,
|
||||
profilerData,
|
||||
}) => {
|
||||
const { entityTypeFQN } = useParams<Record<string, string>>();
|
||||
const [countMetrics, setCountMetrics] = useState<MetricChartType>(
|
||||
INITIAL_COUNT_METRIC_VALUE
|
||||
);
|
||||
const [proportionMetrics, setProportionMetrics] = useState<MetricChartType>(
|
||||
INITIAL_PROPORTION_METRIC_VALUE
|
||||
);
|
||||
const [mathMetrics, setMathMetrics] = useState<MetricChartType>(
|
||||
INITIAL_MATH_METRIC_VALUE
|
||||
);
|
||||
const [sumMetrics, setSumMetrics] = useState<MetricChartType>(
|
||||
INITIAL_SUM_METRIC_VALUE
|
||||
);
|
||||
|
||||
const tableState = useMemo(
|
||||
() => [
|
||||
{
|
||||
title: 'Row Count',
|
||||
value: profilerData[0]?.rowCount || 0,
|
||||
},
|
||||
{
|
||||
title: 'Column Count',
|
||||
value: profilerData[0]?.columnCount || 0,
|
||||
},
|
||||
{
|
||||
title: 'Table Sample %',
|
||||
value: `${profilerData[0]?.profileSample || 100}%`,
|
||||
},
|
||||
],
|
||||
[profilerData]
|
||||
);
|
||||
const testSummary = useMemo(
|
||||
() => [
|
||||
{
|
||||
title: 'Success',
|
||||
value: 0,
|
||||
},
|
||||
{
|
||||
title: 'Aborted',
|
||||
value: 0,
|
||||
},
|
||||
{
|
||||
title: 'Failed',
|
||||
value: 0,
|
||||
},
|
||||
],
|
||||
[]
|
||||
);
|
||||
|
||||
const createMetricsChartData = () => {
|
||||
const columnName = getNameFromFQN(entityTypeFQN);
|
||||
const countMetricData: MetricChartType['data'] = [];
|
||||
const proportionMetricData: MetricChartType['data'] = [];
|
||||
const mathMetricData: MetricChartType['data'] = [];
|
||||
const sumMetricData: MetricChartType['data'] = [];
|
||||
profilerData.forEach((item) => {
|
||||
const x = moment.unix(item.timestamp || 0).format('DD/MMM HH:mm');
|
||||
const col = item.columnProfile?.find((col) => col.name === columnName);
|
||||
|
||||
countMetricData.push({
|
||||
name: x,
|
||||
timestamp: item.timestamp || 0,
|
||||
distinctCount: col?.distinctCount || 0,
|
||||
nullCount: col?.nullCount || 0,
|
||||
uniqueCount: col?.uniqueCount || 0,
|
||||
valuesCount: col?.valuesCount || 0,
|
||||
});
|
||||
|
||||
sumMetricData.push({
|
||||
name: x,
|
||||
timestamp: item.timestamp || 0,
|
||||
sum: col?.sum || 0,
|
||||
});
|
||||
|
||||
mathMetricData.push({
|
||||
name: x,
|
||||
timestamp: item.timestamp || 0,
|
||||
max: (col?.max as number) || 0,
|
||||
min: (col?.min as number) || 0,
|
||||
mean: col?.mean || 0,
|
||||
median: col?.median || 0,
|
||||
});
|
||||
|
||||
proportionMetricData.push({
|
||||
name: x,
|
||||
timestamp: item.timestamp || 0,
|
||||
distinctProportion: col?.distinctProportion || 0,
|
||||
nullProportion: col?.nullProportion || 0,
|
||||
uniqueProportion: col?.uniqueProportion || 0,
|
||||
});
|
||||
});
|
||||
|
||||
const countMetricInfo = countMetrics.information.map((item) => ({
|
||||
...item,
|
||||
latestValue: countMetricData[0][item.dataKey],
|
||||
}));
|
||||
const proportionMetricInfo = proportionMetrics.information.map((item) => ({
|
||||
...item,
|
||||
latestValue: parseFloat(
|
||||
`${proportionMetricData[0][item.dataKey]}`
|
||||
).toFixed(2),
|
||||
}));
|
||||
const mathMetricInfo = mathMetrics.information.map((item) => ({
|
||||
...item,
|
||||
latestValue: mathMetricData[0][item.dataKey],
|
||||
}));
|
||||
const sumMetricInfo = sumMetrics.information.map((item) => ({
|
||||
...item,
|
||||
latestValue: sumMetricData[0][item.dataKey],
|
||||
}));
|
||||
|
||||
setCountMetrics((pre) => ({
|
||||
...pre,
|
||||
information: countMetricInfo,
|
||||
data: countMetricData,
|
||||
}));
|
||||
setProportionMetrics((pre) => ({
|
||||
...pre,
|
||||
information: proportionMetricInfo,
|
||||
data: proportionMetricData,
|
||||
}));
|
||||
setMathMetrics((pre) => ({
|
||||
...pre,
|
||||
information: mathMetricInfo,
|
||||
data: mathMetricData,
|
||||
}));
|
||||
setSumMetrics((pre) => ({
|
||||
...pre,
|
||||
information: sumMetricInfo,
|
||||
data: sumMetricData,
|
||||
}));
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
createMetricsChartData();
|
||||
}, [profilerData]);
|
||||
|
||||
return (
|
||||
<Row gutter={[16, 16]}>
|
||||
<Col span={8}>
|
||||
<Card className="tw-rounded-md tw-border tw-h-full">
|
||||
<Row gutter={16}>
|
||||
<Col span={18}>
|
||||
<p className="tw-font-medium tw-text-base">Column summary</p>
|
||||
<Ellipses className="tw-text-grey-muted" rows={4}>
|
||||
{activeColumnDetails.description}
|
||||
</Ellipses>
|
||||
</Col>
|
||||
<Col span={6}>
|
||||
<Statistic
|
||||
title="Data type"
|
||||
value={activeColumnDetails.dataTypeDisplay || ''}
|
||||
valueStyle={{
|
||||
color: '#1890FF',
|
||||
fontSize: '24px',
|
||||
fontWeight: 600,
|
||||
}}
|
||||
/>
|
||||
</Col>
|
||||
</Row>
|
||||
</Card>
|
||||
</Col>
|
||||
<Col span={8}>
|
||||
<ProfilerSummaryCard data={tableState} title="Table Metrics Summary" />
|
||||
</Col>
|
||||
<Col span={8}>
|
||||
<ProfilerSummaryCard
|
||||
showIndicator
|
||||
data={testSummary}
|
||||
title="Quality Tests Summary"
|
||||
/>
|
||||
</Col>
|
||||
<Col span={24}>
|
||||
<ProfilerDetailsCard chartCollection={countMetrics} />
|
||||
</Col>
|
||||
<Col span={24}>
|
||||
<ProfilerDetailsCard
|
||||
chartCollection={proportionMetrics}
|
||||
tickFormatter="%"
|
||||
/>
|
||||
</Col>
|
||||
<Col span={24}>
|
||||
<ProfilerDetailsCard chartCollection={mathMetrics} />
|
||||
</Col>
|
||||
<Col span={24}>
|
||||
<ProfilerDetailsCard chartCollection={sumMetrics} />
|
||||
</Col>
|
||||
</Row>
|
||||
);
|
||||
};
|
||||
|
||||
export default ProfilerTab;
|
||||
@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Column, Table, TableProfile } from '../../generated/entity/data/table';
|
||||
|
||||
export interface ProfilerDashboardProps {
|
||||
onTableChange: (table: Table) => void;
|
||||
table: Table;
|
||||
profilerData: TableProfile[];
|
||||
fetchProfilerData: (tableId: string, days?: number) => void;
|
||||
}
|
||||
|
||||
export type MetricChartType = {
|
||||
information: {
|
||||
title: string;
|
||||
dataKey: string;
|
||||
color: string;
|
||||
latestValue?: string | number;
|
||||
}[];
|
||||
data: Record<string, string | number>[];
|
||||
};
|
||||
|
||||
export interface ProfilerDetailsCardProps {
|
||||
chartCollection: MetricChartType;
|
||||
tickFormatter?: string;
|
||||
}
|
||||
|
||||
export enum ProfilerDashboardTab {
|
||||
SUMMARY = 'Summary',
|
||||
PROFILER = 'Profiler',
|
||||
DATA_QUALITY = 'Data Quality',
|
||||
}
|
||||
|
||||
export type ChartData = {
|
||||
name: string;
|
||||
proportion?: number;
|
||||
value: number;
|
||||
timestamp: number;
|
||||
};
|
||||
|
||||
export type ChartCollection = {
|
||||
data: ChartData[];
|
||||
color: string;
|
||||
};
|
||||
|
||||
export type ChartDataCollection = Record<string, ChartCollection>;
|
||||
|
||||
export interface ProfilerTabProps {
|
||||
profilerData: TableProfile[];
|
||||
activeColumnDetails: Column;
|
||||
}
|
||||
|
||||
export interface ProfilerSummaryCardProps {
|
||||
title: string;
|
||||
data: {
|
||||
title: string;
|
||||
value: number | string;
|
||||
}[];
|
||||
showIndicator?: boolean;
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
.table-profiler-summary {
|
||||
.overall-summary-card:not(:first-child) {
|
||||
border-left: 1px solid rgb(229, 231, 235);
|
||||
padding-left: 16px;
|
||||
}
|
||||
}
|
||||
|
||||
.latest-profiler-data-container {
|
||||
table {
|
||||
border: none;
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.ant-table.ant-table-small .ant-table-thead > tr > th,
|
||||
.ant-table.ant-table-small .ant-table-tbody > tr > td {
|
||||
padding-left: 16px;
|
||||
padding-right: 16px;
|
||||
}
|
||||
|
||||
.ant-table-tbody > tr > td {
|
||||
border-bottom: none;
|
||||
}
|
||||
}
|
||||
@ -21,6 +21,7 @@ import {
|
||||
} from '@testing-library/react';
|
||||
import { ColumnsType } from 'antd/lib/table';
|
||||
import React from 'react';
|
||||
import { MemoryRouter } from 'react-router-dom';
|
||||
import { Column, ColumnProfile } from '../../../generated/entity/data/table';
|
||||
import { MOCK_TABLE } from '../../../mocks/TableData.mock';
|
||||
import { ColumnProfileTableProps } from '../TableProfiler.interface';
|
||||
@ -88,7 +89,7 @@ jest.mock('./ProfilerProgressWidget', () => {
|
||||
</span>
|
||||
));
|
||||
});
|
||||
jest.mock('./TestIndicator', () => {
|
||||
jest.mock('../../common/TestIndicator/TestIndicator', () => {
|
||||
return jest.fn().mockImplementation(({ value, type }) => (
|
||||
<span data-testid="test-indicator">
|
||||
{value} <span>{type}</span>
|
||||
@ -109,7 +110,9 @@ describe('Test ColumnProfileTable component', () => {
|
||||
});
|
||||
|
||||
it('should render without crashing', async () => {
|
||||
render(<ColumnProfileTable {...mockProps} />);
|
||||
render(<ColumnProfileTable {...mockProps} />, {
|
||||
wrapper: MemoryRouter,
|
||||
});
|
||||
|
||||
const container = await screen.findByTestId(
|
||||
'column-profile-table-container'
|
||||
@ -125,7 +128,10 @@ describe('Test ColumnProfileTable component', () => {
|
||||
<ColumnProfileTable
|
||||
{...mockProps}
|
||||
columns={undefined as unknown as Column[]}
|
||||
/>
|
||||
/>,
|
||||
{
|
||||
wrapper: MemoryRouter,
|
||||
}
|
||||
);
|
||||
|
||||
const container = await screen.findByTestId(
|
||||
@ -138,7 +144,9 @@ describe('Test ColumnProfileTable component', () => {
|
||||
});
|
||||
|
||||
it('search box should work as expected', async () => {
|
||||
render(<ColumnProfileTable {...mockProps} />);
|
||||
render(<ColumnProfileTable {...mockProps} />, {
|
||||
wrapper: MemoryRouter,
|
||||
});
|
||||
|
||||
const searchbox = await screen.findByTestId('searchbar');
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
import { Button, Space, Table } from 'antd';
|
||||
import { ColumnsType } from 'antd/lib/table';
|
||||
import React, { FC, useEffect, useMemo, useState } from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import {
|
||||
PRIMERY_COLOR,
|
||||
SECONDARY_COLOR,
|
||||
@ -23,11 +24,12 @@ import { ColumnProfile } from '../../../generated/entity/data/table';
|
||||
import { TestCaseStatus } from '../../../generated/tests/tableTest';
|
||||
import { formatNumberWithComma } from '../../../utils/CommonUtils';
|
||||
import { getCurrentDatasetTab } from '../../../utils/DatasetDetailsUtils';
|
||||
import { getProfilerDashboardWithFqnPath } from '../../../utils/RouterUtils';
|
||||
import Ellipses from '../../common/Ellipses/Ellipses';
|
||||
import Searchbar from '../../common/searchbar/Searchbar';
|
||||
import TestIndicator from '../../common/TestIndicator/TestIndicator';
|
||||
import { ColumnProfileTableProps } from '../TableProfiler.interface';
|
||||
import ProfilerProgressWidget from './ProfilerProgressWidget';
|
||||
import TestIndicator from './TestIndicator';
|
||||
|
||||
const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
||||
columnProfile,
|
||||
@ -57,6 +59,18 @@ const ColumnProfileTable: FC<ColumnProfileTableProps> = ({
|
||||
title: 'Name',
|
||||
dataIndex: 'name',
|
||||
key: 'name',
|
||||
render: (name: string) => {
|
||||
const data = columns.find((col) => col.name === name);
|
||||
|
||||
return (
|
||||
<Link
|
||||
to={getProfilerDashboardWithFqnPath(
|
||||
data?.fullyQualifiedName || ''
|
||||
)}>
|
||||
{name}
|
||||
</Link>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: 'Data Type',
|
||||
|
||||
@ -30,7 +30,7 @@ import {
|
||||
codeMirrorOption,
|
||||
DEFAULT_INCLUDE_PROFILE,
|
||||
PROFILER_METRIC,
|
||||
} from '../../../constants/entity.constants';
|
||||
} from '../../../constants/profiler.constant';
|
||||
import {
|
||||
ColumnProfilerConfig,
|
||||
TableProfilerConfig,
|
||||
@ -49,7 +49,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
|
||||
}) => {
|
||||
const [data, setData] = useState<TableProfilerConfig>();
|
||||
const [sqlQuery, setSqlQuery] = useState<string>('');
|
||||
const [profileSample, setProfileSample] = useState<number>(0);
|
||||
const [profileSample, setProfileSample] = useState<number>(100);
|
||||
const [excludeCol, setExcludeCol] = useState<string[]>([]);
|
||||
const [includeCol, setIncludeCol] = useState<ColumnProfilerConfig[]>(
|
||||
DEFAULT_INCLUDE_PROFILE
|
||||
@ -81,7 +81,7 @@ const ProfilerSettingsModal: React.FC<ProfilerSettingsModalProps> = ({
|
||||
const updateInitialConfig = (tableProfilerConfig: TableProfilerConfig) => {
|
||||
const { includeColumns } = tableProfilerConfig;
|
||||
setSqlQuery(tableProfilerConfig.profileQuery || '');
|
||||
setProfileSample(tableProfilerConfig.profileSample || 0);
|
||||
setProfileSample(tableProfilerConfig.profileSample || 100);
|
||||
setExcludeCol(tableProfilerConfig.excludeColumns || []);
|
||||
if (includeColumns && includeColumns?.length > 0) {
|
||||
const includeColValue = includeColumns.map((col) => {
|
||||
|
||||
@ -51,7 +51,7 @@ export interface ProfilerSettingsModalProps {
|
||||
}
|
||||
|
||||
export interface TestIndicatorProps {
|
||||
value: number;
|
||||
value: number | string;
|
||||
type: TestCaseStatus;
|
||||
}
|
||||
|
||||
|
||||
@ -50,23 +50,6 @@
|
||||
}
|
||||
}
|
||||
|
||||
.test-indicator {
|
||||
display: inline-block;
|
||||
height: 8px;
|
||||
width: 8px;
|
||||
border-radius: 50%;
|
||||
|
||||
&.success {
|
||||
background: @succesColor;
|
||||
}
|
||||
&.failed {
|
||||
background: @failedColor;
|
||||
}
|
||||
&.aborted {
|
||||
background: @abortedColor;
|
||||
}
|
||||
}
|
||||
|
||||
.ant-table-row .ant-table-cell:first-child,
|
||||
.ant-table-thead .ant-table-cell:first-child {
|
||||
padding-left: 16px;
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
import { cleanup, render, screen } from '@testing-library/react';
|
||||
import React from 'react';
|
||||
import { TestCaseStatus } from '../../../generated/tests/tableTest';
|
||||
import { TestIndicatorProps } from '../TableProfiler.interface';
|
||||
import { TestIndicatorProps } from '../../TableProfiler/TableProfiler.interface';
|
||||
import TestIndicator from './TestIndicator';
|
||||
|
||||
const mockProps: TestIndicatorProps = {
|
||||
@ -13,7 +13,8 @@
|
||||
|
||||
import classNames from 'classnames';
|
||||
import React from 'react';
|
||||
import { TestIndicatorProps } from '../TableProfiler.interface';
|
||||
import { TestIndicatorProps } from '../../TableProfiler/TableProfiler.interface';
|
||||
import './testIndicator.less';
|
||||
|
||||
const TestIndicator: React.FC<TestIndicatorProps> = ({ value, type }) => {
|
||||
return (
|
||||
@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
@succesColor: #28a745;
|
||||
@failedColor: #cb2431;
|
||||
@abortedColor: #efae2f;
|
||||
|
||||
.test-indicator {
|
||||
display: inline-block;
|
||||
height: 8px;
|
||||
width: 8px;
|
||||
border-radius: 50%;
|
||||
|
||||
&.success {
|
||||
background: @succesColor;
|
||||
}
|
||||
&.failed {
|
||||
background: @failedColor;
|
||||
}
|
||||
&.aborted {
|
||||
background: @abortedColor;
|
||||
}
|
||||
}
|
||||
@ -65,7 +65,7 @@ interface Props {
|
||||
deleted?: boolean;
|
||||
followers?: number;
|
||||
extraInfo: Array<ExtraInfo>;
|
||||
tier: TagLabel;
|
||||
tier: TagLabel | undefined;
|
||||
tags: Array<EntityTags>;
|
||||
isTagEditable?: boolean;
|
||||
owner?: EntityReference;
|
||||
|
||||
@ -217,6 +217,7 @@ export const ROUTES = {
|
||||
CUSTOM_PROPERTIES: `/custom-properties`,
|
||||
CUSTOM_ENTITY_DETAIL: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
||||
ADD_CUSTOM_PROPERTY: `/custom-properties/${PLACEHOLDER_ENTITY_TYPE_FQN}/add-field`,
|
||||
PROFILER_DASHBOARD: `/profiler-dashboard/${PLACEHOLDER_ENTITY_TYPE_FQN}`,
|
||||
|
||||
// Tasks Routes
|
||||
REQUEST_DESCRIPTION: `/request-description/${PLACEHOLDER_ROUTE_ENTITY_TYPE}/${PLACEHOLDER_ROUTE_ENTITY_FQN}`,
|
||||
|
||||
@ -11,61 +11,8 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { CSMode } from '../enums/codemirror.enum';
|
||||
import { ColumnProfilerConfig } from '../generated/entity/data/table';
|
||||
import { JSON_TAB_SIZE } from './constants';
|
||||
|
||||
export const ENTITY_DELETE_STATE = {
|
||||
loading: 'initial',
|
||||
state: false,
|
||||
softDelete: true,
|
||||
};
|
||||
|
||||
export const PROFILER_METRIC = [
|
||||
'valuesCount',
|
||||
'valuesPercentage',
|
||||
'validCount',
|
||||
'duplicateCount',
|
||||
'nullCount',
|
||||
'nullProportion',
|
||||
'missingPercentage',
|
||||
'missingCount',
|
||||
'uniqueCount',
|
||||
'uniqueProportion',
|
||||
'distinctCount',
|
||||
'distinctProportion',
|
||||
'min',
|
||||
'max',
|
||||
'minLength',
|
||||
'maxLength',
|
||||
'mean',
|
||||
'sum',
|
||||
'stddev',
|
||||
'variance',
|
||||
'median',
|
||||
'histogram',
|
||||
'customMetricsProfile',
|
||||
];
|
||||
|
||||
export const DEFAULT_INCLUDE_PROFILE: ColumnProfilerConfig[] = [
|
||||
{
|
||||
columnName: undefined,
|
||||
metrics: ['all'],
|
||||
},
|
||||
];
|
||||
|
||||
export const codeMirrorOption = {
|
||||
tabSize: JSON_TAB_SIZE,
|
||||
indentUnit: JSON_TAB_SIZE,
|
||||
indentWithTabs: true,
|
||||
lineNumbers: true,
|
||||
lineWrapping: true,
|
||||
styleActiveLine: true,
|
||||
matchBrackets: true,
|
||||
autoCloseBrackets: true,
|
||||
foldGutter: true,
|
||||
gutters: ['CodeMirror-linenumbers', 'CodeMirror-foldgutter'],
|
||||
mode: {
|
||||
name: CSMode.SQL,
|
||||
},
|
||||
};
|
||||
|
||||
@ -11,6 +11,10 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { CSMode } from '../enums/codemirror.enum';
|
||||
import { ColumnProfilerConfig } from '../generated/entity/data/table';
|
||||
import { JSON_TAB_SIZE } from './constants';
|
||||
|
||||
export const excludedMetrics = [
|
||||
'profilDate',
|
||||
'name',
|
||||
@ -24,3 +28,151 @@ export const excludedMetrics = [
|
||||
'missingPercentage',
|
||||
'distinctProportion',
|
||||
];
|
||||
|
||||
export const PROFILER_METRIC = [
|
||||
'valuesCount',
|
||||
'valuesPercentage',
|
||||
'validCount',
|
||||
'duplicateCount',
|
||||
'nullCount',
|
||||
'nullProportion',
|
||||
'missingPercentage',
|
||||
'missingCount',
|
||||
'uniqueCount',
|
||||
'uniqueProportion',
|
||||
'distinctCount',
|
||||
'distinctProportion',
|
||||
'min',
|
||||
'max',
|
||||
'minLength',
|
||||
'maxLength',
|
||||
'mean',
|
||||
'sum',
|
||||
'stddev',
|
||||
'variance',
|
||||
'median',
|
||||
'histogram',
|
||||
'customMetricsProfile',
|
||||
];
|
||||
|
||||
export const PROFILER_FILTER_RANGE = {
|
||||
last3days: { days: 3, title: 'Last 3 days' },
|
||||
last7days: { days: 7, title: 'Last 7 days' },
|
||||
last14days: { days: 14, title: 'Last 14 days' },
|
||||
last30days: { days: 30, title: 'Last 30 days' },
|
||||
last60days: { days: 60, title: 'Last 60 days' },
|
||||
};
|
||||
|
||||
export const DEFAULT_CHART_COLLECTION_VALUE = {
|
||||
distinctCount: { data: [], color: '#1890FF' },
|
||||
uniqueCount: { data: [], color: '#008376' },
|
||||
nullCount: { data: [], color: '#7147E8' },
|
||||
nullProportion: { data: [], color: '#B02AAC' },
|
||||
};
|
||||
|
||||
export const INITIAL_COUNT_METRIC_VALUE = {
|
||||
information: [
|
||||
{
|
||||
title: 'Distinct Count',
|
||||
dataKey: 'distinctCount',
|
||||
color: '#1890FF',
|
||||
},
|
||||
{
|
||||
title: 'Null Count',
|
||||
dataKey: 'nullCount',
|
||||
color: '#7147E8',
|
||||
},
|
||||
{
|
||||
title: 'Unique Count',
|
||||
dataKey: 'uniqueCount',
|
||||
color: '#008376',
|
||||
},
|
||||
{
|
||||
title: 'Values Count',
|
||||
dataKey: 'valuesCount',
|
||||
color: '#B02AAC',
|
||||
},
|
||||
],
|
||||
data: [],
|
||||
};
|
||||
|
||||
export const INITIAL_PROPORTION_METRIC_VALUE = {
|
||||
information: [
|
||||
{
|
||||
title: 'Distinct Proportion',
|
||||
dataKey: 'distinctProportion',
|
||||
color: '#1890FF',
|
||||
},
|
||||
{
|
||||
title: 'Null Proportion',
|
||||
dataKey: 'nullProportion',
|
||||
color: '#7147E8',
|
||||
},
|
||||
{
|
||||
title: 'Unique Proportion',
|
||||
dataKey: 'uniqueProportion',
|
||||
color: '#008376',
|
||||
},
|
||||
],
|
||||
data: [],
|
||||
};
|
||||
|
||||
export const INITIAL_MATH_METRIC_VALUE = {
|
||||
information: [
|
||||
{
|
||||
title: 'Median',
|
||||
dataKey: 'median',
|
||||
color: '#1890FF',
|
||||
},
|
||||
{
|
||||
title: 'Max',
|
||||
dataKey: 'max',
|
||||
color: '#7147E8',
|
||||
},
|
||||
{
|
||||
title: 'Mean',
|
||||
dataKey: 'mean',
|
||||
color: '#008376',
|
||||
},
|
||||
{
|
||||
title: 'Min',
|
||||
dataKey: 'min',
|
||||
color: '#B02AAC',
|
||||
},
|
||||
],
|
||||
data: [],
|
||||
};
|
||||
|
||||
export const INITIAL_SUM_METRIC_VALUE = {
|
||||
information: [
|
||||
{
|
||||
title: 'Sum',
|
||||
dataKey: 'sum',
|
||||
color: '#1890FF',
|
||||
},
|
||||
],
|
||||
data: [],
|
||||
};
|
||||
|
||||
export const DEFAULT_INCLUDE_PROFILE: ColumnProfilerConfig[] = [
|
||||
{
|
||||
columnName: undefined,
|
||||
metrics: ['all'],
|
||||
},
|
||||
];
|
||||
|
||||
export const codeMirrorOption = {
|
||||
tabSize: JSON_TAB_SIZE,
|
||||
indentUnit: JSON_TAB_SIZE,
|
||||
indentWithTabs: true,
|
||||
lineNumbers: true,
|
||||
lineWrapping: true,
|
||||
styleActiveLine: true,
|
||||
matchBrackets: true,
|
||||
autoCloseBrackets: true,
|
||||
foldGutter: true,
|
||||
gutters: ['CodeMirror-linenumbers', 'CodeMirror-foldgutter'],
|
||||
mode: {
|
||||
name: CSMode.SQL,
|
||||
},
|
||||
};
|
||||
|
||||
@ -275,6 +275,18 @@ jest.mock('react-router-dom', () => ({
|
||||
useParams: jest.fn().mockImplementation(() => mockUseParams),
|
||||
}));
|
||||
|
||||
jest.mock('../../utils/CommonUtils', () => ({
|
||||
addToRecentViewed: jest.fn(),
|
||||
getCurrentUserId: jest.fn().mockReturnValue('test'),
|
||||
getEntityMissingError: jest
|
||||
.fn()
|
||||
.mockImplementation(() => <span>Entity missing error</span>),
|
||||
getEntityName: jest.fn().mockReturnValue('getEntityName'),
|
||||
getFeedCounts: jest.fn(),
|
||||
getFields: jest.fn().mockReturnValue('field'),
|
||||
getPartialNameFromTableFQN: jest.fn().mockReturnValue('name'),
|
||||
}));
|
||||
|
||||
describe('Test DatasetDetails page', () => {
|
||||
it('Component should render properly', async () => {
|
||||
const { container } = render(<DatasetDetailsPage />, {
|
||||
|
||||
@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright 2022 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { AxiosError } from 'axios';
|
||||
import { compare } from 'fast-json-patch';
|
||||
import moment from 'moment';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import { useParams } from 'react-router-dom';
|
||||
import {
|
||||
getTableDetailsByFQN,
|
||||
getTableProfilesList,
|
||||
patchTableDetails,
|
||||
} from '../../axiosAPIs/tableAPI';
|
||||
import ErrorPlaceHolder from '../../components/common/error-with-placeholder/ErrorPlaceHolder';
|
||||
import PageContainerV1 from '../../components/containers/PageContainerV1';
|
||||
import Loader from '../../components/Loader/Loader';
|
||||
import ProfilerDashboard from '../../components/ProfilerDashboard/ProfilerDashboard';
|
||||
import { API_RES_MAX_SIZE } from '../../constants/constants';
|
||||
import { Table, TableProfile } from '../../generated/entity/data/table';
|
||||
import jsonData from '../../jsons/en';
|
||||
import {
|
||||
getNameFromFQN,
|
||||
getTableFQNFromColumnFQN,
|
||||
} from '../../utils/CommonUtils';
|
||||
import { showErrorToast } from '../../utils/ToastUtils';
|
||||
|
||||
const ProfilerDashboardPage = () => {
|
||||
const { entityTypeFQN } = useParams<Record<string, string>>();
|
||||
const [table, setTable] = useState<Table>({} as Table);
|
||||
const [profilerData, setProfilerData] = useState<TableProfile[]>([]);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [error, setError] = useState(false);
|
||||
|
||||
const fetchProfilerData = async (tableId: string, days = 3) => {
|
||||
try {
|
||||
const startTs = moment().subtract(days, 'days').unix();
|
||||
|
||||
const data = await getTableProfilesList(tableId, {
|
||||
startTs: startTs,
|
||||
limit: API_RES_MAX_SIZE,
|
||||
});
|
||||
setProfilerData(data.data || []);
|
||||
} catch (error) {
|
||||
showErrorToast(error as AxiosError);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const fetchTableEntity = async (fqn: string) => {
|
||||
try {
|
||||
getTableFQNFromColumnFQN(fqn);
|
||||
const data = await getTableDetailsByFQN(
|
||||
getTableFQNFromColumnFQN(fqn),
|
||||
'tags, usageSummary, owner, followers'
|
||||
);
|
||||
setTable(data ?? ({} as Table));
|
||||
fetchProfilerData(data.id);
|
||||
} catch (error) {
|
||||
showErrorToast(
|
||||
error as AxiosError,
|
||||
jsonData['api-error-messages']['fetch-table-details-error']
|
||||
);
|
||||
setIsLoading(false);
|
||||
setError(true);
|
||||
}
|
||||
};
|
||||
|
||||
const updateTableHandler = async (updatedTable: Table) => {
|
||||
const jsonPatch = compare(table, updatedTable);
|
||||
|
||||
try {
|
||||
const tableRes = await patchTableDetails(table.id, jsonPatch);
|
||||
setTable(tableRes);
|
||||
} catch (error) {
|
||||
showErrorToast(
|
||||
error as AxiosError,
|
||||
jsonData['api-error-messages']['update-entity-error']
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (entityTypeFQN) {
|
||||
fetchTableEntity(entityTypeFQN);
|
||||
} else {
|
||||
setIsLoading(false);
|
||||
setError(true);
|
||||
}
|
||||
}, [entityTypeFQN]);
|
||||
|
||||
if (isLoading) {
|
||||
return <Loader />;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<ErrorPlaceHolder>
|
||||
<p className="tw-text-center">
|
||||
No data found{' '}
|
||||
{entityTypeFQN ? `for column ${getNameFromFQN(entityTypeFQN)}` : ''}
|
||||
</p>
|
||||
</ErrorPlaceHolder>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<PageContainerV1 className="tw-py-4">
|
||||
<ProfilerDashboard
|
||||
fetchProfilerData={fetchProfilerData}
|
||||
profilerData={profilerData}
|
||||
table={table}
|
||||
onTableChange={updateTableHandler}
|
||||
/>
|
||||
</PageContainerV1>
|
||||
);
|
||||
};
|
||||
|
||||
export default ProfilerDashboardPage;
|
||||
@ -23,6 +23,12 @@ const GlobalSettingPage = withSuspenseFallback(
|
||||
React.lazy(() => import('../pages/GlobalSettingPage/GlobalSettingPage'))
|
||||
);
|
||||
|
||||
const ProfilerDashboardPage = withSuspenseFallback(
|
||||
React.lazy(
|
||||
() => import('../pages/ProfilerDashboardPage/ProfilerDashboardPage')
|
||||
)
|
||||
);
|
||||
|
||||
const MyDataPage = withSuspenseFallback(
|
||||
React.lazy(() => import('../pages/MyDataPage/MyDataPage.component'))
|
||||
);
|
||||
@ -275,6 +281,11 @@ const AuthenticatedAppRouter: FunctionComponent = () => {
|
||||
<Route exact component={UserPage} path={ROUTES.USER_PROFILE} />
|
||||
<Route exact component={UserPage} path={ROUTES.USER_PROFILE_WITH_TAB} />
|
||||
<Route exact component={MlModelPage} path={ROUTES.MLMODEL_DETAILS} />
|
||||
<Route
|
||||
exact
|
||||
component={ProfilerDashboardPage}
|
||||
path={ROUTES.PROFILER_DASHBOARD}
|
||||
/>
|
||||
<Route
|
||||
exact
|
||||
component={MlModelPage}
|
||||
|
||||
@ -152,14 +152,9 @@ export const getPartialNameFromTableFQN = (
|
||||
};
|
||||
|
||||
export const getCurrentUserId = (): string => {
|
||||
// TODO: Replace below with USERID from Logged-in data
|
||||
const { id: userId } = !isEmpty(AppState.userDetails)
|
||||
? AppState.userDetails
|
||||
: AppState.users?.length
|
||||
? AppState.users[0]
|
||||
: { id: undefined };
|
||||
const currentUser = AppState.getCurrentUserDetails();
|
||||
|
||||
return userId as string;
|
||||
return currentUser?.id || '';
|
||||
};
|
||||
|
||||
export const pluralize = (count: number, noun: string, suffix = 's') => {
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
import { FQN_SEPARATOR_CHAR } from '../constants/char.constants';
|
||||
import {
|
||||
IN_PAGE_SEARCH_ROUTES,
|
||||
PLACEHOLDER_ENTITY_TYPE_FQN,
|
||||
PLACEHOLDER_GLOSSARY_NAME,
|
||||
PLACEHOLDER_GLOSSARY_TERMS_FQN,
|
||||
PLACEHOLDER_ROUTE_FQN,
|
||||
@ -292,3 +293,11 @@ export const getPath = (pathName: string) => {
|
||||
return getSettingPath();
|
||||
}
|
||||
};
|
||||
|
||||
export const getProfilerDashboardWithFqnPath = (entityTypeFQN: string) => {
|
||||
let path = ROUTES.PROFILER_DASHBOARD;
|
||||
|
||||
path = path.replace(PLACEHOLDER_ENTITY_TYPE_FQN, entityTypeFQN);
|
||||
|
||||
return path;
|
||||
};
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user