yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-03 20:19:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

#13445 Profiler Graph Should not Show a Point if value is None (#14345)

Browse Source 
* #13445 Profiler Graph Should not Show a Point if value is None

* addressing comment

* fixed sonarcloud
This commit is contained in:
Shailesh Parmar 2023-12-12 20:20:10 +05:30 committed by GitHub
parent d3ebe6fb4b
commit 2337e239a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 507 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerLatestValue.tsx
View File

@ -16,6 +16,7 @@ import React from 'react';
import { getStatisticsDisplayValue } from '../../../utils/CommonUtils';
import { ProfilerLatestValueProps } from '../profilerDashboard.interface';
import { isUndefined } from 'lodash';
import '../profiler-dashboard.less';
const ProfilerLatestValue = ({
@ -23,6 +24,18 @@ const ProfilerLatestValue = ({
tickFormatter,
stringValue = false,
}: ProfilerLatestValueProps) => {
const getLatestValue = (value?: number | string) => {
if (isUndefined(value)) {
return '--';
}
if (tickFormatter || stringValue) {
return `${value}${tickFormatter ?? ''}`;
} else {
return getStatisticsDisplayValue(value);
}
};
return (
<Space data-testid="data-summary-container" direction="vertical" size={16}>
{information.map((info) => (
@ -36,11 +49,7 @@ const ProfilerLatestValue = ({
{info.title}
</Typography.Text>
}
value={
tickFormatter || stringValue
? `${info.latestValue}${tickFormatter ?? ''}`
: getStatisticsDisplayValue(info.latestValue)
}
value={getLatestValue(info.latestValue)}
valueStyle={{ color: info.color, fontSize: '18px', fontWeight: 700 }}
/>
))}

149
openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/SingleColumnProfile.tsx
View File

@ -12,31 +12,27 @@
*/
import { Card, Col, Row, Typography } from 'antd';
import { AxiosError } from 'axios';
import { first, isString, last, sortBy } from 'lodash';
import { first, isString, last } from 'lodash';
import React, { FC, useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import DataDistributionHistogram from '../../../components/Chart/DataDistributionHistogram.component';
import ProfilerDetailsCard from '../../../components/ProfilerDashboard/component/ProfilerDetailsCard';
import { DateRangeObject } from '../../../components/ProfilerDashboard/component/TestSummary';
import { MetricChartType } from '../../../components/ProfilerDashboard/profilerDashboard.interface';
import {
DEFAULT_RANGE_DATA,
INITIAL_COUNT_METRIC_VALUE,
INITIAL_MATH_METRIC_VALUE,
INITIAL_PROPORTION_METRIC_VALUE,
INITIAL_QUARTILE_METRIC_VALUE,
INITIAL_SUM_METRIC_VALUE,
INITIAL_COLUMN_METRICS_VALUE,
} from '../../../constants/profiler.constant';
import { ColumnProfile } from '../../../generated/entity/data/container';
import { Table } from '../../../generated/entity/data/table';
import { getColumnProfilerList } from '../../../rest/tableAPI';
import { getEncodedFqn } from '../../../utils/StringsUtils';
import {
calculateColumnProfilerMetrics,
calculateCustomMetrics,
getColumnCustomMetric,
} from '../../../utils/TableProfilerUtils';
import { ColumnMetricsInterface } from '../../../utils/TableProfilerUtils.interface';
import { showErrorToast } from '../../../utils/ToastUtils';
import { customFormatDateTime } from '../../../utils/date-time/DateTimeUtils';
import CustomMetricGraphs from '../CustomMetricGraphs/CustomMetricGraphs.component';
import { useTableProfiler } from '../TableProfilerProvider';
@ -67,22 +63,10 @@ const SingleColumnProfile: FC<SingleColumnProfileProps> = ({
) ?? [],
[tableCustomMetric, activeColumnFqn, tableDetails]
);
const [countMetrics, setCountMetrics] = useState<MetricChartType>(
INITIAL_COUNT_METRIC_VALUE
);
const [proportionMetrics, setProportionMetrics] = useState<MetricChartType>(
INITIAL_PROPORTION_METRIC_VALUE
);
const [mathMetrics, setMathMetrics] = useState<MetricChartType>(
INITIAL_MATH_METRIC_VALUE
);
const [sumMetrics, setSumMetrics] = useState<MetricChartType>(
INITIAL_SUM_METRIC_VALUE
const [columnMetric, setColumnMetric] = useState<ColumnMetricsInterface>(
INITIAL_COLUMN_METRICS_VALUE
);
const [isMinMaxStringData, setIsMinMaxStringData] = useState(false);
const [quartileMetrics, setQuartileMetrics] = useState<MetricChartType>(
INITIAL_QUARTILE_METRIC_VALUE
);
const columnCustomMetrics = useMemo(
() => calculateCustomMetrics(columnProfilerData, customMetrics),
@ -115,116 +99,17 @@ const SingleColumnProfile: FC<SingleColumnProfileProps> = ({
}, [columnProfilerData]);
const createMetricsChartData = () => {
const updateProfilerData = sortBy(columnProfilerData, 'timestamp');
const countMetricData: MetricChartType['data'] = [];
const proportionMetricData: MetricChartType['data'] = [];
const mathMetricData: MetricChartType['data'] = [];
const sumMetricData: MetricChartType['data'] = [];
const quartileMetricData: MetricChartType['data'] = [];
updateProfilerData.forEach((col) => {
const timestamp = customFormatDateTime(col.timestamp, 'MMM dd, hh:mm');
countMetricData.push({
name: timestamp,
timestamp: col.timestamp,
distinctCount: col.distinctCount || 0,
nullCount: col.nullCount || 0,
uniqueCount: col.uniqueCount || 0,
valuesCount: col.valuesCount || 0,
});
sumMetricData.push({
name: timestamp,
timestamp: col.timestamp || 0,
sum: col.sum || 0,
});
mathMetricData.push({
name: timestamp,
timestamp: col.timestamp || 0,
max: col.max || 0,
min: col.min || 0,
mean: col.mean || 0,
});
proportionMetricData.push({
name: timestamp,
timestamp: col.timestamp || 0,
distinctProportion: Math.round((col.distinctProportion || 0) * 100),
nullProportion: Math.round((col.nullProportion || 0) * 100),
uniqueProportion: Math.round((col.uniqueProportion || 0) * 100),
});
quartileMetricData.push({
name: timestamp,
timestamp: col.timestamp || 0,
firstQuartile: col.firstQuartile || 0,
thirdQuartile: col.thirdQuartile || 0,
interQuartileRange: col.interQuartileRange || 0,
median: col.median || 0,
});
const profileMetric = calculateColumnProfilerMetrics({
columnProfilerData,
...columnMetric,
});
const countMetricInfo = countMetrics.information.map((item) => ({
...item,
latestValue:
countMetricData[countMetricData.length - 1]?.[item.dataKey] || 0,
}));
const proportionMetricInfo = proportionMetrics.information.map((item) => ({
...item,
latestValue: parseFloat(
`${
proportionMetricData[proportionMetricData.length - 1]?.[
item.dataKey
] || 0
}`
).toFixed(2),
}));
const mathMetricInfo = mathMetrics.information.map((item) => ({
...item,
latestValue:
mathMetricData[mathMetricData.length - 1]?.[item.dataKey] || 0,
}));
const sumMetricInfo = sumMetrics.information.map((item) => ({
...item,
latestValue: sumMetricData[sumMetricData.length - 1]?.[item.dataKey] || 0,
}));
const quartileMetricInfo = quartileMetrics.information.map((item) => ({
...item,
latestValue:
quartileMetricData[quartileMetricData.length - 1]?.[item.dataKey] || 0,
}));
setCountMetrics((pre) => ({
...pre,
information: countMetricInfo,
data: countMetricData,
}));
setProportionMetrics((pre) => ({
...pre,
information: proportionMetricInfo,
data: proportionMetricData,
}));
setMathMetrics((pre) => ({
...pre,
information: mathMetricInfo,
data: mathMetricData,
}));
setSumMetrics((pre) => ({
...pre,
information: sumMetricInfo,
data: sumMetricData,
}));
setQuartileMetrics((pre) => ({
...pre,
information: quartileMetricInfo,
data: quartileMetricData,
}));
setColumnMetric(profileMetric);
// only min/max category can be string
const isMinMaxString =
isString(updateProfilerData[0]?.min) ||
isString(updateProfilerData[0]?.max);
isString(columnProfilerData[0]?.min) ||
isString(columnProfilerData[0]?.max);
setIsMinMaxStringData(isMinMaxString);
};
@ -243,7 +128,7 @@ const SingleColumnProfile: FC<SingleColumnProfileProps> = ({
gutter={[16, 16]}>
<Col span={24}>
<ProfilerDetailsCard
chartCollection={countMetrics}
chartCollection={columnMetric.countMetrics}
isLoading={isLoading}
name="count"
title={t('label.data-count-plural')}
@ -251,7 +136,7 @@ const SingleColumnProfile: FC<SingleColumnProfileProps> = ({
</Col>
<Col span={24}>
<ProfilerDetailsCard
chartCollection={proportionMetrics}
chartCollection={columnMetric.proportionMetrics}
isLoading={isLoading}
name="proportion"
tickFormatter="%"
@ -260,7 +145,7 @@ const SingleColumnProfile: FC<SingleColumnProfileProps> = ({
</Col>
<Col span={24}>
<ProfilerDetailsCard
chartCollection={mathMetrics}
chartCollection={columnMetric.mathMetrics}
isLoading={isLoading}
name="math"
showYAxisCategory={isMinMaxStringData}
@ -270,7 +155,7 @@ const SingleColumnProfile: FC<SingleColumnProfileProps> = ({
</Col>
<Col span={24}>
<ProfilerDetailsCard
chartCollection={sumMetrics}
chartCollection={columnMetric.sumMetrics}
isLoading={isLoading}
name="sum"
title={t('label.data-aggregate')}
@ -278,7 +163,7 @@ const SingleColumnProfile: FC<SingleColumnProfileProps> = ({
</Col>
<Col span={24}>
<ProfilerDetailsCard
chartCollection={quartileMetrics}
chartCollection={columnMetric.quartileMetrics}
isLoading={isLoading}
name="quartile"
title={t('label.data-quartile-plural')}

8
openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts
View File

@ -417,3 +417,11 @@ export const TEST_CASE_STATUS_OPTION = [
value: value,
})),
];
export const INITIAL_COLUMN_METRICS_VALUE = {
countMetrics: INITIAL_COUNT_METRIC_VALUE,
proportionMetrics: INITIAL_PROPORTION_METRIC_VALUE,
mathMetrics: INITIAL_MATH_METRIC_VALUE,
sumMetrics: INITIAL_SUM_METRIC_VALUE,
quartileMetrics: INITIAL_QUARTILE_METRIC_VALUE,
};

53
openmetadata-ui/src/main/resources/ui/src/utils/ObjectUtils.test.ts Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright 2023 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { isHasKey } from './ObjectUtils';
const mockIsHasKeyData = {
name: 'John',
age: 30,
address: '123 Main St',
};
describe('ObjectUtils', () => {
it('isHasKey should return true if all keys are present in the object', () => {
const keys = ['name', 'age', 'address'];
const result = isHasKey(mockIsHasKeyData, keys, true);
expect(result).toBe(true);
});
it('isHasKey should return false if all keys are not present in the object', () => {
const keys = ['name', 'age', 'address', 'gender'];
const result = isHasKey(mockIsHasKeyData, keys, true);
expect(result).toBe(false);
});
it('isHasKey should return true if at least one key is present in the object', () => {
const keys = ['name', 'gender'];
const result = isHasKey(mockIsHasKeyData, keys);
expect(result).toBe(true);
});
it('isHasKey should return false if none of the keys are present in the object', () => {
const keys = ['gender', 'occupation'];
const result = isHasKey(mockIsHasKeyData, keys);
expect(result).toBe(false);
});
});

32
openmetadata-ui/src/main/resources/ui/src/utils/ObjectUtils.ts Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright 2023 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { has } from 'lodash';
/**
* Checks if the given object has the specified key(s).
*
* @template T - The type of the object.
* @param {T} data - The object to check.
* @param {string[]} keys - The key(s) to check for.
* @param {boolean} checkAllKey - If true, checks if all keys are present; if false, checks if any key is present.
* @returns {boolean} True if the object has the key(s), false otherwise.
*/
export const isHasKey = <T>(
data: T,
keys: string[],
checkAllKey = false
): boolean => {
const func = (item: string) => has(data, item);
return checkAllKey ? keys.every(func) : keys.some(func);
};

26
openmetadata-ui/src/main/resources/ui/src/utils/TableProfilerUtils.interface.ts Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright 2023 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { MetricChartType } from '../components/ProfilerDashboard/profilerDashboard.interface';
import { ColumnProfile } from '../generated/entity/data/table';
export interface ColumnMetricsInterface {
countMetrics: MetricChartType;
proportionMetrics: MetricChartType;
mathMetrics: MetricChartType;
sumMetrics: MetricChartType;
quartileMetrics: MetricChartType;
}
export interface CalculateColumnProfilerMetricsInterface
extends ColumnMetricsInterface {
columnProfilerData: ColumnProfile[];
}

248
openmetadata-ui/src/main/resources/ui/src/utils/TableProfilerUtils.test.ts
View File

@ -12,9 +12,11 @@
*/
import { Table, TableProfile } from '../generated/entity/data/table';
import {
calculateColumnProfilerMetrics,
calculateCustomMetrics,
getColumnCustomMetric,
} from './TableProfilerUtils';
import { CalculateColumnProfilerMetricsInterface } from './TableProfilerUtils.interface';
jest.mock('./date-time/DateTimeUtils', () => {
return {
@ -22,11 +24,71 @@ jest.mock('./date-time/DateTimeUtils', () => {
};
});
const columnFqn = 'fqn1';
const customMetrics = [
{
id: 'id1',
name: 'name1',
expression: 'expression1',
updatedAt: 1701757494892,
updatedBy: 'admin',
},
];
const table = {
fullyQualifiedName: 'fqn',
name: 'name',
columns: [
{
fullyQualifiedName: 'fqn1',
name: 'name1',
customMetrics: customMetrics,
},
],
} as unknown as Table;
const countMetrics = {
information: [
{ label: 'Distinct Count', dataKey: 'distinctCount' },
{ label: 'Null Count', dataKey: 'nullCount' },
{ label: 'Unique Count', dataKey: 'uniqueCount' },
{ label: 'Values Count', dataKey: 'valuesCount' },
],
};
const proportionMetrics = {
information: [
{ label: 'Distinct Proportion', dataKey: 'distinctProportion' },
{ label: 'Null Proportion', dataKey: 'nullProportion' },
{ label: 'Unique Proportion', dataKey: 'uniqueProportion' },
],
};
const mathMetrics = {
information: [
{ label: 'Max', dataKey: 'max' },
{ label: 'Min', dataKey: 'min' },
{ label: 'Mean', dataKey: 'mean' },
],
};
const sumMetrics = {
information: [{ label: 'Sum', dataKey: 'sum' }],
};
const quartileMetrics = {
information: [
{ label: 'First Quartile', dataKey: 'firstQuartile' },
{ label: 'Third Quartile', dataKey: 'thirdQuartile' },
{ label: 'Inter Quartile Range', dataKey: 'interQuartileRange' },
{ label: 'Median', dataKey: 'median' },
],
};
describe('TableProfilerUtils', () => {
it('calculateCustomMetrics should return correct data', () => {
const profiler = [
{
timestamp: 1701757499196,
timestamp: 1701757494892,
profileSampleType: 'PERCENTAGE',
columnCount: 12,
rowCount: 14567,
@ -69,14 +131,14 @@ describe('TableProfilerUtils', () => {
{
CountOfFRAddress: 1467,
formattedTimestamp: 'Dec 05, 11:54',
timestamp: 1701757499196,
timestamp: 1701757494892,
},
],
CountOfUSAddress: [
{
CountOfUSAddress: 15467,
formattedTimestamp: 'Dec 05, 11:54',
timestamp: 1701757499196,
timestamp: 1701757494892,
},
],
});
@ -89,54 +151,12 @@ describe('TableProfilerUtils', () => {
});
it('getColumnCustomMetric should return correct data', () => {
const customMetrics = [
{
id: 'id1',
name: 'name1',
expression: 'expression1',
updatedAt: 1701757494892,
updatedBy: 'admin',
},
];
const table = {
fullyQualifiedName: 'fqn',
name: 'name',
columns: [
{
fullyQualifiedName: 'fqn1',
name: 'name1',
customMetrics: customMetrics,
},
],
} as unknown as Table;
const columnFqn = 'fqn1';
const data = getColumnCustomMetric(table, columnFqn);
expect(data).toEqual(customMetrics);
});
it('getColumnCustomMetric should return undefined if table, fqn and both is not provided', () => {
const columnFqn = 'fqn1';
const customMetrics = [
{
id: 'id1',
name: 'name1',
expression: 'expression1',
updatedAt: 1701757494892,
updatedBy: 'admin',
},
];
const table = {
fullyQualifiedName: 'fqn',
name: 'name',
columns: [
{
fullyQualifiedName: 'fqn1',
name: 'name1',
customMetrics: customMetrics,
},
],
} as unknown as Table;
const withoutTable = getColumnCustomMetric(undefined, columnFqn);
const withoutFqn = getColumnCustomMetric(table, undefined);
const emptyData = getColumnCustomMetric();
@ -145,4 +165,142 @@ describe('TableProfilerUtils', () => {
expect(withoutFqn).toBeUndefined();
expect(emptyData).toBeUndefined();
});
it('calculateColumnProfilerMetrics should calculate column profiler metrics correctly', () => {
const columnProfilerData = [
{
timestamp: 1701757494892,
distinctCount: 100,
nullCount: 10,
uniqueCount: 90,
valuesCount: 200,
sum: 500,
max: 100,
min: 0,
mean: 50,
distinctProportion: 0.5,
nullProportion: 0.05,
uniqueProportion: 0.45,
firstQuartile: 25,
thirdQuartile: 75,
interQuartileRange: 50,
median: 50,
},
];
const result = calculateColumnProfilerMetrics({
columnProfilerData,
countMetrics,
proportionMetrics,
mathMetrics,
sumMetrics,
quartileMetrics,
} as unknown as CalculateColumnProfilerMetricsInterface);
expect(result.countMetrics.data).toEqual([
{
name: 'Dec 05, 11:54',
timestamp: 1701757494892,
distinctCount: 100,
nullCount: 10,
uniqueCount: 90,
valuesCount: 200,
},
]);
expect(result.proportionMetrics.data).toEqual([
{
name: 'Dec 05, 11:54',
timestamp: 1701757494892,
distinctProportion: 50,
nullProportion: 5,
uniqueProportion: 45,
},
]);
expect(result.mathMetrics.data).toEqual([
{
name: 'Dec 05, 11:54',
timestamp: 1701757494892,
max: 100,
min: 0,
mean: 50,
},
]);
expect(result.sumMetrics.data).toEqual([
{
name: 'Dec 05, 11:54',
timestamp: 1701757494892,
sum: 500,
},
]);
expect(result.quartileMetrics.data).toEqual([
{
name: 'Dec 05, 11:54',
timestamp: 1701757494892,
firstQuartile: 25,
thirdQuartile: 75,
interQuartileRange: 50,
median: 50,
},
]);
});
it('calculateColumnProfilerMetrics should only calculate metric based on available data', () => {
const columnProfilerData = [
{
timestamp: 1701757494892,
distinctCount: 100,
nullCount: 10,
uniqueCount: 90,
valuesCount: 200,
max: 100,
min: 0,
distinctProportion: 0.5,
nullProportion: 0.05,
uniqueProportion: 0.45,
},
];
const result = calculateColumnProfilerMetrics({
columnProfilerData,
countMetrics,
proportionMetrics,
mathMetrics,
sumMetrics,
quartileMetrics,
} as unknown as CalculateColumnProfilerMetricsInterface);
expect(result.countMetrics.data).toEqual([
{
name: 'Dec 05, 11:54',
timestamp: 1701757494892,
distinctCount: 100,
nullCount: 10,
uniqueCount: 90,
valuesCount: 200,
},
]);
expect(result.proportionMetrics.data).toEqual([
{
name: 'Dec 05, 11:54',
timestamp: 1701757494892,
distinctProportion: 50,
nullProportion: 5,
uniqueProportion: 45,
},
]);
expect(result.mathMetrics.data).toEqual([
{
name: 'Dec 05, 11:54',
timestamp: 1701757494892,
max: 100,
min: 0,
},
]);
expect(result.sumMetrics.data).toEqual([]);
expect(result.quartileMetrics.data).toEqual([]);
});
});

161
openmetadata-ui/src/main/resources/ui/src/utils/TableProfilerUtils.ts
View File

@ -11,12 +11,17 @@
* limitations under the License.
*/
import { findLast, isUndefined, sortBy } from 'lodash';
import { findLast, isUndefined, last, sortBy } from 'lodash';
import { MetricChartType } from '../components/ProfilerDashboard/profilerDashboard.interface';
import { SystemProfile } from '../generated/api/data/createTableProfile';
import { Table, TableProfile } from '../generated/entity/data/table';
import { CustomMetric } from '../generated/tests/customMetric';
import { customFormatDateTime } from './date-time/DateTimeUtils';
import { isHasKey } from './ObjectUtils';
import {
CalculateColumnProfilerMetricsInterface,
ColumnMetricsInterface,
} from './TableProfilerUtils.interface';
export const calculateRowCountMetrics = (
profiler: TableProfile[],
@ -31,13 +36,13 @@ export const calculateRowCountMetrics = (
rowCountMetricData.push({
name: timestamp,
timestamp: data.timestamp,
rowCount: Number(data.rowCount),
rowCount: data.rowCount,
});
});
const countMetricInfo = currentMetrics.information.map((item) => ({
...item,
latestValue:
rowCountMetricData[rowCountMetricData.length - 1]?.[item.dataKey] || 0,
rowCountMetricData[rowCountMetricData.length - 1]?.[item.dataKey],
}));
return { data: rowCountMetricData, information: countMetricInfo };
@ -58,13 +63,13 @@ export const calculateSystemMetrics = (
operationMetrics.push({
name: timestamp,
timestamp: Number(data.timestamp),
[data.operation || 'value']: Number(data.rowsAffected),
[data.operation ?? 'value']: data.rowsAffected,
});
operationDateMetrics.push({
name: timestamp,
timestamp: Number(data.timestamp),
data: data.rowsAffected || 0,
[data.operation || 'value']: 5,
data: data.rowsAffected,
[data.operation ?? 'value']: 5,
});
});
const operationMetricsInfo = currentMetrics.information.map((item) => {
@ -76,7 +81,7 @@ export const calculateSystemMetrics = (
return {
...item,
stackId: stackId,
latestValue: operation?.rowsAffected ?? 0,
latestValue: operation?.rowsAffected,
};
});
const operationDateMetricsInfo = currentMetrics.information.map((item) => {
@ -145,3 +150,145 @@ export const getColumnCustomMetric = (table?: Table, columnFqn?: string) => {
(column) => column.fullyQualifiedName === columnFqn
)?.customMetrics;
};
export const calculateColumnProfilerMetrics = ({
columnProfilerData,
countMetrics,
proportionMetrics,
mathMetrics,
sumMetrics,
quartileMetrics,
}: CalculateColumnProfilerMetricsInterface): ColumnMetricsInterface => {
const updateProfilerData = sortBy(columnProfilerData, 'timestamp');
const countMetricData: MetricChartType['data'] = [];
const proportionMetricData: MetricChartType['data'] = [];
const mathMetricData: MetricChartType['data'] = [];
const sumMetricData: MetricChartType['data'] = [];
const quartileMetricData: MetricChartType['data'] = [];
updateProfilerData.forEach((col) => {
const { timestamp, sum } = col;
const name = customFormatDateTime(timestamp, 'MMM dd, hh:mm');
const defaultData = { name, timestamp };
if (
isHasKey(col, [
'distinctCount',
'nullCount',
'uniqueCount',
'valuesCount',
])
) {
const { distinctCount, nullCount, uniqueCount, valuesCount } = col;
countMetricData.push({
...defaultData,
distinctCount,
nullCount,
uniqueCount,
valuesCount,
});
}
if (isHasKey(col, ['sum'])) {
sumMetricData.push({
...defaultData,
sum,
});
}
if (isHasKey(col, ['max', 'min', 'mean'])) {
const { max, min, mean } = col;
mathMetricData.push({
...defaultData,
max,
min,
mean,
});
}
if (
isHasKey(col, [
'distinctProportion',
'nullProportion',
'uniqueProportion',
])
) {
const { distinctProportion, nullProportion, uniqueProportion } = col;
proportionMetricData.push({
...defaultData,
distinctProportion: isUndefined(distinctProportion)
? undefined
: Math.round(distinctProportion * 100),
nullProportion: isUndefined(nullProportion)
? undefined
: Math.round(nullProportion * 100),
uniqueProportion: isUndefined(uniqueProportion)
? undefined
: Math.round(uniqueProportion * 100),
});
}
if (
isHasKey(col, [
'firstQuartile',
'thirdQuartile',
'interQuartileRange',
'median',
])
) {
const { firstQuartile, thirdQuartile, interQuartileRange, median } = col;
quartileMetricData.push({
...defaultData,
firstQuartile,
thirdQuartile,
interQuartileRange,
median,
});
}
});
const countMetricInfo = countMetrics.information.map((item) => ({
...item,
latestValue: last(countMetricData)?.[item.dataKey],
}));
const proportionMetricInfo = proportionMetrics.information.map((item) => ({
...item,
latestValue: isUndefined(last(proportionMetricData)?.[item.dataKey])
? undefined
: parseFloat(`${last(proportionMetricData)?.[item.dataKey]}`).toFixed(2),
}));
const mathMetricInfo = mathMetrics.information.map((item) => ({
...item,
latestValue: last(mathMetricData)?.[item.dataKey],
}));
const sumMetricInfo = sumMetrics.information.map((item) => ({
...item,
latestValue: last(sumMetricData)?.[item.dataKey],
}));
const quartileMetricInfo = quartileMetrics.information.map((item) => ({
...item,
latestValue: last(quartileMetricData)?.[item.dataKey],
}));
return {
countMetrics: {
information: countMetricInfo,
data: countMetricData,
},
proportionMetrics: {
information: proportionMetricInfo,
data: proportionMetricData,
},
mathMetrics: {
information: mathMetricInfo,
data: mathMetricData,
},
sumMetrics: {
information: sumMetricInfo,
data: sumMetricData,
},
quartileMetrics: {
information: quartileMetricInfo,
data: quartileMetricData,
},
};
};