diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Chart/Chart.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Chart/Chart.interface.ts index fddadca537d..9dd77c183ac 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Chart/Chart.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Chart/Chart.interface.ts @@ -11,6 +11,7 @@ * limitations under the License. */ +import { ColumnProfile } from 'generated/entity/data/table'; import { MetricChartType } from '../ProfilerDashboard/profilerDashboard.interface'; export interface CustomBarChartProps { @@ -18,3 +19,10 @@ export interface CustomBarChartProps { name: string; tickFormatter?: string; } + +export interface DataDistributionHistogramProps { + data: { + firstDayData?: ColumnProfile; + currentDayData?: ColumnProfile; + }; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Chart/DataDistributionHistogram.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Chart/DataDistributionHistogram.component.tsx new file mode 100644 index 00000000000..b05f514b56d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Chart/DataDistributionHistogram.component.tsx @@ -0,0 +1,132 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Col, Row, Tag } from 'antd'; +import ErrorPlaceHolder from 'components/common/error-with-placeholder/ErrorPlaceHolder'; +import { GRAPH_BACKGROUND_COLOR } from 'constants/constants'; +import { DEFAULT_HISTOGRAM_DATA } from 'constants/profiler.constant'; +import { HistogramClass } from 'generated/entity/data/table'; +import { isUndefined, map } from 'lodash'; +import React from 'react'; +import { useTranslation } from 'react-i18next'; +import { + Bar, + BarChart, + CartesianGrid, + Legend, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; +import { axisTickFormatter, tooltipFormatter } from 'utils/ChartUtils'; +import { getFormattedDateFromSeconds } from 'utils/TimeUtils'; +import { DataDistributionHistogramProps } from './Chart.interface'; + +const DataDistributionHistogram = ({ + data, +}: DataDistributionHistogramProps) => { + const { t } = useTranslation(); + const showSingleGraph = + isUndefined(data.firstDayData?.histogram) || + isUndefined(data.currentDayData?.histogram); + + if ( + isUndefined(data.firstDayData?.histogram) && + isUndefined(data.currentDayData?.histogram) + ) { + return ( + + + +

{t('message.no-data-available')}

+ + + + ); + } + + return ( + + {map(data, (columnProfile, key) => { + if (isUndefined(columnProfile?.histogram)) { + return; + } + + const histogramData = + (columnProfile?.histogram as HistogramClass) || + DEFAULT_HISTOGRAM_DATA; + + const graphData = histogramData.frequencies?.map((frequency, i) => ({ + name: histogramData?.boundaries?.[i], + frequency, + })); + + const graphDate = getFormattedDateFromSeconds( + columnProfile?.timestamp || 0, + 'dd/MMM' + ); + + return ( + + + + {graphDate} + + + {`${t('label.skew')}: ${ + columnProfile?.nonParametricSkew || '--' + }`} + + + + + + + axisTickFormatter(props)} + /> + + tooltipFormatter(value)} + /> + + + + + + + ); + })} + + ); +}; + +export default DataDistributionHistogram; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Chart/DataDistributionHistogram.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Chart/DataDistributionHistogram.test.tsx new file mode 100644 index 00000000000..41ba7eb52ed --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Chart/DataDistributionHistogram.test.tsx @@ -0,0 +1,164 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { queryByAttribute, render, screen } from '@testing-library/react'; +import React from 'react'; +import DataDistributionHistogram from './DataDistributionHistogram.component'; + +const MOCK_HISTOGRAM_DATA = [ + { + name: 'shop_id', + timestamp: 1678375427, + valuesCount: 14567.0, + nullCount: 0.0, + nullProportion: 0.0, + uniqueCount: 14567.0, + uniqueProportion: 1.0, + distinctCount: 14509.0, + distinctProportion: 1.0, + min: 1.0, + max: 587.0, + mean: 45.0, + sum: 1367.0, + stddev: 35.0, + median: 7654.0, + firstQuartile: 7.4, + thirdQuartile: 8766.5, + interQuartileRange: 8002.1, + nonParametricSkew: -0.567, + histogram: { + boundaries: [ + '5.00 to 100.00', + '100.00 to 200.00', + '200.00 to 300.00', + '300.00 and up', + ], + frequencies: [101, 235, 123, 98], + }, + }, + { + name: 'shop_id', + timestamp: 1678202627, + valuesCount: 10256.0, + nullCount: 0.0, + nullProportion: 0.0, + uniqueCount: 10098.0, + uniqueProportion: 0.91, + distinctCount: 10256.0, + distinctProportion: 1.0, + min: 1.0, + max: 542.0, + mean: 45.0, + sum: 1367.0, + stddev: 35.0, + median: 7344.0, + firstQuartile: 7.4, + thirdQuartile: 8005.5, + interQuartileRange: 8069.1, + nonParametricSkew: -0.567, + histogram: { + boundaries: [ + '5.00 to 100.00', + '100.00 to 200.00', + '200.00 to 300.00', + '300.00 and up', + ], + frequencies: [56, 62, 66, 99], + }, + }, +]; + +const COLUMN_PROFILER = { + name: 'shop_id', + timestamp: 1678169698, + valuesCount: 10256.0, + nullCount: 0.0, + nullProportion: 0.0, + uniqueCount: 10098.0, + uniqueProportion: 0.91, + distinctCount: 10256.0, + distinctProportion: 1.0, + min: 1.0, + max: 542.0, + mean: 45.0, + sum: 1367.0, + stddev: 35.0, + median: 7344.0, +}; + +jest.mock('components/common/error-with-placeholder/ErrorPlaceHolder', () => { + return jest.fn().mockImplementation(({ children }) =>
{children}
); +}); + +describe('DataDistributionHistogram component test', () => { + it('Component should render', async () => { + const { container } = render( + + ); + const skewTags = await screen.findAllByTestId('skew-tag'); + const date = await screen.findAllByTestId('date'); + + expect(await screen.findByTestId('chart-container')).toBeInTheDocument(); + expect( + queryByAttribute('id', container, 'firstDayData-histogram') + ).toBeInTheDocument(); + expect( + queryByAttribute('id', container, 'currentDayData-histogram') + ).toBeInTheDocument(); + expect(skewTags).toHaveLength(2); + expect(date).toHaveLength(2); + }); + + it('Render one graph if histogram data is available in only one profile data', async () => { + const { container } = render( + + ); + const skewTags = await screen.findAllByTestId('skew-tag'); + const date = await screen.findAllByTestId('date'); + + expect(await screen.findByTestId('chart-container')).toBeInTheDocument(); + expect( + queryByAttribute('id', container, 'firstDayData-histogram') + ).not.toBeInTheDocument(); + expect( + queryByAttribute('id', container, 'currentDayData-histogram') + ).toBeInTheDocument(); + expect(skewTags).toHaveLength(1); + expect(date).toHaveLength(1); + }); + + it('No data placeholder should render when firstDay & currentDay data is undefined', async () => { + render( + + ); + + expect( + await screen.findByText('message.no-data-available') + ).toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.test.tsx index e8122661346..6a34f80f4e4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.test.tsx @@ -45,6 +45,11 @@ jest.mock('./ProfilerSummaryCard', () => { jest.mock('./ProfilerDetailsCard', () => { return jest.fn().mockImplementation(() =>
ProfilerDetailsCard
); }); +jest.mock('components/Chart/DataDistributionHistogram.component', () => { + return jest + .fn() + .mockImplementation(() =>
DataDistributionHistogram
); +}); jest.mock('react-i18next', () => ({ // this mock makes sure any components using the translate hook can use it without a warning being shown @@ -65,6 +70,7 @@ describe('Test ProfilerTab component', () => { const pageContainer = await screen.findByTestId('profiler-tab-container'); const description = await screen.findByTestId('description'); + const histogram = await screen.findByTestId('histogram-metrics'); const dataTypeContainer = await screen.findByTestId('data-type-container'); const ProfilerSummaryCards = await screen.findAllByText( 'ProfilerSummaryCard' @@ -76,8 +82,9 @@ describe('Test ProfilerTab component', () => { expect(pageContainer).toBeInTheDocument(); expect(description).toBeInTheDocument(); expect(dataTypeContainer).toBeInTheDocument(); + expect(histogram).toBeInTheDocument(); expect(ProfilerSummaryCards).toHaveLength(2); - expect(ProfilerDetailsCards).toHaveLength(4); + expect(ProfilerDetailsCards).toHaveLength(5); }); it('ProfilerTab component should render properly with empty data', async () => { @@ -97,6 +104,7 @@ describe('Test ProfilerTab component', () => { const pageContainer = await screen.findByTestId('profiler-tab-container'); const description = await screen.findByTestId('description'); const dataTypeContainer = await screen.findByTestId('data-type-container'); + const histogram = await screen.findByTestId('histogram-metrics'); const ProfilerSummaryCards = await screen.findAllByText( 'ProfilerSummaryCard' ); @@ -107,8 +115,9 @@ describe('Test ProfilerTab component', () => { expect(pageContainer).toBeInTheDocument(); expect(description).toBeInTheDocument(); expect(dataTypeContainer).toBeInTheDocument(); + expect(histogram).toBeInTheDocument(); expect(ProfilerSummaryCards).toHaveLength(2); - expect(ProfilerDetailsCards).toHaveLength(4); + expect(ProfilerDetailsCards).toHaveLength(5); }); it('ProfilerTab component should render properly even if getListTestCase API fails', async () => { @@ -123,6 +132,7 @@ describe('Test ProfilerTab component', () => { const pageContainer = await screen.findByTestId('profiler-tab-container'); const description = await screen.findByTestId('description'); + const histogram = await screen.findByTestId('histogram-metrics'); const dataTypeContainer = await screen.findByTestId('data-type-container'); const ProfilerSummaryCards = await screen.findAllByText( 'ProfilerSummaryCard' @@ -134,7 +144,8 @@ describe('Test ProfilerTab component', () => { expect(pageContainer).toBeInTheDocument(); expect(description).toBeInTheDocument(); expect(dataTypeContainer).toBeInTheDocument(); + expect(histogram).toBeInTheDocument(); expect(ProfilerSummaryCards).toHaveLength(2); - expect(ProfilerDetailsCards).toHaveLength(4); + expect(ProfilerDetailsCards).toHaveLength(5); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.tsx index b877824f23b..b87a0f84c80 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/ProfilerTab.tsx @@ -13,7 +13,8 @@ import { Card, Col, Row, Statistic, Typography } from 'antd'; import { AxiosError } from 'axios'; -import { sortBy } from 'lodash'; +import DataDistributionHistogram from 'components/Chart/DataDistributionHistogram.component'; +import { first, last, sortBy } from 'lodash'; import React, { useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useParams } from 'react-router-dom'; @@ -23,6 +24,7 @@ import { INITIAL_COUNT_METRIC_VALUE, INITIAL_MATH_METRIC_VALUE, INITIAL_PROPORTION_METRIC_VALUE, + INITIAL_QUARTILE_METRIC_VALUE, INITIAL_SUM_METRIC_VALUE, INITIAL_TEST_RESULT_SUMMARY, } from '../../../constants/profiler.constant'; @@ -58,6 +60,9 @@ const ProfilerTab: React.FC = ({ const [sumMetrics, setSumMetrics] = useState( INITIAL_SUM_METRIC_VALUE ); + const [quartileMetrics, setQuartileMetrics] = useState( + INITIAL_QUARTILE_METRIC_VALUE + ); const [tableTests, setTableTests] = useState({ tests: [], results: INITIAL_TEST_RESULT_SUMMARY, @@ -105,12 +110,20 @@ const ProfilerTab: React.FC = ({ ]; }, [tableTests]); + const { firstDay, currentDay } = useMemo(() => { + return { + firstDay: last(profilerData), + currentDay: first(profilerData), + }; + }, [profilerData]); + const createMetricsChartData = () => { const updateProfilerData = sortBy(profilerData, 'timestamp'); const countMetricData: MetricChartType['data'] = []; const proportionMetricData: MetricChartType['data'] = []; const mathMetricData: MetricChartType['data'] = []; const sumMetricData: MetricChartType['data'] = []; + const quartileMetricData: MetricChartType['data'] = []; updateProfilerData.forEach((col) => { const x = getFormattedDateFromSeconds(col.timestamp); @@ -135,7 +148,6 @@ const ProfilerTab: React.FC = ({ max: (col.max as number) || 0, min: (col.min as number) || 0, mean: col.mean || 0, - median: col.median || 0, }); proportionMetricData.push({ @@ -145,6 +157,15 @@ const ProfilerTab: React.FC = ({ nullProportion: Math.round((col.nullProportion || 0) * 100), uniqueProportion: Math.round((col.uniqueProportion || 0) * 100), }); + + quartileMetricData.push({ + name: x, + timestamp: col.timestamp || 0, + firstQuartile: col.firstQuartile || 0, + thirdQuartile: col.thirdQuartile || 0, + interQuartileRange: col.interQuartileRange || 0, + median: col.median || 0, + }); }); const countMetricInfo = countMetrics.information.map((item) => ({ @@ -171,6 +192,11 @@ const ProfilerTab: React.FC = ({ ...item, latestValue: sumMetricData[sumMetricData.length - 1]?.[item.dataKey] || 0, })); + const quartileMetricInfo = quartileMetrics.information.map((item) => ({ + ...item, + latestValue: + quartileMetricData[quartileMetricData.length - 1]?.[item.dataKey] || 0, + })); setCountMetrics((pre) => ({ ...pre, @@ -192,6 +218,11 @@ const ProfilerTab: React.FC = ({ information: sumMetricInfo, data: sumMetricData, })); + setQuartileMetrics((pre) => ({ + ...pre, + information: quartileMetricInfo, + data: quartileMetricData, + })); }; const fetchAllTests = async () => { @@ -227,7 +258,10 @@ const ProfilerTab: React.FC = ({ }, []); return ( - + @@ -294,6 +328,30 @@ const ProfilerTab: React.FC = ({ + + + + + + + + + {t('label.data-distribution')} + + + + + + + + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts index d164b344979..86e26dfff85 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts @@ -13,6 +13,7 @@ import { t } from 'i18next'; import { StepperStepType } from 'Models'; +import i18n from 'utils/i18next/LocalUtil'; import { CSMode } from '../enums/codemirror.enum'; import { DMLOperationType } from '../generated/api/data/createTableProfile'; import { @@ -170,25 +171,20 @@ export const INITIAL_PROPORTION_METRIC_VALUE = { export const INITIAL_MATH_METRIC_VALUE = { information: [ - { - title: t('label.median'), - dataKey: 'median', - color: '#1890FF', - }, { title: t('label.max'), dataKey: 'max', - color: '#7147E8', + color: '#1890FF', }, { title: t('label.mean'), dataKey: 'mean', - color: '#008376', + color: '#7147E8', }, { title: t('label.min'), dataKey: 'min', - color: '#B02AAC', + color: '#008376', }, ], data: [], @@ -204,6 +200,31 @@ export const INITIAL_SUM_METRIC_VALUE = { ], data: [], }; +export const INITIAL_QUARTILE_METRIC_VALUE = { + information: [ + { + title: i18n.t('label.first-quartile'), + dataKey: 'firstQuartile', + color: '#1890FF', + }, + { + title: i18n.t('label.median'), + dataKey: 'median', + color: '#7147E8', + }, + { + title: i18n.t('label.inter-quartile-range'), + dataKey: 'interQuartileRange', + color: '#008376', + }, + { + title: i18n.t('label.third-quartile'), + dataKey: 'thirdQuartile', + color: '#B02AAC', + }, + ], + data: [], +}; export const INITIAL_ROW_METRIC_VALUE = { information: [ @@ -327,3 +348,8 @@ export const PROFILE_SAMPLE_OPTIONS = [ value: ProfileSampleType.Rows, }, ]; + +export const DEFAULT_HISTOGRAM_DATA = { + boundaries: [], + frequencies: [], +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index 3876dcb4696..1c582fb4221 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -165,6 +165,7 @@ "data-asset-type": "Data Asset Type", "data-assets-report": "Data Assets Report", "data-assets-with-tier-plural": "Data Assets with Tiers", + "data-distribution": "Data Distribution", "data-entity": "Data {{entity}}", "data-insight": "Data Insight", "data-insight-active-user-summary": "Most Active Users", @@ -295,6 +296,7 @@ "filter-plural": "Filters", "first": "First", "first-lowercase": "first", + "first-quartile": "First Quartile", "flush-interval-secs": "Flush Interval (secs)", "follow": "Follow", "followed-lowercase": "followed", @@ -365,6 +367,7 @@ "install-service-connectors": "Install Service Connectors", "instance-lowercase": "instance", "integration-plural": "Integrations", + "inter-quartile-range": "Inter Quartile Range", "interval": "Interval", "interval-type": "Interval Type", "interval-unit": "Interval Unit", @@ -676,6 +679,7 @@ "show-deleted-team": "Show Deleted Team", "show-or-hide-advanced-config": "{{showAdv}} Advanced Config", "sign-in-with-sso": "Sign in with {{sso}}", + "skew": "Skew", "slack": "Slack", "soft-delete": "Soft Delete", "soft-lowercase": "soft", @@ -746,6 +750,7 @@ "testing-connection": "Testing Connection", "tests-summary": "Tests Summary", "text": "Text", + "third-quartile": "Third Quartile", "thread": "Thread", "three-dash-symbol": "---", "three-dots-symbol": "•••", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index 5d4ad9994ba..5732cf4a5b0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -165,6 +165,7 @@ "data-asset-type": "Type de Resources de Données", "data-assets-report": "Data Assets Report", "data-assets-with-tier-plural": "Data Assets with Tiers", + "data-distribution": "Data Distribution", "data-entity": "Data {{entity}}", "data-insight": "Data Insight", "data-insight-active-user-summary": "Utilisateurs les plus Actfis", @@ -295,6 +296,7 @@ "filter-plural": "Filters", "first": "First", "first-lowercase": "first", + "first-quartile": "First Quartile", "flush-interval-secs": "Flush Interval (secs)", "follow": "Follow", "followed-lowercase": "followed", @@ -365,6 +367,7 @@ "install-service-connectors": "Install Service Connectors", "instance-lowercase": "instance", "integration-plural": "Integrations", + "inter-quartile-range": "Inter Quartile Range", "interval": "Interval", "interval-type": "Type d'Interval", "interval-unit": "Unité d'Interval", @@ -676,6 +679,7 @@ "show-deleted-team": "Show Deleted Team", "show-or-hide-advanced-config": "{{showAdv}} Config Avancée", "sign-in-with-sso": "Sign in with {{sso}}", + "skew": "Skew", "slack": "Slack", "soft-delete": "Suppression Logique (Soft delete)", "soft-lowercase": "soft", @@ -746,6 +750,7 @@ "testing-connection": "Testing Connection", "tests-summary": "Tests Summary", "text": "Text", + "third-quartile": "Third Quartile", "thread": "Thread", "three-dash-symbol": "---", "three-dots-symbol": "•••", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index 53b4ea4f6d9..355a52445fd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -165,6 +165,7 @@ "data-asset-type": "数据资产类型", "data-assets-report": "数据资产报告", "data-assets-with-tier-plural": "分层的数据资产", + "data-distribution": "Data Distribution", "data-entity": "数据 {{entity}}", "data-insight": "数据洞察", "data-insight-active-user-summary": "最活跃用户", @@ -295,6 +296,7 @@ "filter-plural": "过滤器", "first": "First", "first-lowercase": "first", + "first-quartile": "First Quartile", "flush-interval-secs": "刷新间隔 (secs)", "follow": "Follow", "followed-lowercase": "被关注", @@ -365,6 +367,7 @@ "install-service-connectors": "Install Service Connectors", "instance-lowercase": "instance", "integration-plural": "Integrations", + "inter-quartile-range": "Inter Quartile Range", "interval": "间隔", "interval-type": "间隔类型", "interval-unit": "间隔单位", @@ -676,6 +679,7 @@ "show-deleted-team": "Show Deleted Team", "show-or-hide-advanced-config": "{{showAdv}} 高级配置", "sign-in-with-sso": "Sign in with {{sso}}", + "skew": "Skew", "slack": "Slack", "soft-delete": "软删除", "soft-lowercase": "soft", @@ -746,6 +750,7 @@ "testing-connection": "Testing Connection", "tests-summary": "Tests Summary", "text": "Text", + "third-quartile": "Third Quartile", "thread": "Thread", "three-dash-symbol": "---", "three-dots-symbol": "•••",