yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-23 16:38:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Implement Data Quality Dashboards (Incident Manager + Data Quality) (#19231)

Browse Source 
* Fix: Implement Data Quality Dashboards (Incident Manager + Data Quality)

* added icon for test case status

* added filters in the api

* added filters for dq

* added filter of table/column

* added test coverage

* address the comments
This commit is contained in:
Shailesh Parmar 2025-01-22 18:53:43 +05:30 committed by GitHub
parent fd2575d244
commit 2c06bcf327
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 510 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/DataQuality.interface.ts
View File

@ -46,6 +46,11 @@ export type TestCaseSearchParams = {
dataQualityDimension?: string;
};
export type DataQualityPageParams = TestCaseSearchParams & {
owner?: string;
tags?: string[];
};
export interface IncidentTypeAreaChartWidgetProps {
title: string;
incidentStatusType: TestCaseResolutionStatusTypes;

64
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/SummaryPannel/SummaryPanel.component.tsx
View File

@ -11,56 +11,110 @@
* limitations under the License.
*/
import { Col, Row } from 'antd';
import React, { FC } from 'react';
import React, { FC, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { ReactComponent as TestCaseAbortedIcon } from '../../../assets/svg/aborted-status.svg';
import { ReactComponent as TestCaseIcon } from '../../../assets/svg/all-activity-v2.svg';
import { ReactComponent as TestCaseFailedIcon } from '../../../assets/svg/failed-status.svg';
import { ReactComponent as DataAssetsCoverageIcon } from '../../../assets/svg/ic-data-assets-coverage.svg';
import { ReactComponent as HealthCheckIcon } from '../../../assets/svg/ic-green-heart-border.svg';
import { ReactComponent as TestCaseSuccessIcon } from '../../../assets/svg/success-colored.svg';
import { SummaryCard } from '../../../components/common/SummaryCard/SummaryCard.component';
import { PRIMARY_COLOR } from '../../../constants/Color.constants';
import { SummaryPanelProps } from './SummaryPanel.interface';
export const SummaryPanel: FC<SummaryPanelProps> = ({
testSummary: summary,
isLoading = false,
showAdditionalSummary = false,
}: SummaryPanelProps) => {
const { t } = useTranslation();
const spanValue = useMemo(
() => (showAdditionalSummary ? 8 : 6),
[showAdditionalSummary]
);
return (
<Row wrap gutter={[16, 16]}>
<Col span={6}>
<Col span={spanValue}>
<SummaryCard
inverseLabel
cardBackgroundClass="bg-primary"
className="h-full"
isLoading={isLoading}
showProgressBar={false}
title={t('label.total-entity', { entity: t('label.test-plural') })}
titleIcon={
<TestCaseIcon color={PRIMARY_COLOR} height={16} width={16} />
}
total={summary?.total ?? 0}
value={summary?.total ?? 0}
/>
</Col>
<Col span={6}>
<Col span={spanValue}>
<SummaryCard
inverseLabel
cardBackgroundClass="bg-success"
isLoading={isLoading}
title={t('label.success')}
titleIcon={<TestCaseSuccessIcon height={16} width={16} />}
total={summary?.total ?? 0}
type="success"
value={summary?.success ?? 0}
/>
</Col>
<Col span={6}>
<Col span={spanValue}>
<SummaryCard
inverseLabel
cardBackgroundClass="bg-aborted"
isLoading={isLoading}
title={t('label.aborted')}
titleIcon={<TestCaseAbortedIcon height={16} width={16} />}
total={summary?.total ?? 0}
type="aborted"
value={summary?.aborted ?? 0}
/>
</Col>
<Col span={6}>
<Col span={spanValue}>
<SummaryCard
inverseLabel
cardBackgroundClass="bg-failed"
isLoading={isLoading}
title={t('label.failed')}
titleIcon={<TestCaseFailedIcon height={16} width={16} />}
total={summary?.total ?? 0}
type="failed"
value={summary?.failed ?? 0}
/>
</Col>
{showAdditionalSummary && (
<>
<Col span={spanValue}>
<SummaryCard
inverseLabel
cardBackgroundClass="bg-success"
isLoading={isLoading}
title={t('label.healthy-data-asset-plural')}
titleIcon={<HealthCheckIcon height={16} width={16} />}
total={summary?.totalDQEntities ?? 0}
type="success"
value={summary?.healthy ?? 0}
/>
</Col>
<Col span={spanValue}>
<SummaryCard
inverseLabel
cardBackgroundClass="bg-primary"
isLoading={isLoading}
title={t('label.data-asset-plural-coverage')}
titleIcon={<DataAssetsCoverageIcon height={16} width={16} />}
total={summary?.totalEntityCount ?? 0}
type="acknowledged"
value={summary?.totalDQEntities ?? 0}
/>
</Col>
</>
)}
</Row>
);
};

1
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/SummaryPannel/SummaryPanel.interface.ts
View File

@ -15,4 +15,5 @@ import { TestSummary } from '../../../generated/tests/testSuite';
export interface SummaryPanelProps {
testSummary: TestSummary;
isLoading?: boolean;
showAdditionalSummary?: boolean;
}

8
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/SummaryPannel/SummaryPanel.test.tsx
View File

@ -55,6 +55,14 @@ describe('SummaryPanel component', () => {
expect(summaryCards).toHaveLength(4);
});
it('Show additional summary card if showAdditionalSummary is true', async () => {
render(<SummaryPanel showAdditionalSummary testSummary={mockSummary} />);
const summaryCards = await screen.findAllByText('SummaryCard.component');
expect(summaryCards).toHaveLength(6);
});
it('should not call getTestCaseExecutionSummary API, if testSummary data is provided', async () => {
const mockGetTestCaseExecutionSummary =
getTestCaseExecutionSummary as jest.Mock;

5
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx
View File

@ -111,8 +111,8 @@ export const TestCases = () => {
return params as TestCaseSearchParams;
}, [location.search]);
const { searchValue = '' } = params;
const { searchValue = '' } = params;
const [testCase, setTestCase] = useState<TestCase[]>([]);
const [isLoading, setIsLoading] = useState<boolean>(true);
const [selectedFilter, setSelectedFilter] = useState<string[]>([
@ -416,7 +416,6 @@ export const TestCases = () => {
key: filter,
label: startCase(name),
value: filter,
onClick: handleMenuClick,
}));
}, []);
@ -497,6 +496,7 @@ export const TestCases = () => {
menu={{
items: filterMenu,
selectedKeys: selectedFilter,
onClick: handleMenuClick,
}}
trigger={['click']}>
<Button
@ -641,6 +641,7 @@ export const TestCases = () => {
</Col>
<Col span={24}>
<SummaryPanel
showAdditionalSummary
isLoading={isTestCaseSummaryLoading}
testSummary={testCaseSummary}
/>

1
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx
View File

@ -321,6 +321,7 @@ export const TestSuites = () => {
<Col span={24}>
<SummaryPanel
showAdditionalSummary
isLoading={isTestCaseSummaryLoading}
testSummary={testCaseSummary}
/>

22
openmetadata-ui/src/main/resources/ui/src/components/common/SummaryCard/SummaryCard.component.tsx
View File

@ -27,6 +27,9 @@ export const SummaryCard = ({
showProgressBar = true,
className,
isLoading = false,
inverseLabel,
titleIcon,
cardBackgroundClass,
}: SummaryCardProps) => {
const percent = useMemo(() => {
if (isNumber(value)) {
@ -48,14 +51,19 @@ export const SummaryCard = ({
return (
<Space
className={classNames('summary-card', className)}
className={classNames('summary-card', cardBackgroundClass, className)}
data-testid="summary-card-container">
<div>
<Typography.Paragraph
className="summary-card-title"
data-testid="summary-card-title">
{title}
</Typography.Paragraph>
<div
className={classNames({ 'inverse-label': inverseLabel })}
data-testid="summary-card-label">
<Space align="center" size={8}>
{titleIcon}
<Typography.Paragraph
className="summary-card-title"
data-testid="summary-card-title">
{title}
</Typography.Paragraph>
</Space>
<Typography.Paragraph
className="summary-card-description"
data-testid="summary-card-description">

7
openmetadata-ui/src/main/resources/ui/src/components/common/SummaryCard/SummaryCard.interface.ts
View File

@ -18,4 +18,11 @@ export interface SummaryCardProps {
showProgressBar?: boolean;
className?: string;
isLoading?: boolean;
inverseLabel?: boolean;
titleIcon?: React.ReactElement;
cardBackgroundClass?:
| 'bg-success'
| 'bg-failed'
| 'bg-aborted'
| 'bg-primary';
}

27
openmetadata-ui/src/main/resources/ui/src/components/common/SummaryCard/SummaryCard.test.tsx
View File

@ -67,4 +67,31 @@ describe('SummaryCard component', () => {
expect(title.textContent).toStrictEqual('summary title');
expect(description.textContent).toStrictEqual('description');
});
it("label should be inverse, if 'inverseLabel' is true", async () => {
render(<SummaryCard {...mockProps} inverseLabel />);
const label = await screen.findByTestId('summary-card-label');
expect(label).toHaveClass('inverse-label');
});
it("should render title icon, if 'titleIcon' is provided", async () => {
render(
<SummaryCard
{...mockProps}
titleIcon={<span data-testid="title-icon">icon</span>}
/>
);
expect(await screen.findByTestId('title-icon')).toBeInTheDocument();
});
it("should render card background based on 'cardBackgroundClass'", async () => {
render(<SummaryCard {...mockProps} cardBackgroundClass="bg-success" />);
const container = await screen.findByTestId('summary-card-container');
expect(container).toHaveClass('bg-success');
});
});

18
openmetadata-ui/src/main/resources/ui/src/components/common/SummaryCard/summary-card.style.less
View File

@ -25,6 +25,11 @@
margin-bottom: 4px;
}
.inverse-label {
display: flex;
flex-direction: column-reverse;
}
.summary-card-description {
font-weight: 500;
font-size: 18px;
@ -37,6 +42,19 @@
}
}
&.bg-success {
background-color: @green-8;
}
&.bg-failed {
background-color: @red-8;
}
&.bg-aborted {
background-color: @yellow-8;
}
&.bg-primary {
background-color: @blue-8;
}
.new.ant-progress-circle {
.ant-progress-circle-path {
stroke: @blue-3 !important;

8
openmetadata-ui/src/main/resources/ui/src/pages/DataQuality/DataQualityPage.interface.ts
View File

@ -12,7 +12,8 @@
*/
import { EntityType } from '../../enums/entity.enum';
import { TestSummary } from '../../generated/tests/testCase';
import { TestCaseStatus, TestSummary } from '../../generated/tests/testCase';
import { TestCaseType } from '../../rest/testAPI';
export enum DataQualityPageTabs {
TEST_SUITES = 'test-suites',
@ -35,4 +36,9 @@ export type DataQualityDashboardChartFilters = {
endTs?: number;
entityFQN?: string;
entityType?: EntityType;
serviceName?: string;
testPlatforms?: string[];
dataQualityDimension?: string;
testCaseStatus?: TestCaseStatus;
testCaseType?: TestCaseType;
};

80
openmetadata-ui/src/main/resources/ui/src/pages/DataQuality/DataQualityProvider.test.tsx
View File

@ -13,7 +13,11 @@
/* eslint-disable i18next/no-literal-string */
import { act, render, screen } from '@testing-library/react';
import React from 'react';
import { fetchTestCaseSummary } from '../../rest/dataQualityDashboardAPI';
import {
fetchEntityCoveredWithDQ,
fetchTestCaseSummary,
fetchTotalEntityCount,
} from '../../rest/dataQualityDashboardAPI';
import { DataQualityPageTabs } from './DataQualityPage.interface';
import DataQualityProvider, {
useDataQualityProvider,
@ -30,6 +34,10 @@ const mockPermissionsData = {
const mockUseParam = { tab: DataQualityPageTabs.TABLES } as {
tab?: DataQualityPageTabs;
};
const mockLocation = {
search: '',
};
jest.mock('../../context/PermissionProvider/PermissionProvider', () => ({
usePermissionProvider: () => mockPermissionsData,
}));
@ -38,11 +46,43 @@ jest.mock('react-router-dom', () => {
useParams: jest.fn().mockImplementation(() => mockUseParam),
};
});
jest.mock('../../hooks/useCustomLocation/useCustomLocation', () => {
return jest.fn().mockImplementation(() => mockLocation);
});
jest.mock('../../rest/dataQualityDashboardAPI', () => ({
fetchTestCaseSummary: jest.fn().mockImplementation(() => {
return new Promise((resolve) => {
setTimeout(() => {
resolve({ data: [] });
resolve({
data: [
{
document_count: '4',
'testCaseResult.testCaseStatus': 'success',
},
{
document_count: '3',
'testCaseResult.testCaseStatus': 'failed',
},
{
document_count: '1',
'testCaseResult.testCaseStatus': 'aborted',
},
],
});
}, 2000); // Simulate a delay
});
}),
fetchEntityCoveredWithDQ: jest.fn().mockImplementation(() => {
return new Promise((resolve) => {
setTimeout(() => {
resolve({ data: [{ originEntityFQN: '1' }] });
}, 2000); // Simulate a delay
});
}),
fetchTotalEntityCount: jest.fn().mockImplementation(() => {
return new Promise((resolve) => {
setTimeout(() => {
resolve({ data: [{ fullyQualifiedName: '29' }] });
}, 2000); // Simulate a delay
});
}),
@ -87,7 +127,41 @@ describe('DataQualityProvider', () => {
expect(await screen.findByText('tables component')).toBeInTheDocument();
});
it('should call fetchTestCaseSummary', async () => {
it('should call fetchTestCaseSummary, fetchEntityCoveredWithDQ & fetchTotalEntityCount', async () => {
expect(await screen.findByText('tables component')).toBeInTheDocument();
expect(fetchTestCaseSummary).toHaveBeenCalledTimes(1);
expect(fetchEntityCoveredWithDQ).toHaveBeenCalledTimes(2);
expect(fetchTotalEntityCount).toHaveBeenCalledTimes(1);
});
it('should call fetchTestCaseSummary, fetchEntityCoveredWithDQ & fetchTotalEntityCount based on prams change', async () => {
mockLocation.search =
'?testCaseType=table&testCaseStatus=Success&tier=Tier.Tier1';
expect(await screen.findByText('tables component')).toBeInTheDocument();
expect(fetchTestCaseSummary).toHaveBeenCalledWith({
entityFQN: undefined,
ownerFqn: undefined,
testCaseStatus: 'Success',
testCaseType: 'table',
tier: ['Tier.Tier1'],
});
expect(fetchEntityCoveredWithDQ).toHaveBeenCalledWith(
{
entityFQN: undefined,
ownerFqn: undefined,
testCaseStatus: 'Success',
testCaseType: 'table',
tier: ['Tier.Tier1'],
},
true
);
expect(fetchTotalEntityCount).toHaveBeenCalledWith({
entityFQN: undefined,
ownerFqn: undefined,
testCaseStatus: 'Success',
testCaseType: 'table',
tier: ['Tier.Tier1'],
});
});
});

71
openmetadata-ui/src/main/resources/ui/src/pages/DataQuality/DataQualityProvider.tsx
View File

@ -11,6 +11,8 @@
* limitations under the License.
*/
import { AxiosError } from 'axios';
import { pick } from 'lodash';
import QueryString from 'qs';
import React, {
createContext,
useContext,
@ -19,10 +21,16 @@ import React, {
useState,
} from 'react';
import { useParams } from 'react-router-dom';
import { DataQualityPageParams } from '../../components/DataQuality/DataQuality.interface';
import { INITIAL_TEST_SUMMARY } from '../../constants/TestSuite.constant';
import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider';
import { TestSummary } from '../../generated/tests/testCase';
import { fetchTestCaseSummary } from '../../rest/dataQualityDashboardAPI';
import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation';
import {
fetchEntityCoveredWithDQ,
fetchTestCaseSummary,
fetchTotalEntityCount,
} from '../../rest/dataQualityDashboardAPI';
import { transformToTestCaseStatusObject } from '../../utils/DataQuality/DataQualityUtils';
import { showErrorToast } from '../../utils/ToastUtils';
import {
@ -36,6 +44,17 @@ export const DataQualityContext = createContext<DataQualityContextInterface>(
const DataQualityProvider = ({ children }: { children: React.ReactNode }) => {
const { tab: activeTab } = useParams<{ tab: DataQualityPageTabs }>();
const location = useCustomLocation();
const params = useMemo(() => {
const search = location.search;
const params = QueryString.parse(
search.startsWith('?') ? search.substring(1) : search
);
return params as DataQualityPageParams;
}, [location.search]);
const [testCaseSummary, setTestCaseSummary] =
useState<TestSummary>(INITIAL_TEST_SUMMARY);
const [isTestCaseSummaryLoading, setIsTestCaseSummaryLoading] =
@ -52,25 +71,65 @@ const DataQualityProvider = ({ children }: { children: React.ReactNode }) => {
};
}, [testCaseSummary, isTestCaseSummaryLoading, activeTab]);
const fetchTestSummary = async () => {
const fetchTestSummary = async (params?: DataQualityPageParams) => {
const filters = {
...pick(params, [
'tags',
'serviceName',
'testPlatforms',
'dataQualityDimension',
'testCaseStatus',
'testCaseType',
]),
ownerFqn: params?.owner ? JSON.parse(params.owner)?.name : undefined,
tier: params?.tier ? [params.tier] : undefined,
entityFQN: params?.tableFqn,
};
setIsTestCaseSummaryLoading(true);
try {
const { data } = await fetchTestCaseSummary();
const { data } = await fetchTestCaseSummary(filters);
const { data: unhealthyData } = await fetchEntityCoveredWithDQ(
filters,
true
);
const { data: totalDQCoverage } = await fetchEntityCoveredWithDQ(
filters,
false
);
const { data: entityCount } = await fetchTotalEntityCount(filters);
const unhealthy = parseInt(unhealthyData[0].originEntityFQN);
const total = parseInt(totalDQCoverage[0].originEntityFQN);
let totalEntityCount = parseInt(entityCount[0].fullyQualifiedName);
if (total > totalEntityCount) {
totalEntityCount = total;
}
const updatedData = transformToTestCaseStatusObject(data);
setTestCaseSummary(updatedData);
setTestCaseSummary({
...updatedData,
unhealthy,
healthy: total - unhealthy,
totalDQEntities: total,
totalEntityCount,
});
} catch (error) {
showErrorToast(error as AxiosError);
} finally {
setIsTestCaseSummaryLoading(false);
}
};
useEffect(() => {
if (testCasePermission?.ViewAll || testCasePermission?.ViewBasic) {
fetchTestSummary();
fetchTestSummary(params);
} else {
setIsTestCaseSummaryLoading(false);
}
}, []);
}, [params]);
return (
<DataQualityContext.Provider value={dataQualityContextValue}>

151
openmetadata-ui/src/main/resources/ui/src/rest/dataQualityDashboardAPI.test.ts
View File

@ -16,6 +16,7 @@ import { EntityType } from '../enums/entity.enum';
import { TestCaseStatus } from '../generated/tests/testCase';
import { TestCaseResolutionStatusTypes } from '../generated/tests/testCaseResolutionStatus';
import {
buildDataQualityDashboardFilters,
buildMustEsFilterForOwner,
buildMustEsFilterForTags,
} from '../utils/DataQuality/DataQualityUtils';
@ -37,21 +38,27 @@ jest.mock('./testAPI', () => ({
jest.mock('../utils/DataQuality/DataQualityUtils', () => ({
buildMustEsFilterForOwner: jest.fn(),
buildMustEsFilterForTags: jest.fn(),
buildDataQualityDashboardFilters: jest.fn().mockReturnValue([]),
}));
describe('dataQualityDashboardAPI', () => {
describe('fetchTotalEntityCount', () => {
it('should call getDataQualityReport with correct query when ownerFqn is provided', async () => {
const filters = { ownerFqn: 'owner1' };
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue({
term: {
'owners.fullyQualifiedName': 'owner1',
(buildDataQualityDashboardFilters as jest.Mock).mockReturnValueOnce([
{
term: {
'owners.fullyQualifiedName': 'owner1',
},
},
});
]);
await fetchTotalEntityCount(filters);
expect(buildMustEsFilterForOwner).toHaveBeenCalledWith('owner1');
expect(buildDataQualityDashboardFilters).toHaveBeenCalledWith({
filters: { ownerFqn: 'owner1' },
isTableApi: true,
});
expect(getDataQualityReport).toHaveBeenCalledWith({
q: JSON.stringify({
query: {
@ -73,6 +80,16 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when tags are provided', async () => {
const filters = { tags: ['tag1', 'tag2'] };
(buildDataQualityDashboardFilters as jest.Mock).mockReturnValueOnce([
{
bool: {
should: [
{ term: { 'tags.tagFQN': 'tag1' } },
{ term: { 'tags.tagFQN': 'tag2' } },
],
},
},
]);
await fetchTotalEntityCount(filters);
@ -100,6 +117,16 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when tier is provided', async () => {
const filters = { tier: ['tier1', 'tier2'] };
(buildDataQualityDashboardFilters as jest.Mock).mockReturnValueOnce([
{
bool: {
should: [
{ term: { 'tier.tagFQN': 'tier1' } },
{ term: { 'tier.tagFQN': 'tier2' } },
],
},
},
]);
await fetchTotalEntityCount(filters);
@ -127,15 +154,30 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when all filters are provided', async () => {
const filters = { ownerFqn: 'owner1', tags: ['tag1'], tier: ['tier1'] };
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue({
term: {
'owners.fullyQualifiedName': 'owner1',
(buildDataQualityDashboardFilters as jest.Mock).mockReturnValueOnce([
{
term: {
'owners.fullyQualifiedName': 'owner1',
},
},
});
{
bool: {
should: [{ term: { 'tags.tagFQN': 'tag1' } }],
},
},
{
bool: {
should: [{ term: { 'tier.tagFQN': 'tier1' } }],
},
},
]);
await fetchTotalEntityCount(filters);
expect(buildMustEsFilterForOwner).toHaveBeenCalledWith('owner1');
expect(buildDataQualityDashboardFilters).toHaveBeenCalledWith({
filters: { ownerFqn: 'owner1', tags: ['tag1'], tier: ['tier1'] },
isTableApi: true,
});
expect(getDataQualityReport).toHaveBeenCalledWith({
q: JSON.stringify({
query: {
@ -350,6 +392,9 @@ describe('dataQualityDashboardAPI', () => {
func: fetchEntityCoveredWithDQ,
index: 'testCase',
aggregationQuery: `bucketName=entityWithTests:aggType=cardinality:field=originEntityFQN`,
params: {
unhealthy: false,
},
},
{
functionName: 'fetchTestCaseSummaryByDimension',
@ -364,15 +409,16 @@ describe('dataQualityDashboardAPI', () => {
describe(`${testData.functionName}`, () => {
it('should call getDataQualityReport with correct query when ownerFqn is provided', async () => {
const filters = { ownerFqn: testCaseData.filters.ownerFqn };
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue(
testCaseData.ownerExpectedQuery
);
(buildDataQualityDashboardFilters as jest.Mock).mockReturnValueOnce([
testCaseData.ownerExpectedQuery,
]);
await testData.func(filters);
expect(buildMustEsFilterForOwner).toHaveBeenCalledWith(
testCaseData.filters.ownerFqn
);
expect(buildDataQualityDashboardFilters).toHaveBeenCalledWith({
filters,
...testData.params,
});
expect(getDataQualityReport).toHaveBeenCalledWith({
q: testCaseData.test1.q,
index: testData.index,
@ -382,15 +428,16 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when tags are provided', async () => {
const filters = { tags: testCaseData.filters.tags };
(buildMustEsFilterForTags as jest.Mock).mockReturnValue(
testCaseData.test2.expected
);
(buildDataQualityDashboardFilters as jest.Mock).mockReturnValueOnce([
testCaseData.test2.expected,
]);
await testData.func(filters);
expect(buildMustEsFilterForTags).toHaveBeenCalledWith(
testCaseData.filters.tags
);
expect(buildDataQualityDashboardFilters).toHaveBeenCalledWith({
filters,
...testData.params,
});
expect(getDataQualityReport).toHaveBeenCalledWith({
q: testCaseData.test2.q,
index: testData.index,
@ -400,15 +447,16 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when tier is provided', async () => {
const filters = { tier: testCaseData.filters.tier };
(buildMustEsFilterForTags as jest.Mock).mockReturnValue(
testCaseData.test3.expected
);
(buildDataQualityDashboardFilters as jest.Mock).mockReturnValueOnce([
testCaseData.test3.expected,
]);
await testData.func(filters);
expect(buildMustEsFilterForTags).toHaveBeenCalledWith(
testCaseData.filters.tier
);
expect(buildDataQualityDashboardFilters).toHaveBeenCalledWith({
filters,
...testData.params,
});
expect(getDataQualityReport).toHaveBeenCalledWith({
q: testCaseData.test3.q,
index: testData.index,
@ -418,22 +466,19 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when all filters are provided', async () => {
const filters = testCaseData.filters;
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue(
testCaseData.ownerExpectedQuery
);
(buildMustEsFilterForTags as jest.Mock).mockReturnValue(
testCaseData.test4.expected
);
(buildDataQualityDashboardFilters as jest.Mock).mockReturnValueOnce([
testCaseData.ownerExpectedQuery,
testCaseData.test4.expected,
]);
await testData.func(filters);
expect(buildMustEsFilterForOwner).toHaveBeenCalledWith(
testCaseData.filters.ownerFqn
);
expect(buildMustEsFilterForTags).toHaveBeenCalledWith([
...testCaseData.filters.tags,
...testCaseData.filters.tier,
]);
expect(buildDataQualityDashboardFilters).toHaveBeenCalledWith({
filters,
...testData.params,
});
expect(getDataQualityReport).toHaveBeenCalledWith({
q: testCaseData.test4.q,
index: testData.index,
@ -486,7 +531,7 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when ownerFqn is provided', async () => {
const status = TestCaseResolutionStatusTypes.Assigned;
const filters = { ownerFqn: 'owner1' };
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue({
(buildMustEsFilterForOwner as jest.Mock).mockReturnValueOnce({
term: {
'owners.fullyQualifiedName': 'owner1',
},
@ -527,7 +572,7 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when tags and tier are provided', async () => {
const status = TestCaseResolutionStatusTypes.New;
const filters = { tags: ['tag1'], tier: ['tier1'] };
(buildMustEsFilterForTags as jest.Mock).mockReturnValue({
(buildMustEsFilterForTags as jest.Mock).mockReturnValueOnce({
nested: {
path: 'tags',
query: {
@ -587,12 +632,12 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when all filters are provided', async () => {
const status = TestCaseResolutionStatusTypes.Resolved;
const filters = { ownerFqn: 'owner1', tags: ['tag1'], tier: ['tier1'] };
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue({
(buildMustEsFilterForOwner as jest.Mock).mockReturnValueOnce({
term: {
'owners.fullyQualifiedName': 'owner1',
},
});
(buildMustEsFilterForTags as jest.Mock).mockReturnValue({
(buildMustEsFilterForTags as jest.Mock).mockReturnValueOnce({
nested: {
path: 'tags',
query: {
@ -732,7 +777,7 @@ describe('dataQualityDashboardAPI', () => {
startTs: 1729073964962,
endTs: 1729678764965,
};
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue(
(buildMustEsFilterForOwner as jest.Mock).mockReturnValueOnce(
testCaseData.ownerExpectedQuery
);
@ -788,7 +833,7 @@ describe('dataQualityDashboardAPI', () => {
startTs: 1729073964962,
endTs: 1729678764965,
};
(buildMustEsFilterForTags as jest.Mock).mockReturnValue({
(buildMustEsFilterForTags as jest.Mock).mockReturnValueOnce({
nested: {
path: 'tags',
query: {
@ -863,10 +908,10 @@ describe('dataQualityDashboardAPI', () => {
startTs: 1729073964962,
endTs: 1729678764965,
};
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue(
(buildMustEsFilterForOwner as jest.Mock).mockReturnValueOnce(
testCaseData.ownerExpectedQuery
);
(buildMustEsFilterForTags as jest.Mock).mockReturnValue({
(buildMustEsFilterForTags as jest.Mock).mockReturnValueOnce({
nested: {
path: 'tags',
query: {
@ -1011,7 +1056,7 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when ownerFqn is provided', async () => {
const status = TestCaseStatus.Failed;
const filters = { ownerFqn: testCaseData.filters.ownerFqn };
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue(
(buildMustEsFilterForOwner as jest.Mock).mockReturnValueOnce(
testCaseData.ownerExpectedQuery
);
@ -1053,7 +1098,7 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when tags and tier are provided', async () => {
const status = TestCaseStatus.Aborted;
const filters = { tags: ['tag1'], tier: ['tier1'] };
(buildMustEsFilterForTags as jest.Mock).mockReturnValue({
(buildMustEsFilterForTags as jest.Mock).mockReturnValueOnce({
nested: {
path: 'tags',
query: {
@ -1113,12 +1158,12 @@ describe('dataQualityDashboardAPI', () => {
it('should call getDataQualityReport with correct query when all filters are provided', async () => {
const status = TestCaseStatus.Failed;
const filters = { ownerFqn: 'owner1', tags: ['tag1'], tier: ['tier1'] };
(buildMustEsFilterForOwner as jest.Mock).mockReturnValue({
(buildMustEsFilterForOwner as jest.Mock).mockReturnValueOnce({
term: {
'owners.fullyQualifiedName': 'owner1',
},
});
(buildMustEsFilterForTags as jest.Mock).mockReturnValue({
(buildMustEsFilterForTags as jest.Mock).mockReturnValueOnce({
nested: {
path: 'tags',
query: {

90
openmetadata-ui/src/main/resources/ui/src/rest/dataQualityDashboardAPI.ts
View File

@ -16,6 +16,7 @@ import { TestCaseStatus } from '../generated/tests/testCase';
import { TestCaseResolutionStatusTypes } from '../generated/tests/testCaseResolutionStatus';
import { DataQualityDashboardChartFilters } from '../pages/DataQuality/DataQualityPage.interface';
import {
buildDataQualityDashboardFilters,
buildMustEsFilterForOwner,
buildMustEsFilterForTags,
} from '../utils/DataQuality/DataQualityUtils';
@ -25,27 +26,7 @@ export const fetchEntityCoveredWithDQ = (
filters?: DataQualityDashboardChartFilters,
unhealthy = false
) => {
const mustFilter = [];
if (unhealthy) {
mustFilter.push({
terms: {
'testCaseStatus.keyword': ['Failed', 'Aborted'],
},
});
}
if (filters?.ownerFqn) {
mustFilter.push(buildMustEsFilterForOwner(filters.ownerFqn));
}
if (filters?.tags || filters?.tier) {
mustFilter.push(
buildMustEsFilterForTags([
...(filters?.tags ?? []),
...(filters?.tier ?? []),
])
);
}
const mustFilter = buildDataQualityDashboardFilters({ filters, unhealthy });
return getDataQualityReport({
q: JSON.stringify({
@ -63,35 +44,10 @@ export const fetchEntityCoveredWithDQ = (
export const fetchTotalEntityCount = (
filters?: DataQualityDashboardChartFilters
) => {
const mustFilter = [];
if (filters?.ownerFqn) {
mustFilter.push(buildMustEsFilterForOwner(filters.ownerFqn));
}
if (filters?.tags) {
mustFilter.push({
bool: {
should: filters.tags.map((tag) => ({
term: {
'tags.tagFQN': tag,
},
})),
},
});
}
if (filters?.tier) {
mustFilter.push({
bool: {
should: filters.tier.map((tag) => ({
term: {
'tier.tagFQN': tag,
},
})),
},
});
}
const mustFilter = buildDataQualityDashboardFilters({
filters,
isTableApi: true,
});
return getDataQualityReport({
q: JSON.stringify({
@ -109,26 +65,7 @@ export const fetchTotalEntityCount = (
export const fetchTestCaseSummary = (
filters?: DataQualityDashboardChartFilters
) => {
const mustFilter = [];
if (filters?.ownerFqn) {
mustFilter.push(buildMustEsFilterForOwner(filters.ownerFqn));
}
if (filters?.tags || filters?.tier) {
mustFilter.push(
buildMustEsFilterForTags([
...(filters?.tags ?? []),
...(filters?.tier ?? []),
])
);
}
if (filters?.entityFQN) {
mustFilter.push({
term: {
entityFQN: filters.entityFQN,
},
});
}
const mustFilter = buildDataQualityDashboardFilters({ filters });
return getDataQualityReport({
q: JSON.stringify({
@ -147,18 +84,7 @@ export const fetchTestCaseSummary = (
export const fetchTestCaseSummaryByDimension = (
filters?: DataQualityDashboardChartFilters
) => {
const mustFilter = [];
if (filters?.ownerFqn) {
mustFilter.push(buildMustEsFilterForOwner(filters.ownerFqn));
}
if (filters?.tags || filters?.tier) {
mustFilter.push(
buildMustEsFilterForTags([
...(filters?.tags ?? []),
...(filters?.tier ?? []),
])
);
}
const mustFilter = buildDataQualityDashboardFilters({ filters });
return getDataQualityReport({
q: JSON.stringify({

112
openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts
View File

@ -32,7 +32,8 @@ import {
TestDataType,
TestDefinition,
} from '../../generated/tests/testDefinition';
import { ListTestCaseParamsBySearch } from '../../rest/testAPI';
import { DataQualityDashboardChartFilters } from '../../pages/DataQuality/DataQualityPage.interface';
import { ListTestCaseParamsBySearch, TestCaseType } from '../../rest/testAPI';
import { generateEntityLink } from '../TableUtils';
/**
@ -218,6 +219,115 @@ export const buildMustEsFilterForOwner = (
};
};
export const buildDataQualityDashboardFilters = (data: {
filters?: DataQualityDashboardChartFilters;
unhealthy?: boolean;
isTableApi?: boolean;
}) => {
const { filters, unhealthy = false, isTableApi = false } = data;
const mustFilter = [];
if (unhealthy) {
mustFilter.push({
terms: {
'testCaseStatus.keyword': ['Failed', 'Aborted'],
},
});
}
if (filters?.ownerFqn) {
mustFilter.push(buildMustEsFilterForOwner(filters.ownerFqn));
}
if (filters?.tags && isTableApi) {
mustFilter.push({
bool: {
should: filters.tags.map((tag) => ({
term: {
'tags.tagFQN': tag,
},
})),
},
});
}
if (filters?.tier && isTableApi) {
mustFilter.push({
bool: {
should: filters.tier.map((tag) => ({
term: {
'tier.tagFQN': tag,
},
})),
},
});
}
if ((filters?.tags || filters?.tier) && !isTableApi) {
mustFilter.push(
buildMustEsFilterForTags([
...(filters?.tags ?? []),
...(filters?.tier ?? []),
])
);
}
if (filters?.entityFQN) {
mustFilter.push({
term: {
[isTableApi ? 'fullyQualifiedName.keyword' : 'entityFQN']:
filters.entityFQN,
},
});
}
if (filters?.serviceName) {
mustFilter.push({
term: {
'service.name.keyword': filters.serviceName,
},
});
}
if (filters?.testPlatforms) {
mustFilter.push({
terms: {
testPlatforms: filters.testPlatforms,
},
});
}
if (filters?.dataQualityDimension) {
mustFilter.push({
term: {
dataQualityDimension: filters.dataQualityDimension,
},
});
}
if (filters?.testCaseStatus) {
mustFilter.push({
term: {
'testCaseResult.testCaseStatus': filters.testCaseStatus,
},
});
}
if (filters?.testCaseType) {
if (filters.testCaseType === TestCaseType.table) {
mustFilter.push({
bool: { must_not: [{ regexp: { entityLink: '.*::columns::.*' } }] },
});
}
if (filters.testCaseType === TestCaseType.column) {
mustFilter.push({ regexp: { entityLink: '.*::columns::.*' } });
}
}
return mustFilter;
};
export const getDimensionIcon = (dimension: DataQualityDimensions) => {
switch (dimension) {
case DataQualityDimensions.Accuracy: