From 3998982d54a742bba20e822bf22ee4cf36866e0a Mon Sep 17 00:00:00 2001 From: Teddy Date: Sun, 18 May 2025 21:35:13 +0200 Subject: [PATCH] ISSUE #21216: added include field to incident list (#21221) * fix: added include field to incident list * added ui changes for allowing deleted entries in entity page and fix the count on incident page * fix: error retrieving soft deleted test cases * fix: ran java linting * Update openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.component.tsx Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> * fix testcase playwright * fix playwright test --------- Co-authored-by: Ashish Gupta Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> (cherry picked from commit 0b483ecb865341425abf47132185bdbde6051bc7) --- .../TestCaseResolutionStatusRepository.java | 5 ++- .../TestCaseResolutionStatusResource.java | 9 +++- .../service/security/mask/PIIMasker.java | 2 +- .../dqtests/TestCaseResourceTest.java | 39 ++++++++++++++++- .../e2e/Features/IncidentManager.spec.ts | 12 +++--- .../TableProfilerProvider.test.tsx | 32 ++++++++++++-- .../TableProfiler/TableProfilerProvider.tsx | 2 + .../IncidentManager.component.tsx | 13 ++++-- .../IncidentManager/IncidentManager.test.tsx | 24 +++++++++++ .../DataQuality/DataQualityUtils.test.ts | 42 +++++++++++++++++++ .../src/utils/DataQuality/DataQualityUtils.ts | 7 ++++ 11 files changed, 170 insertions(+), 17 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java index 2ac68b9dfda..3a67676af8e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java @@ -374,10 +374,11 @@ public class TestCaseResolutionStatusRepository public static String addOriginEntityFQNJoin(ListFilter filter, String condition) { // if originEntityFQN is present, we need to join with test_case table - if (filter.getQueryParam("originEntityFQN") != null) { + if ((filter.getQueryParam("originEntityFQN") != null) + || (filter.getQueryParam("include") != null)) { condition = """ - INNER JOIN (SELECT entityFQN AS testCaseEntityFQN,fqnHash AS testCaseHash FROM test_case) tc \ + INNER JOIN (SELECT entityFQN AS testCaseEntityFQN,fqnHash AS testCaseHash, deleted FROM test_case) tc \ ON entityFQNHash = testCaseHash """ + condition; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java index f916c328535..1bd9570327e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java @@ -37,6 +37,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus; import org.openmetadata.schema.tests.type.TestCaseResolutionStatus; import org.openmetadata.schema.tests.type.TestCaseResolutionStatusTypes; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.ListFilter; @@ -137,6 +138,12 @@ public class TestCaseResolutionStatusResource schema = @Schema(type = "String")) @QueryParam("assignee") String assignee, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include, @Parameter(description = "Test case fully qualified name", schema = @Schema(type = "String")) @QueryParam("testCaseFQN") String testCaseFQN, @@ -159,7 +166,7 @@ public class TestCaseResolutionStatusResource } authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); - ListFilter filter = new ListFilter(null); + ListFilter filter = new ListFilter(include); filter.addQueryParam("testCaseResolutionStatusType", testCaseResolutionStatusType); filter.addQueryParam("assignee", assignee); filter.addQueryParam("entityFQNHash", FullyQualifiedName.buildHash(testCaseFQN)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java index 8749c5e8f87..8c16880574e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java @@ -185,7 +185,7 @@ public class PIIMasker { Entity.TABLE, testCaseLink.getEntityFQN(), "owners,tags,columns", - Include.NON_DELETED); + Include.ALL); entityFQNToTable.put(testCaseLink.getEntityFQN(), table); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index b2567f2463b..b813d84a8a0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -1525,8 +1525,11 @@ public class TestCaseResourceTest extends EntityResourceTest testCases = new ArrayList<>(); + Long startTs = System.currentTimeMillis() - 1000; + for (int i = 0; i < 2; i++) { + // We'll create random test cases + TestCase testCaseEntity = + createEntity(createRequest(getEntityName(test) + i), ADMIN_AUTH_HEADERS); + testCases.add(testCaseEntity); + // Adding failed test case, which will create a NEW incident + postTestCaseResult( + testCaseEntity.getFullyQualifiedName(), + new CreateTestCaseResult() + .withResult("result") + .withTestCaseStatus(TestCaseStatus.Failed) + .withTimestamp(TestUtils.dateToTimestamp("2024-01-01")), + ADMIN_AUTH_HEADERS); + } + Long endTs = System.currentTimeMillis() + 1000; + ResultList entities = + getTestCaseFailureStatus(1000, null, false, startTs, endTs, null); + assertTrue( + entities.getData().stream() + .anyMatch( + tcrs -> tcrs.getTestCaseReference().getId().equals(testCases.get(0).getId()))); + deleteEntityByName(testCases.get(0).getFullyQualifiedName(), true, false, ADMIN_AUTH_HEADERS); + entities = getTestCaseFailureStatus(1000, null, false, startTs, endTs, null); + assertTrue( + entities.getData().stream() + .noneMatch( + tcrs -> tcrs.getTestCaseReference().getId().equals(testCases.get(0).getId()))); + } + @Test void patch_TestCaseResultFailure(TestInfo test) throws HttpResponseException { TestCase testCaseEntity = createEntity(createRequest(getEntityName(test)), ADMIN_AUTH_HEADERS); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/IncidentManager.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/IncidentManager.spec.ts index f67fd574d7c..076c4c4750b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/IncidentManager.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/IncidentManager.spec.ts @@ -254,7 +254,7 @@ test.describe('Incident Manager', PLAYWRIGHT_INGESTION_TAG_OBJ, () => { ); const incidentDetailsRes = page.waitForResponse( - '/api/v1/dataQuality/testCases/testCaseIncidentStatus?latest=true&startTs=*&endTs=*&limit=*' + '/api/v1/dataQuality/testCases/testCaseIncidentStatus?*' ); await sidebarClick(page, SidebarItem.INCIDENT_MANAGER); await incidentDetailsRes; @@ -456,7 +456,7 @@ test.describe('Incident Manager', PLAYWRIGHT_INGESTION_TAG_OBJ, () => { }; const testCase1 = table1.testCasesResponseData[0]?.['name']; const incidentDetailsRes = page.waitForResponse( - '/api/v1/dataQuality/testCases/testCaseIncidentStatus?latest=true&startTs=*&endTs=*&limit=*' + '/api/v1/dataQuality/testCases/testCaseIncidentStatus?*' ); await sidebarClick(page, SidebarItem.INCIDENT_MANAGER); await incidentDetailsRes; @@ -485,7 +485,7 @@ test.describe('Incident Manager', PLAYWRIGHT_INGESTION_TAG_OBJ, () => { ).not.toBeVisible(); const nonAssigneeFilterRes = page.waitForResponse( - '/api/v1/dataQuality/testCases/testCaseIncidentStatus?latest=true&startTs=*&endTs=*&limit=*' + '/api/v1/dataQuality/testCases/testCaseIncidentStatus?*' ); await page .getByTestId('select-assignee') @@ -508,7 +508,7 @@ test.describe('Incident Manager', PLAYWRIGHT_INGESTION_TAG_OBJ, () => { ).not.toBeVisible(); const nonStatusFilterRes = page.waitForResponse( - '/api/v1/dataQuality/testCases/testCaseIncidentStatus?latest=true&startTs=*&endTs=*&limit=*' + '/api/v1/dataQuality/testCases/testCaseIncidentStatus?*' ); await page.getByTestId('status-select').getByLabel('close-circle').click(); await nonStatusFilterRes; @@ -534,7 +534,7 @@ test.describe('Incident Manager', PLAYWRIGHT_INGESTION_TAG_OBJ, () => { ).toBeVisible(); const nonTestCaseFilterRes = page.waitForResponse( - '/api/v1/dataQuality/testCases/testCaseIncidentStatus?latest=true&startTs=*&endTs=*&limit=*' + '/api/v1/dataQuality/testCases/testCaseIncidentStatus?*' ); await page .getByTestId('test-case-select') @@ -544,7 +544,7 @@ test.describe('Incident Manager', PLAYWRIGHT_INGESTION_TAG_OBJ, () => { await page.click('[data-testid="date-picker-menu"]'); const timeSeriesFilterRes = page.waitForResponse( - '/api/v1/dataQuality/testCases/testCaseIncidentStatus?latest=true&startTs=*&endTs=*&limit=*' + '/api/v1/dataQuality/testCases/testCaseIncidentStatus?*' ); await page.getByRole('menuitem', { name: 'Yesterday' }).click(); await timeSeriesFilterRes; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.test.tsx index 1961dfa5f24..3f995eb3de1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.test.tsx @@ -73,19 +73,23 @@ const mockPermissions = { } as OperationPermission; describe('TableProfilerProvider', () => { - beforeEach(() => { + it('renders children without crashing', async () => { render(
Test Children
 ); - }); - it('renders children without crashing', async () => { expect(await screen.findByText('Test Children')).toBeInTheDocument(); }); it('test cases should be fetch on data quality tab', async () => { + render( + +
Test Children
+ + ); + const mockGetListTestCase = getListTestCaseBySearch as jest.Mock; expect(mockGetListTestCase).toHaveBeenCalledTimes(1); @@ -94,6 +98,28 @@ describe('TableProfilerProvider', () => { fields: ['testCaseResult', 'incidentId'], includeAllTests: true, limit: 10, + include: 'non-deleted', + }); + }); + + it('test cases should be fetch on data quality tab with deleted', async () => { + render( + +
Test Children
+ + ); + + const mockGetListTestCase = getListTestCaseBySearch as jest.Mock; + + expect(mockGetListTestCase).toHaveBeenCalledTimes(1); + expect(mockGetListTestCase).toHaveBeenCalledWith({ + entityLink: 'entityLink', + fields: ['testCaseResult', 'incidentId'], + includeAllTests: true, + limit: 10, + include: 'deleted', }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.tsx index 9d60330450d..301c664f6de 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/TableProfilerProvider.tsx @@ -35,6 +35,7 @@ import { TabSpecificField } from '../../../../enums/entity.enum'; import { Table } from '../../../../generated/entity/data/table'; import { ProfileSampleType } from '../../../../generated/metadataIngestion/databaseServiceProfilerPipeline'; import { TestCase } from '../../../../generated/tests/testCase'; +import { Include } from '../../../../generated/type/include'; import { usePaging } from '../../../../hooks/paging/usePaging'; import useCustomLocation from '../../../../hooks/useCustomLocation/useCustomLocation'; import { useFqn } from '../../../../hooks/useFqn'; @@ -222,6 +223,7 @@ export const TableProfilerProvider = ({ entityLink: generateEntityLink(datasetFQN ?? ''), includeAllTests: true, limit: testCasePaging.pageSize, + include: isTableDeleted ? Include.Deleted : Include.NonDeleted, }); setAllTestCases(data); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.component.tsx index 450f61cc695..c5961a69f28 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.component.tsx @@ -35,6 +35,7 @@ import { TestCaseResolutionStatus, TestCaseResolutionStatusTypes, } from '../../generated/tests/testCaseResolutionStatus'; +import { Include } from '../../generated/type/include'; import { usePaging } from '../../hooks/paging/usePaging'; import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation'; import { @@ -154,6 +155,7 @@ const IncidentManager = ({ const { data, paging } = await getListTestCaseIncidentStatus({ limit: pageSize, latest: true, + include: tableDetails?.deleted ? Include.Deleted : Include.NonDeleted, originEntityFQN: tableDetails?.fullyQualifiedName, ...params, }); @@ -463,7 +465,7 @@ const IncidentManager = ({ return ( ); @@ -487,7 +489,7 @@ const IncidentManager = ({ return ( handleSeveritySubmit(severity, record)} /> @@ -507,7 +509,12 @@ const IncidentManager = ({ ), }, ], - [testCaseListData.data, testCasePermissions, isPermissionLoading] + [ + tableDetails?.deleted, + testCaseListData.data, + testCasePermissions, + isPermissionLoading, + ] ); if ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.test.tsx index da39daee7b0..b1177a3b863 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/IncidentManager/IncidentManager.test.tsx @@ -13,6 +13,7 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import QueryString from 'qs'; import React from 'react'; +import { Table } from '../../generated/entity/data/table'; import { MOCK_PERMISSIONS } from '../../mocks/Glossary.mock'; import { getListTestCaseIncidentStatus } from '../../rest/incidentManagerAPI'; import IncidentManager from './IncidentManager.component'; @@ -153,6 +154,29 @@ describe('IncidentManagerPage', () => { latest: true, limit: 10, startTs: 1709556624254, + include: 'non-deleted', + }); + }); + + it('Incident should be fetch with deleted', async () => { + const mockGetListTestCaseIncidentStatus = + getListTestCaseIncidentStatus as jest.Mock; + await act(async () => { + render(); + }); + + const timeFilterButton = await screen.findByTestId('time-filter'); + + await act(async () => { + fireEvent.click(timeFilterButton); + }); + + expect(mockGetListTestCaseIncidentStatus).toHaveBeenCalledWith({ + endTs: 1710161424255, + latest: true, + limit: 10, + startTs: 1709556624254, + include: 'deleted', }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.test.ts index 65cc3599578..6f371f562aa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.test.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.test.ts @@ -20,6 +20,7 @@ import { } from '../../generated/tests/testDefinition'; import { ListTestCaseParamsBySearch } from '../../rest/testAPI'; import { + buildDataQualityDashboardFilters, buildMustEsFilterForOwner, buildMustEsFilterForTags, buildTestCaseParams, @@ -738,4 +739,45 @@ describe('DataQualityUtils', () => { ); }); }); + + describe('buildDataQualityDashboardFilters', () => { + it('should include deleted:false filter by default', () => { + const result = buildDataQualityDashboardFilters({}); + + expect(result).toEqual([ + { + term: { + deleted: false, + }, + }, + ]); + }); + + it('should include deleted:false filter along with other filters', () => { + const result = buildDataQualityDashboardFilters({ + filters: { + serviceName: 'test-service', + testPlatforms: ['DBT'], + }, + }); + + expect(result).toEqual([ + { + term: { + 'service.name.keyword': 'test-service', + }, + }, + { + terms: { + testPlatforms: ['DBT'], + }, + }, + { + term: { + deleted: false, + }, + }, + ]); + }); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts index 2adcd9ffd78..4a6be174e39 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts @@ -325,6 +325,13 @@ export const buildDataQualityDashboardFilters = (data: { } } + // Add the deleted filter to the mustFilter array + mustFilter.push({ + term: { + deleted: false, + }, + }); + return mustFilter; };