diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts index bc06577e8cd..8812c88a7f3 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts @@ -299,6 +299,40 @@ export const prepareDataQualityTestCases = (token: string) => { ); }); + cy.request({ + method: 'PATCH', + url: `/api/v1/tables/name/${filterTable.databaseSchema}.${filterTable.name}`, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json-patch+json', + }, + body: [ + { + op: 'add', + path: '/tags/0', + value: { + tagFQN: 'PII.None', + name: 'None', + description: 'Non PII', + source: 'Classification', + labelType: 'Manual', + state: 'Confirmed', + }, + }, + { + op: 'add', + path: '/tags/1', + value: { + tagFQN: 'Tier.Tier2', + name: 'Tier2', + source: 'Classification', + labelType: 'Manual', + state: 'Confirmed', + }, + }, + ], + }); + prepareDataQualityTestCasesViaREST({ testSuite: filterTableTestSuite, token, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts index 3f4a89f3ecd..58e781bafa2 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts @@ -106,6 +106,16 @@ const verifyFilterTestCase = () => { cy.get(`[data-testid="${testCase}"]`).scrollIntoView().should('be.visible'); }); }; +const verifyFilter2TestCase = (negation = false) => { + filterTable2TestCases.map((testCase) => { + negation + ? cy.get(`[data-testid="${testCase}"]`).should('not.exist') + : cy + .get(`[data-testid="${testCase}"]`) + .scrollIntoView() + .should('be.visible'); + }); +}; describe( 'Data Quality and Profiler should work properly', @@ -967,6 +977,12 @@ describe( 'getTestCase' ); + interceptURL( + 'GET', + `/api/v1/search/query?q=*index=tag_search_index*`, + 'searchTags' + ); + cy.sidebarClick(SidebarItem.DATA_QUALITY); cy.get('[data-testid="by-test-cases"]').click(); @@ -988,6 +1004,19 @@ describe( waitForAnimations: true, }); cy.get('[value="lastRunRange"]').click({ waitForAnimations: true }); + cy.get('[data-testid="advanced-filter"]').click({ + waitForAnimations: true, + }); + cy.get('[value="serviceName"]').click({ waitForAnimations: true }); + + cy.get('[data-testid="advanced-filter"]').click({ + waitForAnimations: true, + }); + cy.get('[value="tags"]').click({ waitForAnimations: true }); + cy.get('[data-testid="advanced-filter"]').click({ + waitForAnimations: true, + }); + cy.get('[value="tier"]').click({ waitForAnimations: true }); // Test case search filter cy.get( @@ -1000,6 +1029,78 @@ describe( cy.get('.ant-input-clear-icon').click(); verifyResponseStatusCode('@getTestCase', 200); + // Test case filter by service name + interceptURL( + 'GET', + `/api/v1/dataQuality/testCases/search/list?*serviceName=${DATABASE_SERVICE.service.name}*`, + 'getTestCaseByServiceName' + ); + interceptURL( + 'GET', + `/api/v1/search/query?q=*index=database_service_search_index*`, + 'searchService' + ); + cy.get('#serviceName') + .scrollIntoView() + .type(DATABASE_SERVICE.service.name); + verifyResponseStatusCode('@searchService', 200); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .find(`[data-testid="${DATABASE_SERVICE.service.name}"]`) + .click({ force: true }); + verifyResponseStatusCode('@getTestCaseByServiceName', 200); + verifyFilterTestCase(); + verifyFilter2TestCase(); + // remove service filter + cy.get('[data-testid="advanced-filter"]').click({ + waitForAnimations: true, + }); + cy.get('[value="serviceName"]').click({ waitForAnimations: true }); + verifyResponseStatusCode('@getTestCase', 200); + + // Test case filter by Tags + interceptURL( + 'GET', + `/api/v1/dataQuality/testCases/search/list?*tags=${'PII.None'}*`, + 'getTestCaseByTags' + ); + cy.get('#tags').scrollIntoView().click().type('PII.None'); + verifyResponseStatusCode('@searchTags', 200); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .find(`[data-testid="${'PII.None'}"]`) + .click({ force: true }); + verifyResponseStatusCode('@getTestCaseByTags', 200); + verifyFilterTestCase(); + verifyFilter2TestCase(true); + // remove service filter + cy.get('[data-testid="advanced-filter"]').click({ + waitForAnimations: true, + }); + cy.get('[value="tags"]').click({ waitForAnimations: true }); + verifyResponseStatusCode('@getTestCase', 200); + + // Test case filter by Tier + interceptURL( + 'GET', + `/api/v1/dataQuality/testCases/search/list?*tier=${'Tier.Tier2'}*`, + 'getTestCaseByTier' + ); + cy.get('#tier').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .find(`[data-testid="${'Tier.Tier2'}"]`) + .click({ force: true }); + verifyResponseStatusCode('@getTestCaseByTier', 200); + verifyFilterTestCase(); + verifyFilter2TestCase(true); + // remove service filter + cy.get('[data-testid="advanced-filter"]').click({ + waitForAnimations: true, + }); + cy.get('[value="tier"]').click({ waitForAnimations: true }); + verifyResponseStatusCode('@getTestCase', 200); + // Test case filter by table name interceptURL( 'GET', @@ -1021,10 +1122,7 @@ describe( .click({ force: true }); verifyResponseStatusCode('@searchTestCaseByTable', 200); verifyFilterTestCase(); - - filterTable2TestCases.map((testCase) => { - cy.get(`[data-testid="${testCase}"]`).should('not.exist'); - }); + verifyFilter2TestCase(true); // Test case filter by test type interceptURL( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx index 3095cd066d0..f54d680bcad 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx @@ -51,6 +51,7 @@ import { INITIAL_PAGING_VALUE, PAGE_SIZE, PAGE_SIZE_BASE, + TIER_CATEGORY, } from '../../../constants/constants'; import { TEST_CASE_FILTERS, @@ -65,6 +66,7 @@ import { TestCase } from '../../../generated/tests/testCase'; import { usePaging } from '../../../hooks/paging/usePaging'; import { DataQualityPageTabs } from '../../../pages/DataQuality/DataQualityPage.interface'; import { searchQuery } from '../../../rest/searchAPI'; +import { getTags } from '../../../rest/tagAPI'; import { getListTestCaseBySearch, ListTestCaseParamsBySearch, @@ -73,6 +75,7 @@ import { buildTestCaseParams } from '../../../utils/DataQuality/DataQualityUtils import { getEntityName } from '../../../utils/EntityUtils'; import { getDataQualityPagePath } from '../../../utils/RouterUtils'; import { generateEntityLink } from '../../../utils/TableUtils'; +import tagClassBase from '../../../utils/TagClassBase'; import { showErrorToast } from '../../../utils/ToastUtils'; import DatePickerMenu from '../../common/DatePickerMenu/DatePickerMenu.component'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; @@ -90,7 +93,10 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { const { permissions } = usePermissionProvider(); const { testCase: testCasePermission } = permissions; const [tableOptions, setTableOptions] = useState([]); - const [isTableLoading, setIsTableLoading] = useState(false); + const [isOptionsLoading, setIsOptionsLoading] = useState(false); + const [tagOptions, setTagOptions] = useState([]); + const [tierOptions, setTierOptions] = useState([]); + const [serviceOptions, setServiceOptions] = useState([]); const params = useMemo(() => { const search = location.search; @@ -204,6 +210,7 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { buildTestCaseParams(params, selectedFilter), isUndefined ); + if (!isEqual(filters, updatedParams)) { fetchTestCases(INITIAL_PAGING_VALUE, updatedParams); } @@ -211,40 +218,74 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { setFilters(updatedParams); }; - const handleMenuClick = ({ key }: { key: string }) => { - setSelectedFilter((prevSelected) => { - if (prevSelected.includes(key)) { - const updatedValue = prevSelected.filter( - (selected) => selected !== key - ); - const updatedFilters = omitBy( - buildTestCaseParams(filters, updatedValue), - isUndefined - ); - form.setFieldsValue({ [key]: undefined }); - if (!isEqual(filters, updatedFilters)) { - fetchTestCases(INITIAL_PAGING_VALUE, updatedFilters); - } - setFilters(updatedFilters); + const fetchTierOptions = async () => { + try { + setIsOptionsLoading(true); + const { data } = await getTags({ + parent: 'Tier', + }); - return updatedValue; - } + const options = data.map((hit) => { + return { + label: ( + + + {hit.fullyQualifiedName} + + + {getEntityName(hit)} + + + ), + value: hit.fullyQualifiedName, + }; + }); - return [...prevSelected, key]; - }); + setTierOptions(options); + } catch (error) { + setTierOptions([]); + } finally { + setIsOptionsLoading(false); + } }; - const filterMenu: ItemType[] = useMemo(() => { - return entries(TEST_CASE_FILTERS).map(([name, filter]) => ({ - key: filter, - label: startCase(name), - value: filter, - onClick: handleMenuClick, - })); - }, [filters]); + const fetchTagOptions = async (search?: string) => { + setIsOptionsLoading(true); + try { + const { data } = await tagClassBase.getTags(search ?? '', 1); + + const options = data + .filter( + ({ data: { classification } }) => + classification?.name !== TIER_CATEGORY + ) + .map(({ label, value }) => { + return { + label: ( + + + {value} + + {label} + + ), + value: value, + }; + }); + + setTagOptions(options); + } catch (error) { + setTagOptions([]); + } finally { + setIsOptionsLoading(false); + } + }; const fetchTableData = async (search = WILD_CARD_CHAR) => { - setIsTableLoading(true); + setIsOptionsLoading(true); try { const response = await searchQuery({ query: `*${search}*`, @@ -277,14 +318,99 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { } catch (error) { setTableOptions([]); } finally { - setIsTableLoading(false); + setIsOptionsLoading(false); } }; + const fetchServiceOptions = async (search = WILD_CARD_CHAR) => { + setIsOptionsLoading(true); + try { + const response = await searchQuery({ + query: `*${search}*`, + pageNumber: 1, + pageSize: PAGE_SIZE_BASE, + searchIndex: SearchIndex.DATABASE_SERVICE, + fetchSource: true, + includeFields: ['name', 'fullyQualifiedName', 'displayName'], + }); + + const options = response.hits.hits.map((hit) => { + return { + label: ( + + + {hit._source.fullyQualifiedName} + + + {getEntityName(hit._source)} + + + ), + value: hit._source.fullyQualifiedName, + }; + }); + setServiceOptions(options); + } catch (error) { + setServiceOptions([]); + } finally { + setIsOptionsLoading(false); + } + }; + + const handleMenuClick = ({ key }: { key: string }) => { + setSelectedFilter((prevSelected) => { + if (prevSelected.includes(key)) { + const updatedValue = prevSelected.filter( + (selected) => selected !== key + ); + const updatedFilters = omitBy( + buildTestCaseParams(filters, updatedValue), + isUndefined + ); + form.setFieldsValue({ [key]: undefined }); + if (!isEqual(filters, updatedFilters)) { + fetchTestCases(INITIAL_PAGING_VALUE, updatedFilters); + } + setFilters(updatedFilters); + + return updatedValue; + } + + return [...prevSelected, key]; + }); + + // Fetch options based on the selected filter + key === TEST_CASE_FILTERS.tier && fetchTierOptions(); + key === TEST_CASE_FILTERS.tags && fetchTagOptions(); + key === TEST_CASE_FILTERS.table && fetchTableData(); + key === TEST_CASE_FILTERS.service && fetchServiceOptions(); + }; + + const filterMenu: ItemType[] = useMemo(() => { + return entries(TEST_CASE_FILTERS).map(([name, filter]) => ({ + key: filter, + label: startCase(name), + value: filter, + onClick: handleMenuClick, + })); + }, [filters]); + const debounceFetchTableData = useCallback(debounce(fetchTableData, 1000), [ fetchTableData, ]); + const debounceFetchTagOptions = useCallback(debounce(fetchTagOptions, 1000), [ + fetchTagOptions, + ]); + + const debounceFetchServiceOptions = useCallback( + debounce(fetchServiceOptions, 1000), + [fetchServiceOptions] + ); + useEffect(() => { if (testCasePermission?.ViewAll || testCasePermission?.ViewBasic) { if (tab === DataQualityPageTabs.TEST_CASES) { @@ -295,10 +421,6 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { } }, [tab, searchValue, testCasePermission, pageSize]); - useEffect(() => { - fetchTableData(); - }, []); - const pagingData = useMemo( () => ({ paging, @@ -363,7 +485,7 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { allowClear showSearch data-testid="table-select-filter" - loading={isTableLoading} + loading={isOptionsLoading} options={tableOptions} placeholder={t('label.table')} onSearch={debounceFetchTableData} @@ -420,6 +542,52 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { )} + {selectedFilter.includes(TEST_CASE_FILTERS.tags) && ( + + + + )} + {selectedFilter.includes(TEST_CASE_FILTERS.service) && ( + +