diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java index abcca07d941..cf746857134 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java @@ -90,7 +90,8 @@ public class SearchListFilter extends Filter { private String getIncludeCondition() { String domain = getQueryParam("domain"); if (!nullOrEmpty(domain)) { - return String.format("{\"term\": {\"domain.fullyQualifiedName\": \"%s\"}}", domain); + return String.format( + "{\"term\": {\"domain.fullyQualifiedName\": \"%s\"}}", escapeDoubleQuotes(domain)); } return ""; } @@ -142,7 +143,10 @@ public class SearchListFilter extends Filter { conditions.add( includeAllTests ? String.format( - "{\"prefix\": {\"entityFQN\": \"%s\"}}", escapeDoubleQuotes(entityFQN)) + "{\"bool\":{\"should\": [" + + "{\"prefix\": {\"entityFQN\": \"%s%s\"}}," + + "{\"term\": {\"entityFQN\": \"%s\"}}]}}", + escapeDoubleQuotes(entityFQN), Entity.SEPARATOR, escapeDoubleQuotes(entityFQN)) : String.format( "{\"term\": {\"entityFQN\": \"%s\"}}", escapeDoubleQuotes(entityFQN))); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 0d8a10ddf6c..ea151540200 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -54,7 +54,6 @@ import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.FieldChange; -import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.UsageDetails; import org.openmetadata.service.Entity; @@ -359,10 +358,6 @@ public class SearchRepository { || entityType.equalsIgnoreCase(Entity.STORAGE_SERVICE) || entityType.equalsIgnoreCase(Entity.SEARCH_SERVICE)) { parentMatch = new ImmutablePair<>("service.id", entityId); - } else if (entityType.equalsIgnoreCase(Entity.TABLE)) { - EntityInterface entity = - Entity.getEntity(entityType, UUID.fromString(entityId), "", Include.ALL); - parentMatch = new ImmutablePair<>("entityFQN", entity.getFullyQualifiedName()); } else { parentMatch = new ImmutablePair<>(entityType + ".id", entityId); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index 397e7133cae..48d12359d49 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -710,18 +710,30 @@ public class TestCaseResourceTest extends EntityResourceTest testCases = new ArrayList<>(); for (int i = 0; i < tablesNum; i++) { - CreateTable tableReq = - tableResourceTest - .createRequest(testInfo, i) - .withDatabaseSchema(DATABASE_SCHEMA.getFullyQualifiedName()) - .withColumns( - List.of( - new Column() - .withName(C1) - .withDisplayName("c1") - .withDataType(ColumnDataType.VARCHAR) - .withDataLength(10))) - .withOwner(USER1_REF); + CreateTable tableReq; + // Add entity FQN with same prefix to validate listing + // with AllTest=true returns all columns and table test for the + // specific entityFQN (and does not include tests from the other entityFQN + // witgh the same prefix + if (i == 0) { + tableReq = tableResourceTest.createRequest("test_getSimplelistFromSearch"); + tableReq.getName(); + } else if (i == 1) { + tableReq = tableResourceTest.createRequest("test_getSimplelistFromSearch_a"); + tableReq.getName(); + } else { + tableReq = tableResourceTest.createRequest(testInfo, i); + } + tableReq + .withDatabaseSchema(DATABASE_SCHEMA.getFullyQualifiedName()) + .withColumns( + List.of( + new Column() + .withName(C1) + .withDisplayName("c1") + .withDataType(ColumnDataType.VARCHAR) + .withDataLength(10))) + .withOwner(USER1_REF); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); tables.add(table); CreateTestSuite createTestSuite = @@ -764,11 +776,11 @@ public class TestCaseResourceTest extends EntityResourceTest allEntities = listEntitiesFromSearch(queryParams, testCasesNum, 0, ADMIN_AUTH_HEADERS); assertEquals(testCasesNum, allEntities.getData().size()); - queryParams.put("q", "test_getSimplelistFromSearcha"); + queryParams.put("q", "test_getSimplelistFromSearchc"); allEntities = listEntitiesFromSearch(queryParams, testCasesNum, 0, ADMIN_AUTH_HEADERS); assertEquals(1, allEntities.getData().size()); org.assertj.core.api.Assertions.assertThat(allEntities.getData().get(0).getName()) - .contains("test_getSimplelistFromSearcha"); + .contains("test_getSimplelistFromSearchc"); queryParams.clear(); queryParams.put("entityLink", testCaseForEL.getEntityLink()); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.ts index f15466c289e..7eea0542c65 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.ts @@ -11,6 +11,7 @@ * limitations under the License. */ +import { isArray } from 'lodash'; import { DASHBOARD_SERVICE_DETAILS, DATABASE_DETAILS, @@ -68,15 +69,18 @@ export const createEntityTable = ({ }); // Create Database Schema - cy.request({ - method: 'POST', - url: `/api/v1/databaseSchemas`, - headers: { Authorization: `Bearer ${token}` }, - body: schema, - }).then((response) => { - expect(response.status).to.eq(201); + const schemaData = isArray(schema) ? schema : [schema]; + schemaData.map((schema) => { + cy.request({ + method: 'POST', + url: `/api/v1/databaseSchemas`, + headers: { Authorization: `Bearer ${token}` }, + body: schema, + }).then((response) => { + expect(response.status).to.eq(201); - createdEntityIds.databaseSchemaId = response.body.id; + createdEntityIds.databaseSchemaId = response.body.id; + }); }); tables.forEach((body) => { @@ -144,19 +148,20 @@ export const hardDeleteService = ({ serviceFqn, token, serviceType }) => { }); }; -export const generateRandomTable = ( - tableName?: string, - columns?: ColumnType[] -) => { +export const generateRandomTable = (data?: { + tableName?: string; + columns?: ColumnType[]; + databaseSchema?: string; +}) => { const id = uuid(); - const name = tableName ?? `cypress-table-${id}`; + const name = data?.tableName ?? `cypress-table-${id}`; const table = { name, description: `cypress-table-description-${id}`, displayName: name, columns: [ - ...(columns ?? []), + ...(data?.columns ?? []), { name: `cypress-column-${id}`, description: `cypress-column-description-${id}`, @@ -164,7 +169,9 @@ export const generateRandomTable = ( dataTypeDisplay: 'numeric', }, ], - databaseSchema: `${DATABASE_SERVICE_DETAILS.name}.${DATABASE_DETAILS.name}.${SCHEMA_DETAILS.name}`, + databaseSchema: + data?.databaseSchema ?? + `${DATABASE_SERVICE_DETAILS.name}.${DATABASE_DETAILS.name}.${SCHEMA_DETAILS.name}`, }; return table; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts index 1e67e67ed3b..bc06577e8cd 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts @@ -12,10 +12,14 @@ */ import { uuid } from '../../constants/constants'; import { EntityType } from '../../constants/Entity.interface'; -import { DATABASE_SERVICE } from '../../constants/EntityConstant'; +import { + DATABASE_DETAILS, + DATABASE_SERVICE, + DATABASE_SERVICE_DETAILS, +} from '../../constants/EntityConstant'; import { interceptURL, verifyResponseStatusCode } from '../common'; import { generateRandomTable } from '../EntityUtils'; -import { visitEntityDetailsPage } from './Entity'; +import { createEntityViaREST, visitEntityDetailsPage } from './Entity'; const tableFqn = `${DATABASE_SERVICE.entity.databaseSchema}.${DATABASE_SERVICE.entity.name}`; @@ -40,33 +44,63 @@ const testCase2 = { testDefinition: 'columnValuesToBeInSet', testSuite: testSuite.name, }; -const filterTable = generateRandomTable(); -const customTable = generateRandomTable(`cypress-table-${uuid()}-COLUMN`, [ - { - name: `user_id`, - description: `cypress-column-description`, - dataType: 'STRING', - dataTypeDisplay: 'string', - }, -]); +const filterTableName = `cypress-table-${uuid()}`; +const testSchema = { + name: `cy-database-schema-${uuid()}`, + database: `${DATABASE_SERVICE_DETAILS.name}.${DATABASE_DETAILS.name}`, +}; +const filterTable = generateRandomTable({ tableName: filterTableName }); +const filterTable2 = generateRandomTable({ + tableName: `${filterTableName}-model`, +}); +const customTable = generateRandomTable({ + tableName: `cypress-table-${uuid()}-COLUMN`, + columns: [ + { + name: `user_id`, + description: `cypress-column-description`, + dataType: 'STRING', + dataTypeDisplay: 'string', + }, + ], +}); const filterTableFqn = `${filterTable.databaseSchema}.${filterTable.name}`; const filterTableTestSuite = { name: `${filterTableFqn}.testSuite`, executableEntityReference: filterTableFqn, }; +const filterTableFqn2 = `${filterTable2.databaseSchema}.${filterTable2.name}`; +const filterTableTestSuite2 = { + name: `${filterTableFqn2}.testSuite`, + executableEntityReference: filterTableFqn2, +}; + const testCases = [ `cy_first_table_column_count_to_be_between_${uuid()}`, `cy_second_table_column_count_to_be_between_${uuid()}`, `cy_third_table_column_count_to_be_between_${uuid()}`, ]; +const smilerNameTestCase = testCases.map((test) => `${test}_version_2`); + +export const domainDetails1 = { + name: `Cypress%Domain.${uuid()}`, + description: 'Cypress domain description', + domainType: 'Aggregate', + experts: [], +}; + export const DATA_QUALITY_TEST_CASE_DATA = { testCase1, testCase2, filterTable, + filterTable2, customTable, + testSchema, filterTableTestCases: testCases, + filterTable2TestCases: smilerNameTestCase, + domainDetail: domainDetails1, }; // it will run 6 time with given wait time -> [20000, 10000, 5000, 2500, 1250, 625] const verifyPipelineSuccessStatus = (time = 20000) => { @@ -146,7 +180,78 @@ export const triggerTestCasePipeline = ({ verifyPipelineSuccessStatus(); }; +const prepareDataQualityTestCasesViaREST = ({ + testSuite, + token, + testCases, + tableName, + serviceName, +}) => { + cy.request({ + method: 'POST', + url: `/api/v1/dataQuality/testSuites/executable`, + headers: { Authorization: `Bearer ${token}` }, + body: testSuite, + }).then((testSuiteResponse) => { + // creating test case + + testCases.forEach((testCase) => { + cy.request({ + method: 'POST', + url: `/api/v1/dataQuality/testCases`, + headers: { Authorization: `Bearer ${token}` }, + body: { + name: testCase, + entityLink: `<#E::table::${testSuite.executableEntityReference}>`, + parameterValues: [ + { name: 'minColValue', value: 12 }, + { name: 'maxColValue', value: 24 }, + ], + testDefinition: 'tableColumnCountToBeBetween', + testSuite: testSuite.name, + }, + }); + }); + cy.request({ + method: 'POST', + url: `/api/v1/services/ingestionPipelines`, + headers: { Authorization: `Bearer ${token}` }, + body: { + airflowConfig: {}, + name: `${testSuite.executableEntityReference}_test_suite`, + pipelineType: 'TestSuite', + service: { + id: testSuiteResponse.body.id, + type: 'testSuite', + }, + sourceConfig: { + config: { + type: 'TestSuite', + entityFullyQualifiedName: testSuite.executableEntityReference, + }, + }, + }, + }).then((response) => + cy.request({ + method: 'POST', + url: `/api/v1/services/ingestionPipelines/deploy/${response.body.id}`, + headers: { Authorization: `Bearer ${token}` }, + }) + ); + }); + + triggerTestCasePipeline({ + serviceName: serviceName, + tableName: tableName, + }); +}; + export const prepareDataQualityTestCases = (token: string) => { + createEntityViaREST({ + body: domainDetails1, + endPoint: EntityType.Domain, + token, + }); cy.request({ method: 'POST', url: `/api/v1/dataQuality/testSuites/executable`, @@ -194,62 +299,18 @@ export const prepareDataQualityTestCases = (token: string) => { ); }); - cy.request({ - method: 'POST', - url: `/api/v1/dataQuality/testSuites/executable`, - headers: { Authorization: `Bearer ${token}` }, - body: filterTableTestSuite, - }).then((testSuiteResponse) => { - // creating test case - - testCases.forEach((testCase) => { - cy.request({ - method: 'POST', - url: `/api/v1/dataQuality/testCases`, - headers: { Authorization: `Bearer ${token}` }, - body: { - name: testCase, - entityLink: `<#E::table::${filterTableTestSuite.executableEntityReference}>`, - parameterValues: [ - { name: 'minColValue', value: 12 }, - { name: 'maxColValue', value: 24 }, - ], - testDefinition: 'tableColumnCountToBeBetween', - testSuite: filterTableTestSuite.name, - }, - }); - }); - cy.request({ - method: 'POST', - url: `/api/v1/services/ingestionPipelines`, - headers: { Authorization: `Bearer ${token}` }, - body: { - airflowConfig: {}, - name: `${filterTableTestSuite.executableEntityReference}_test_suite`, - pipelineType: 'TestSuite', - service: { - id: testSuiteResponse.body.id, - type: 'testSuite', - }, - sourceConfig: { - config: { - type: 'TestSuite', - entityFullyQualifiedName: - filterTableTestSuite.executableEntityReference, - }, - }, - }, - }).then((response) => - cy.request({ - method: 'POST', - url: `/api/v1/services/ingestionPipelines/deploy/${response.body.id}`, - headers: { Authorization: `Bearer ${token}` }, - }) - ); - }); - - triggerTestCasePipeline({ - serviceName: DATABASE_SERVICE.service.name, + prepareDataQualityTestCasesViaREST({ + testSuite: filterTableTestSuite, + token, + testCases: testCases, tableName: filterTable.name, + serviceName: DATABASE_SERVICE.service.name, + }); + prepareDataQualityTestCasesViaREST({ + testSuite: filterTableTestSuite2, + token, + testCases: smilerNameTestCase, + tableName: filterTable2.name, + serviceName: DATABASE_SERVICE.service.name, }); }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts index 99bbbde60f6..036b85790a6 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts @@ -25,6 +25,7 @@ import { DATA_QUALITY_TEST_CASE_DATA, prepareDataQualityTestCases, } from '../../common/Utils/DataQuality'; +import { addDomainToEntity } from '../../common/Utils/Domain'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; import { handleIngestionRetry, @@ -48,8 +49,16 @@ import { GlobalSettingOptions } from '../../constants/settings.constant'; const OWNER1 = 'Aaron Johnson'; const OWNER2 = 'Cynthia Meyer'; -const { testCase1, testCase2, filterTable, filterTableTestCases, customTable } = - DATA_QUALITY_TEST_CASE_DATA; +const { + testCase1, + testCase2, + filterTable, + filterTable2, + filterTableTestCases, + filterTable2TestCases, + customTable, + domainDetail, +} = DATA_QUALITY_TEST_CASE_DATA; const TEAM_ENTITY = customTable.name; const serviceName = DATABASE_SERVICE.service.name; const goToProfilerTab = (data?: { service: string; entityName: string }) => { @@ -111,7 +120,12 @@ describe( createEntityTable({ token, ...DATABASE_SERVICE, - tables: [DATABASE_SERVICE.entity, filterTable, customTable], + tables: [ + DATABASE_SERVICE.entity, + filterTable, + filterTable2, + customTable, + ], }); prepareDataQualityTestCases(token); @@ -982,11 +996,26 @@ describe( `/api/v1/dataQuality/testCases/search/list?*entityLink=*${filterTable.name}*`, 'searchTestCaseByTable' ); + interceptURL( + 'GET', + `/api/v1/search/query?q=*index=table_search_index*`, + 'searchTable' + ); cy.get('#tableFqn').scrollIntoView().type(filterTable.name); - selectOptionFromDropdown(filterTable.name); + verifyResponseStatusCode('@searchTable', 200); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .find( + `[data-testid="${filterTable.databaseSchema}.${filterTable.name}"]` + ) + .click({ force: true }); verifyResponseStatusCode('@searchTestCaseByTable', 200); verifyFilterTestCase(); + filterTable2TestCases.map((testCase) => { + cy.get(`[data-testid="${testCase}"]`).should('not.exist'); + }); + // Test case filter by test type interceptURL( 'GET', @@ -1061,6 +1090,49 @@ describe( verifyFilterTestCase(); }); + it('Filter with domain', () => { + visitEntityDetailsPage({ + term: filterTable.name, + serviceName: serviceName, + entity: EntityType.Table, + }); + + addDomainToEntity(domainDetail.name); + + interceptURL( + 'GET', + '/api/v1/dataQuality/testCases/search/list?*', + 'getTestCase' + ); + cy.get('[data-testid="domain-dropdown"]').click(); + cy.get(`li[data-menu-id*='${domainDetail.name}']`).click(); + cy.sidebarClick(SidebarItem.DATA_QUALITY); + + cy.get('[data-testid="by-test-cases"]').click(); + verifyResponseStatusCode('@getTestCase', 200); + + cy.get('[data-testid="advanced-filter"]').click({ + waitForAnimations: true, + }); + cy.get('[value="tableFqn"]').click({ waitForAnimations: true }); + + // Test case filter by table name + interceptURL( + 'GET', + `/api/v1/dataQuality/testCases/search/list?*entityLink=*${filterTable.name}*`, + 'searchTestCaseByTable' + ); + cy.get('#tableFqn').scrollIntoView().type(filterTable.name); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .find( + `[data-testid="${filterTable.databaseSchema}.${filterTable.name}"]` + ) + .click({ force: true }); + verifyResponseStatusCode('@searchTestCaseByTable', 200); + verifyFilterTestCase(); + }); + it('Update profiler setting modal', () => { const profilerSetting = { profileSample: '60', diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx index 84c47a7e570..3095cd066d0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx @@ -20,6 +20,7 @@ import { Row, Select, Space, + Typography, } from 'antd'; import { useForm } from 'antd/lib/form/Form'; import { ItemType } from 'antd/lib/menu/hooks/useItems'; @@ -254,10 +255,24 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { includeFields: ['name', 'fullyQualifiedName', 'displayName'], }); - const options = response.hits.hits.map((hit) => ({ - label: getEntityName(hit._source), - value: hit._source.fullyQualifiedName, - })); + const options = response.hits.hits.map((hit) => { + return { + label: ( + + + {hit._source.fullyQualifiedName} + + + {getEntityName(hit._source)} + + + ), + value: hit._source.fullyQualifiedName, + }; + }); setTableOptions(options); } catch (error) { setTableOptions([]); @@ -341,7 +356,7 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => { {selectedFilter.includes(TEST_CASE_FILTERS.table) && (