yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-27 16:55:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

[MINOR] test case listing with prefix (#16382)

Browse Source 
* fix: test case listing with prefix

* fix: domain filtering

* added cypress for domain and data quality filters

---------

Co-authored-by: Shailesh Parmar <shailesh.parmar.webdev@gmail.com>
This commit is contained in:
Teddy 2024-05-22 15:22:20 +02:00 committed by GitHub
parent 8deed33150
commit 5555c3db88
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 282 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java
View File

@ -90,7 +90,8 @@ public class SearchListFilter extends Filter<SearchListFilter> {
private String getIncludeCondition() {
String domain = getQueryParam("domain");
if (!nullOrEmpty(domain)) {
return String.format("{\"term\": {\"domain.fullyQualifiedName\": \"%s\"}}", domain);
return String.format(
"{\"term\": {\"domain.fullyQualifiedName\": \"%s\"}}", escapeDoubleQuotes(domain));
}
return "";
}
@ -142,7 +143,10 @@ public class SearchListFilter extends Filter<SearchListFilter> {
conditions.add(
includeAllTests
? String.format(
"{\"prefix\": {\"entityFQN\": \"%s\"}}", escapeDoubleQuotes(entityFQN))
"{\"bool\":{\"should\": ["
+ "{\"prefix\": {\"entityFQN\": \"%s%s\"}},"
+ "{\"term\": {\"entityFQN\": \"%s\"}}]}}",
escapeDoubleQuotes(entityFQN), Entity.SEPARATOR, escapeDoubleQuotes(entityFQN))
: String.format(
"{\"term\": {\"entityFQN\": \"%s\"}}", escapeDoubleQuotes(entityFQN)));
}

5
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
View File

@ -54,7 +54,6 @@ import org.openmetadata.schema.tests.TestSuite;
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.FieldChange;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.UsageDetails;
import org.openmetadata.service.Entity;
@ -359,10 +358,6 @@ public class SearchRepository {
|| entityType.equalsIgnoreCase(Entity.STORAGE_SERVICE)
|| entityType.equalsIgnoreCase(Entity.SEARCH_SERVICE)) {
parentMatch = new ImmutablePair<>("service.id", entityId);
} else if (entityType.equalsIgnoreCase(Entity.TABLE)) {
EntityInterface entity =
Entity.getEntity(entityType, UUID.fromString(entityId), "", Include.ALL);
parentMatch = new ImmutablePair<>("entityFQN", entity.getFullyQualifiedName());
} else {
parentMatch = new ImmutablePair<>(entityType + ".id", entityId);
}

40
openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
View File

@ -710,18 +710,30 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
List<TestCase> testCases = new ArrayList<>();
for (int i = 0; i < tablesNum; i++) {
CreateTable tableReq =
tableResourceTest
.createRequest(testInfo, i)
.withDatabaseSchema(DATABASE_SCHEMA.getFullyQualifiedName())
.withColumns(
List.of(
new Column()
.withName(C1)
.withDisplayName("c1")
.withDataType(ColumnDataType.VARCHAR)
.withDataLength(10)))
.withOwner(USER1_REF);
CreateTable tableReq;
// Add entity FQN with same prefix to validate listing
// with AllTest=true returns all columns and table test for the
// specific entityFQN (and does not include tests from the other entityFQN
// witgh the same prefix
if (i == 0) {
tableReq = tableResourceTest.createRequest("test_getSimplelistFromSearch");
tableReq.getName();
} else if (i == 1) {
tableReq = tableResourceTest.createRequest("test_getSimplelistFromSearch_a");
tableReq.getName();
} else {
tableReq = tableResourceTest.createRequest(testInfo, i);
}
tableReq
.withDatabaseSchema(DATABASE_SCHEMA.getFullyQualifiedName())
.withColumns(
List.of(
new Column()
.withName(C1)
.withDisplayName("c1")
.withDataType(ColumnDataType.VARCHAR)
.withDataLength(10)))
.withOwner(USER1_REF);
Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS);
tables.add(table);
CreateTestSuite createTestSuite =
@ -764,11 +776,11 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
ResultList<TestCase> allEntities =
listEntitiesFromSearch(queryParams, testCasesNum, 0, ADMIN_AUTH_HEADERS);
assertEquals(testCasesNum, allEntities.getData().size());
queryParams.put("q", "test_getSimplelistFromSearcha");
queryParams.put("q", "test_getSimplelistFromSearchc");
allEntities = listEntitiesFromSearch(queryParams, testCasesNum, 0, ADMIN_AUTH_HEADERS);
assertEquals(1, allEntities.getData().size());
org.assertj.core.api.Assertions.assertThat(allEntities.getData().get(0).getName())
.contains("test_getSimplelistFromSearcha");
.contains("test_getSimplelistFromSearchc");
queryParams.clear();
queryParams.put("entityLink", testCaseForEL.getEntityLink());

37
openmetadata-ui/src/main/resources/ui/cypress/common/EntityUtils.ts
View File

@ -11,6 +11,7 @@
* limitations under the License.
*/
import { isArray } from 'lodash';
import {
DASHBOARD_SERVICE_DETAILS,
DATABASE_DETAILS,
@ -68,15 +69,18 @@ export const createEntityTable = ({
});
// Create Database Schema
cy.request({
method: 'POST',
url: `/api/v1/databaseSchemas`,
headers: { Authorization: `Bearer ${token}` },
body: schema,
}).then((response) => {
expect(response.status).to.eq(201);
const schemaData = isArray(schema) ? schema : [schema];
schemaData.map((schema) => {
cy.request({
method: 'POST',
url: `/api/v1/databaseSchemas`,
headers: { Authorization: `Bearer ${token}` },
body: schema,
}).then((response) => {
expect(response.status).to.eq(201);
createdEntityIds.databaseSchemaId = response.body.id;
createdEntityIds.databaseSchemaId = response.body.id;
});
});
tables.forEach((body) => {
@ -144,19 +148,20 @@ export const hardDeleteService = ({ serviceFqn, token, serviceType }) => {
});
};
export const generateRandomTable = (
tableName?: string,
columns?: ColumnType[]
) => {
export const generateRandomTable = (data?: {
tableName?: string;
columns?: ColumnType[];
databaseSchema?: string;
}) => {
const id = uuid();
const name = tableName ?? `cypress-table-${id}`;
const name = data?.tableName ?? `cypress-table-${id}`;
const table = {
name,
description: `cypress-table-description-${id}`,
displayName: name,
columns: [
...(columns ?? []),
...(data?.columns ?? []),
{
name: `cypress-column-${id}`,
description: `cypress-column-description-${id}`,
@ -164,7 +169,9 @@ export const generateRandomTable = (
dataTypeDisplay: 'numeric',
},
],
databaseSchema: `${DATABASE_SERVICE_DETAILS.name}.${DATABASE_DETAILS.name}.${SCHEMA_DETAILS.name}`,
databaseSchema:
data?.databaseSchema ??
`${DATABASE_SERVICE_DETAILS.name}.${DATABASE_DETAILS.name}.${SCHEMA_DETAILS.name}`,
};
return table;

195
openmetadata-ui/src/main/resources/ui/cypress/common/Utils/DataQuality.ts
View File

@ -12,10 +12,14 @@
*/
import { uuid } from '../../constants/constants';
import { EntityType } from '../../constants/Entity.interface';
import { DATABASE_SERVICE } from '../../constants/EntityConstant';
import {
DATABASE_DETAILS,
DATABASE_SERVICE,
DATABASE_SERVICE_DETAILS,
} from '../../constants/EntityConstant';
import { interceptURL, verifyResponseStatusCode } from '../common';
import { generateRandomTable } from '../EntityUtils';
import { visitEntityDetailsPage } from './Entity';
import { createEntityViaREST, visitEntityDetailsPage } from './Entity';
const tableFqn = `${DATABASE_SERVICE.entity.databaseSchema}.${DATABASE_SERVICE.entity.name}`;
@ -40,33 +44,63 @@ const testCase2 = {
testDefinition: 'columnValuesToBeInSet',
testSuite: testSuite.name,
};
const filterTable = generateRandomTable();
const customTable = generateRandomTable(`cypress-table-${uuid()}-COLUMN`, [
{
name: `user_id`,
description: `cypress-column-description`,
dataType: 'STRING',
dataTypeDisplay: 'string',
},
]);
const filterTableName = `cypress-table-${uuid()}`;
const testSchema = {
name: `cy-database-schema-${uuid()}`,
database: `${DATABASE_SERVICE_DETAILS.name}.${DATABASE_DETAILS.name}`,
};
const filterTable = generateRandomTable({ tableName: filterTableName });
const filterTable2 = generateRandomTable({
tableName: `${filterTableName}-model`,
});
const customTable = generateRandomTable({
tableName: `cypress-table-${uuid()}-COLUMN`,
columns: [
{
name: `user_id`,
description: `cypress-column-description`,
dataType: 'STRING',
dataTypeDisplay: 'string',
},
],
});
const filterTableFqn = `${filterTable.databaseSchema}.${filterTable.name}`;
const filterTableTestSuite = {
name: `${filterTableFqn}.testSuite`,
executableEntityReference: filterTableFqn,
};
const filterTableFqn2 = `${filterTable2.databaseSchema}.${filterTable2.name}`;
const filterTableTestSuite2 = {
name: `${filterTableFqn2}.testSuite`,
executableEntityReference: filterTableFqn2,
};
const testCases = [
`cy_first_table_column_count_to_be_between_${uuid()}`,
`cy_second_table_column_count_to_be_between_${uuid()}`,
`cy_third_table_column_count_to_be_between_${uuid()}`,
];
const smilerNameTestCase = testCases.map((test) => `${test}_version_2`);
export const domainDetails1 = {
name: `Cypress%Domain.${uuid()}`,
description: 'Cypress domain description',
domainType: 'Aggregate',
experts: [],
};
export const DATA_QUALITY_TEST_CASE_DATA = {
testCase1,
testCase2,
filterTable,
filterTable2,
customTable,
testSchema,
filterTableTestCases: testCases,
filterTable2TestCases: smilerNameTestCase,
domainDetail: domainDetails1,
};
// it will run 6 time with given wait time -> [20000, 10000, 5000, 2500, 1250, 625]
const verifyPipelineSuccessStatus = (time = 20000) => {
@ -146,7 +180,78 @@ export const triggerTestCasePipeline = ({
verifyPipelineSuccessStatus();
};
const prepareDataQualityTestCasesViaREST = ({
testSuite,
token,
testCases,
tableName,
serviceName,
}) => {
cy.request({
method: 'POST',
url: `/api/v1/dataQuality/testSuites/executable`,
headers: { Authorization: `Bearer ${token}` },
body: testSuite,
}).then((testSuiteResponse) => {
// creating test case
testCases.forEach((testCase) => {
cy.request({
method: 'POST',
url: `/api/v1/dataQuality/testCases`,
headers: { Authorization: `Bearer ${token}` },
body: {
name: testCase,
entityLink: `<#E::table::${testSuite.executableEntityReference}>`,
parameterValues: [
{ name: 'minColValue', value: 12 },
{ name: 'maxColValue', value: 24 },
],
testDefinition: 'tableColumnCountToBeBetween',
testSuite: testSuite.name,
},
});
});
cy.request({
method: 'POST',
url: `/api/v1/services/ingestionPipelines`,
headers: { Authorization: `Bearer ${token}` },
body: {
airflowConfig: {},
name: `${testSuite.executableEntityReference}_test_suite`,
pipelineType: 'TestSuite',
service: {
id: testSuiteResponse.body.id,
type: 'testSuite',
},
sourceConfig: {
config: {
type: 'TestSuite',
entityFullyQualifiedName: testSuite.executableEntityReference,
},
},
},
}).then((response) =>
cy.request({
method: 'POST',
url: `/api/v1/services/ingestionPipelines/deploy/${response.body.id}`,
headers: { Authorization: `Bearer ${token}` },
})
);
});
triggerTestCasePipeline({
serviceName: serviceName,
tableName: tableName,
});
};
export const prepareDataQualityTestCases = (token: string) => {
createEntityViaREST({
body: domainDetails1,
endPoint: EntityType.Domain,
token,
});
cy.request({
method: 'POST',
url: `/api/v1/dataQuality/testSuites/executable`,
@ -194,62 +299,18 @@ export const prepareDataQualityTestCases = (token: string) => {
);
});
cy.request({
method: 'POST',
url: `/api/v1/dataQuality/testSuites/executable`,
headers: { Authorization: `Bearer ${token}` },
body: filterTableTestSuite,
}).then((testSuiteResponse) => {
// creating test case
testCases.forEach((testCase) => {
cy.request({
method: 'POST',
url: `/api/v1/dataQuality/testCases`,
headers: { Authorization: `Bearer ${token}` },
body: {
name: testCase,
entityLink: `<#E::table::${filterTableTestSuite.executableEntityReference}>`,
parameterValues: [
{ name: 'minColValue', value: 12 },
{ name: 'maxColValue', value: 24 },
],
testDefinition: 'tableColumnCountToBeBetween',
testSuite: filterTableTestSuite.name,
},
});
});
cy.request({
method: 'POST',
url: `/api/v1/services/ingestionPipelines`,
headers: { Authorization: `Bearer ${token}` },
body: {
airflowConfig: {},
name: `${filterTableTestSuite.executableEntityReference}_test_suite`,
pipelineType: 'TestSuite',
service: {
id: testSuiteResponse.body.id,
type: 'testSuite',
},
sourceConfig: {
config: {
type: 'TestSuite',
entityFullyQualifiedName:
filterTableTestSuite.executableEntityReference,
},
},
},
}).then((response) =>
cy.request({
method: 'POST',
url: `/api/v1/services/ingestionPipelines/deploy/${response.body.id}`,
headers: { Authorization: `Bearer ${token}` },
})
);
});
triggerTestCasePipeline({
serviceName: DATABASE_SERVICE.service.name,
prepareDataQualityTestCasesViaREST({
testSuite: filterTableTestSuite,
token,
testCases: testCases,
tableName: filterTable.name,
serviceName: DATABASE_SERVICE.service.name,
});
prepareDataQualityTestCasesViaREST({
testSuite: filterTableTestSuite2,
token,
testCases: smilerNameTestCase,
tableName: filterTable2.name,
serviceName: DATABASE_SERVICE.service.name,
});
};

80
openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.ts
View File

@ -25,6 +25,7 @@ import {
DATA_QUALITY_TEST_CASE_DATA,
prepareDataQualityTestCases,
} from '../../common/Utils/DataQuality';
import { addDomainToEntity } from '../../common/Utils/Domain';
import { visitEntityDetailsPage } from '../../common/Utils/Entity';
import {
handleIngestionRetry,
@ -48,8 +49,16 @@ import { GlobalSettingOptions } from '../../constants/settings.constant';
const OWNER1 = 'Aaron Johnson';
const OWNER2 = 'Cynthia Meyer';
const { testCase1, testCase2, filterTable, filterTableTestCases, customTable } =
DATA_QUALITY_TEST_CASE_DATA;
const {
testCase1,
testCase2,
filterTable,
filterTable2,
filterTableTestCases,
filterTable2TestCases,
customTable,
domainDetail,
} = DATA_QUALITY_TEST_CASE_DATA;
const TEAM_ENTITY = customTable.name;
const serviceName = DATABASE_SERVICE.service.name;
const goToProfilerTab = (data?: { service: string; entityName: string }) => {
@ -111,7 +120,12 @@ describe(
createEntityTable({
token,
...DATABASE_SERVICE,
tables: [DATABASE_SERVICE.entity, filterTable, customTable],
tables: [
DATABASE_SERVICE.entity,
filterTable,
filterTable2,
customTable,
],
});
prepareDataQualityTestCases(token);
@ -982,11 +996,26 @@ describe(
`/api/v1/dataQuality/testCases/search/list?*entityLink=*${filterTable.name}*`,
'searchTestCaseByTable'
);
interceptURL(
'GET',
`/api/v1/search/query?q=*index=table_search_index*`,
'searchTable'
);
cy.get('#tableFqn').scrollIntoView().type(filterTable.name);
selectOptionFromDropdown(filterTable.name);
verifyResponseStatusCode('@searchTable', 200);
cy.get('.ant-select-dropdown')
.not('.ant-select-dropdown-hidden')
.find(
`[data-testid="${filterTable.databaseSchema}.${filterTable.name}"]`
)
.click({ force: true });
verifyResponseStatusCode('@searchTestCaseByTable', 200);
verifyFilterTestCase();
filterTable2TestCases.map((testCase) => {
cy.get(`[data-testid="${testCase}"]`).should('not.exist');
});
// Test case filter by test type
interceptURL(
'GET',
@ -1061,6 +1090,49 @@ describe(
verifyFilterTestCase();
});
it('Filter with domain', () => {
visitEntityDetailsPage({
term: filterTable.name,
serviceName: serviceName,
entity: EntityType.Table,
});
addDomainToEntity(domainDetail.name);
interceptURL(
'GET',
'/api/v1/dataQuality/testCases/search/list?*',
'getTestCase'
);
cy.get('[data-testid="domain-dropdown"]').click();
cy.get(`li[data-menu-id*='${domainDetail.name}']`).click();
cy.sidebarClick(SidebarItem.DATA_QUALITY);
cy.get('[data-testid="by-test-cases"]').click();
verifyResponseStatusCode('@getTestCase', 200);
cy.get('[data-testid="advanced-filter"]').click({
waitForAnimations: true,
});
cy.get('[value="tableFqn"]').click({ waitForAnimations: true });
// Test case filter by table name
interceptURL(
'GET',
`/api/v1/dataQuality/testCases/search/list?*entityLink=*${filterTable.name}*`,
'searchTestCaseByTable'
);
cy.get('#tableFqn').scrollIntoView().type(filterTable.name);
cy.get('.ant-select-dropdown')
.not('.ant-select-dropdown-hidden')
.find(
`[data-testid="${filterTable.databaseSchema}.${filterTable.name}"]`
)
.click({ force: true });
verifyResponseStatusCode('@searchTestCaseByTable', 200);
verifyFilterTestCase();
});
it('Update profiler setting modal', () => {
const profilerSetting = {
profileSample: '60',

25
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx
View File

@ -20,6 +20,7 @@ import {
Row,
Select,
Space,
Typography,
} from 'antd';
import { useForm } from 'antd/lib/form/Form';
import { ItemType } from 'antd/lib/menu/hooks/useItems';
@ -254,10 +255,24 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => {
includeFields: ['name', 'fullyQualifiedName', 'displayName'],
});
const options = response.hits.hits.map((hit) => ({
label: getEntityName(hit._source),
value: hit._source.fullyQualifiedName,
}));
const options = response.hits.hits.map((hit) => {
return {
label: (
<Space
data-testid={hit._source.fullyQualifiedName}
direction="vertical"
size={0}>
<Typography.Text className="text-xs text-grey-muted">
{hit._source.fullyQualifiedName}
</Typography.Text>
<Typography.Text className="text-sm">
{getEntityName(hit._source)}
</Typography.Text>
</Space>
),
value: hit._source.fullyQualifiedName,
};
});
setTableOptions(options);
} catch (error) {
setTableOptions([]);
@ -341,7 +356,7 @@ export const TestCases = ({ summaryPanel }: { summaryPanel: ReactNode }) => {
</Form.Item>
{selectedFilter.includes(TEST_CASE_FILTERS.table) && (
<Form.Item
className="m-0 w-52"
className="m-0 w-80"
label={t('label.table')}
name="tableFqn">
<Select

4
openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.test.tsx
View File

@ -137,7 +137,9 @@ describe('DataQualityTab test', () => {
expect(testName).toBeInTheDocument();
expect(tableLink).toBeInTheDocument();
expect(tableLink.textContent).toEqual('dim_address');
expect(tableLink.textContent).toEqual(
'sample_data.ecommerce_db.shopify.dim_address'
);
expect(columnName).toBeInTheDocument();
expect(editButton).toBeInTheDocument();
expect(deleteButton).toBeInTheDocument();

3
openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx
View File

@ -179,7 +179,6 @@ const DataQualityTab: React.FC<DataQualityTabProps> = ({
width: 150,
render: (entityLink: string) => {
const tableFqn = getEntityFQN(entityLink);
const name = getNameFromFQN(tableFqn);
return (
<Link
@ -195,7 +194,7 @@ const DataQualityTab: React.FC<DataQualityTabProps> = ({
}),
}}
onClick={(e) => e.stopPropagation()}>
{name}
{tableFqn}
</Link>
);
},