yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-11 19:20:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

UI: Added Data Quality test suite listing page and detail page (#7014)

Browse Source 
* initial setup for data quality form

* added stepper and form component

* added select/add test suite step form

* added form for table test and cron step

* added data quality table test form flow

* added test suite listing page

* bug fix for profiler

* added column test form

* render right panel information dynamically

* updated test as per new changes

* updated data test id

* Fixed ingestionPipeline

* Fixed pytest + python format

* miner fix, added sql editor

* removed filter for duplicate test check

* miner fix

* updated all API call for test suite ingestion page

* updated test suite view redirection

* updated ingestion infinite loader

* added edit action for test case

* added edit/add ingestion flow

* added after action for form

* remove duplicate close action

* fixed alignment issue

* added expand on row click functionality and change expand/collapse icon

* used antd component

* resolving conflict

* Fixed#7040: Test-suite count support in counts API (#7041)

* updated test suite count

* fixed failing unit test

Co-authored-by: Teddy Crepineau <teddy.crepineau@gmail.com>
Co-authored-by: Parth Panchal <83201188+parthp2107@users.noreply.github.com>
This commit is contained in:
Shailesh Parmar 2022-08-30 19:00:51 +05:30 committed by GitHub
parent 5a3fcafb3b
commit 3bcef4f58c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
45 changed files with 2002 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java
View File

@ -2784,7 +2784,8 @@ public interface CollectionDAO {
.withMlmodelCount(rs.getInt("mlmodelCount"))
.withServicesCount(rs.getInt("servicesCount"))
.withUserCount(rs.getInt("userCount"))
.withTeamCount(rs.getInt("teamCount"));
.withTeamCount(rs.getInt("teamCount"))
.withTestSuiteCount(rs.getInt("testSuiteCount"));
}
}
@ -2814,7 +2815,8 @@ public interface CollectionDAO {
+ "(SELECT COUNT(*) FROM pipeline_service_entity)+ "
+ "(SELECT COUNT(*) FROM mlmodel_service_entity)) as servicesCount, "
+ "(SELECT COUNT(*) FROM user_entity WHERE JSON_EXTRACT(json, '$.isBot') IS NULL OR JSON_EXTRACT(json, '$.isBot') = FALSE) as userCount, "
+ "(SELECT COUNT(*) FROM team_entity) as teamCount",
+ "(SELECT COUNT(*) FROM team_entity) as teamCount, "
+ "(SELECT COUNT(*) FROM test_suite) as testSuiteCount",
connectionType = MYSQL)
@ConnectionAwareSqlQuery(
value =
@ -2829,7 +2831,8 @@ public interface CollectionDAO {
+ "(SELECT COUNT(*) FROM pipeline_service_entity)+ "
+ "(SELECT COUNT(*) FROM mlmodel_service_entity)) as servicesCount, "
+ "(SELECT COUNT(*) FROM user_entity WHERE json#>'{isBot}' IS NULL OR ((json#>'{isBot}')::boolean) = FALSE) as userCount, "
+ "(SELECT COUNT(*) FROM team_entity) as teamCount",
+ "(SELECT COUNT(*) FROM team_entity) as teamCount, "
+ "(SELECT COUNT(*) FROM test_suite) as testSuiteCount",
connectionType = POSTGRES)
@RegisterRowMapper(EntitiesCountRowMapper.class)
EntitiesCount getAggregatedEntitiesCount() throws StatementException;

4
catalog-rest-service/src/main/resources/json/schema/entity/utils/entitiesCount.json
View File

@ -37,6 +37,10 @@
"teamCount": {
"description": "Team Count",
"type": "integer"
},
"testSuiteCount": {
"description": "Test Suite Count",
"type": "integer"
}
},
"additionalProperties": false

10
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/util/UtilResourceTest.java
View File

@ -25,10 +25,12 @@ import org.openmetadata.catalog.api.services.CreateMlModelService;
import org.openmetadata.catalog.api.services.CreatePipelineService;
import org.openmetadata.catalog.api.teams.CreateTeam;
import org.openmetadata.catalog.api.teams.CreateUser;
import org.openmetadata.catalog.api.tests.CreateTestSuite;
import org.openmetadata.catalog.entity.data.Table;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.dashboards.DashboardResourceTest;
import org.openmetadata.catalog.resources.databases.TableResourceTest;
import org.openmetadata.catalog.resources.dqtests.TestSuiteResourceTest;
import org.openmetadata.catalog.resources.pipelines.PipelineResourceTest;
import org.openmetadata.catalog.resources.services.DashboardServiceResourceTest;
import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest;
@ -73,6 +75,7 @@ public class UtilResourceTest extends CatalogApplicationTest {
int beforeServiceCount = getEntitiesCount().getServicesCount();
int beforeUserCount = getEntitiesCount().getUserCount();
int beforeTeamCount = getEntitiesCount().getTeamCount();
int beforeTestSuiteCount = getEntitiesCount().getTestSuiteCount();
// Create Table
TableResourceTest tableResourceTest = new TableResourceTest();
@ -109,6 +112,11 @@ public class UtilResourceTest extends CatalogApplicationTest {
CreateTeam createTeam = teamResourceTest.createRequest(test);
teamResourceTest.createEntity(createTeam, ADMIN_AUTH_HEADERS);
// Create Test Suite
TestSuiteResourceTest testSuiteResourceTest = new TestSuiteResourceTest();
CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(test);
testSuiteResourceTest.createEntity(createTestSuite, ADMIN_AUTH_HEADERS);
// Get count after adding entities
int afterTableCount = getEntitiesCount().getTableCount();
int afterDashboardCount = getEntitiesCount().getDashboardCount();
@ -117,6 +125,7 @@ public class UtilResourceTest extends CatalogApplicationTest {
int afterServiceCount = getEntitiesCount().getServicesCount();
int afterUserCount = getEntitiesCount().getUserCount();
int afterTeamCount = getEntitiesCount().getTeamCount();
int afterTestSuiteCount = getEntitiesCount().getTestSuiteCount();
int actualCount = 1;
@ -127,6 +136,7 @@ public class UtilResourceTest extends CatalogApplicationTest {
Assertions.assertEquals(afterTableCount - beforeTableCount, actualCount);
Assertions.assertEquals(afterTeamCount - beforeTeamCount, actualCount);
Assertions.assertEquals(afterTopicCount - beforeTopicCount, actualCount);
Assertions.assertEquals(afterTestSuiteCount - beforeTestSuiteCount, actualCount);
}
@Test

3
openmetadata-ui/src/main/resources/ui/src/assets/svg/arrow-down.svg Normal file
View File

@ -0,0 +1,3 @@
<svg width="10" height="6" viewBox="0 0 10 6" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M0.292433 0.304675C0.479735 0.117581 0.733736 0.0124767 0.998581 0.0124767C1.26342 0.0124767 1.51743 0.117581 1.70473 0.304675L4.99376 3.59106L8.28278 0.304675C8.37492 0.209357 8.48513 0.133328 8.60699 0.0810238C8.72885 0.0287201 8.85991 0.00118918 8.99253 3.76812e-05C9.12515 -0.00111382 9.25667 0.0241369 9.37941 0.0743168C9.50216 0.124497 9.61368 0.198601 9.70746 0.292305C9.80124 0.386009 9.8754 0.497437 9.92562 0.620086C9.97584 0.742736 10.0011 0.874151 9.99996 1.00666C9.99881 1.13918 9.97126 1.27013 9.91891 1.39189C9.86656 1.51365 9.79047 1.62377 9.69508 1.71583L5.6999 5.7078C5.5126 5.8949 5.2586 6 4.99376 6C4.72891 6 4.47491 5.8949 4.28761 5.7078L0.292433 1.71583C0.105189 1.52868 0 1.27489 0 1.01026C0 0.745624 0.105189 0.491826 0.292433 0.304675Z" fill="black"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 927 B

3
openmetadata-ui/src/main/resources/ui/src/assets/svg/arrow-right.svg Normal file
View File

@ -0,0 +1,3 @@
<svg width="6" height="10" viewBox="0 0 6 10" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M0.304675 9.70757C0.117581 9.52027 0.0124767 9.26626 0.0124767 9.00142C0.0124767 8.73658 0.117581 8.48257 0.304675 8.29527L3.59106 5.00624L0.304675 1.71722C0.209357 1.62508 0.133328 1.51487 0.0810238 1.39301C0.0287201 1.27115 0.00118918 1.14009 3.76812e-05 1.00747C-0.00111382 0.874854 0.0241369 0.743334 0.0743168 0.620585C0.124497 0.497838 0.198601 0.386321 0.292305 0.292542C0.386009 0.198762 0.497437 0.124599 0.620086 0.074378C0.742736 0.0241575 0.874151 -0.00111389 1.00666 3.8147e-05C1.13918 0.00119114 1.27013 0.0287437 1.39189 0.081089C1.51365 0.133435 1.62377 0.209526 1.71583 0.304921L5.7078 4.3001C5.8949 4.4874 6 4.7414 6 5.00624C6 5.27109 5.8949 5.52509 5.7078 5.71239L1.71583 9.70757C1.52868 9.89481 1.27489 10 1.01026 10C0.745624 10 0.491826 9.89481 0.304675 9.70757Z" fill="black"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 950 B

14
openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts
View File

@ -12,6 +12,7 @@
*/
import { AxiosResponse } from 'axios';
import { Operation } from 'fast-json-patch';
import { CreateIngestionPipeline } from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
import { IngestionPipeline } from '../generated/entity/services/ingestionPipelines/ingestionPipeline';
import { Paging } from '../generated/type/paging';
@ -96,6 +97,19 @@ export const updateIngestionPipeline = async (
return response.data;
};
export const patchIngestionPipeline = async (id: string, data: Operation[]) => {
const configOptions = {
headers: { 'Content-type': 'application/json-patch+json' },
};
const response = await APIClient.patch<
Operation[],
AxiosResponse<IngestionPipeline>
>(`/services/ingestionPipelines/${id}`, data, configOptions);
return response.data;
};
export const checkAirflowStatus = (): Promise<AxiosResponse> => {
return APIClient.get('/services/ingestionPipelines/status');
};

81
openmetadata-ui/src/main/resources/ui/src/axiosAPIs/testAPI.ts
View File

@ -11,6 +11,8 @@
* limitations under the License.
*/
import { AxiosResponse } from 'axios';
import { Operation } from 'fast-json-patch';
import { CreateTestCase } from '../generated/api/tests/createTestCase';
import { CreateTestSuite } from '../generated/api/tests/createTestSuite';
import { TestCase, TestCaseResult } from '../generated/tests/testCase';
@ -83,20 +85,16 @@ export const createTestCase = async (data: CreateTestCase) => {
return response.data;
};
// testSuite section
export const getListTestSuites = async (params?: ListParams) => {
const response = await APIClient.get<{
data: TestSuite[];
paging: Paging;
}>(testSuiteUrl, {
params,
});
export const updateTestCaseById = async (id: string, data: Operation[]) => {
const configOptions = {
headers: { 'Content-type': 'application/json-patch+json' },
};
return response.data;
};
export const createTestSuites = async (data: CreateTestSuite) => {
const response = await APIClient.post<TestSuite>(testSuiteUrl, data);
const response = await APIClient.patch<Operation[], AxiosResponse<TestSuite>>(
`${testCaseUrl}/${id}`,
data,
configOptions
);
return response.data;
};
@ -114,3 +112,60 @@ export const getListTestDefinitions = async (
return response.data;
};
export const getTestDefinitionById = async (
id: string,
params?: Pick<ListParams, 'fields' | 'include'>
) => {
const response = await APIClient.get<TestDefinition>(
`${testDefinitionUrl}/${id}`,
{
params,
}
);
return response.data;
};
// testSuite Section
export const getListTestSuites = async (params?: ListParams) => {
const response = await APIClient.get<{
data: TestSuite[];
paging: Paging;
}>(testSuiteUrl, {
params,
});
return response.data;
};
export const createTestSuites = async (data: CreateTestSuite) => {
const response = await APIClient.post<TestSuite>(testSuiteUrl, data);
return response.data;
};
export const getTestSuiteByName = async (
name: string,
params?: ListTestCaseParams
) => {
const response = await APIClient.get<TestSuite>(
`${testSuiteUrl}/name/${name}`,
{ params }
);
return response.data;
};
export const updateTestSuiteById = async (id: string, data: Operation[]) => {
const configOptions = {
headers: { 'Content-type': 'application/json-patch+json' },
};
const response = await APIClient.patch<Operation[], AxiosResponse<TestSuite>>(
`${testSuiteUrl}/${id}`,
data,
configOptions
);
return response.data;
};

10
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.interface.ts
View File

@ -13,6 +13,7 @@
import { ReactNode } from 'react';
import { Table } from '../../generated/entity/data/table';
import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
import { TestCase } from '../../generated/tests/testCase';
import { TestDefinition } from '../../generated/tests/testDefinition';
import { TestSuite } from '../../generated/tests/testSuite';
@ -34,10 +35,12 @@ export interface TestCaseFormProps {
export interface TestSuiteIngestionProps {
testSuite: TestSuite;
ingestionPipeline?: IngestionPipeline;
onCancel: () => void;
}
export interface TestSuiteSchedulerProps {
initialData?: string;
onSubmit: (repeatFrequency: string) => void;
onCancel: () => void;
}
@ -59,3 +62,10 @@ export type SelectTestSuiteType = {
export interface ParameterFormProps {
definition: TestDefinition;
}
export interface EditTestCaseModalProps {
visible: boolean;
testCase: TestCase;
onCancel: () => void;
onUpdate?: () => void;
}

12
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTestV1.tsx
View File

@ -11,7 +11,7 @@
* limitations under the License.
*/
import { Col, Row } from 'antd';
import { Col, Row, Typography } from 'antd';
import { AxiosError } from 'axios';
import { isUndefined } from 'lodash';
import React, { useCallback, useEffect, useMemo, useState } from 'react';
@ -40,7 +40,7 @@ import {
getNameFromFQN,
getPartialNameFromTableFQN,
} from '../../utils/CommonUtils';
import { getSettingPath } from '../../utils/RouterUtils';
import { getTestSuitePath } from '../../utils/RouterUtils';
import { serviceTypeLogo } from '../../utils/ServiceUtils';
import { showErrorToast } from '../../utils/ToastUtils';
import SuccessScreen from '../common/success-screen/SuccessScreen';
@ -126,7 +126,7 @@ const AddDataQualityTestV1: React.FC<AddDataQualityTestProps> = ({ table }) => {
}, [table, entityTypeFQN, isColumnFqn]);
const handleViewTestSuiteClick = () => {
history.push(getSettingPath());
history.push(getTestSuitePath(testSuiteData?.fullyQualifiedName || ''));
};
const handleAirflowStatusCheck = (): Promise<void> => {
@ -284,9 +284,11 @@ const AddDataQualityTestV1: React.FC<AddDataQualityTestProps> = ({ table }) => {
) : (
<Row className="tw-form-container" gutter={[16, 16]}>
<Col span={24}>
<h6 className="tw-heading tw-text-base" data-testid="header">
<Typography.Paragraph
className="tw-heading tw-text-base"
data-testid="header">
{`Add ${isColumnFqn ? 'Column' : 'Table'} Test`}
</h6>
</Typography.Paragraph>
</Col>
<Col span={24}>
<IngestionStepper

222
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/EditTestCaseModal.tsx Normal file
View File

@ -0,0 +1,222 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Col, Form, FormProps, Input, Row, Typography } from 'antd';
import Modal from 'antd/lib/modal/Modal';
import { AxiosError } from 'axios';
import { compare } from 'fast-json-patch';
import { EditorContentRef } from 'Models';
import React, { useCallback, useEffect, useRef, useState } from 'react';
import { Controlled as CodeMirror } from 'react-codemirror2';
import {
getTestDefinitionById,
updateTestCaseById,
} from '../../axiosAPIs/testAPI';
import { codeMirrorOption } from '../../constants/profiler.constant';
import { TestCaseParameterValue } from '../../generated/tests/testCase';
import {
TestDataType,
TestDefinition,
} from '../../generated/tests/testDefinition';
import jsonData from '../../jsons/en';
import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
import RichTextEditor from '../common/rich-text-editor/RichTextEditor';
import Loader from '../Loader/Loader';
import { EditTestCaseModalProps } from './AddDataQualityTest.interface';
import ParameterForm from './components/ParameterForm';
const EditTestCaseModal: React.FC<EditTestCaseModalProps> = ({
visible,
testCase,
onCancel,
onUpdate,
}) => {
const [form] = Form.useForm();
const [selectedDefinition, setSelectedDefinition] =
useState<TestDefinition>();
const [sqlQuery, setSqlQuery] = useState(
testCase?.parameterValues?.[0] ?? {
name: 'sqlExpression',
value: '',
}
);
const [isLoading, setIsLoading] = useState(true);
const markdownRef = useRef<EditorContentRef>();
const GenerateParamsField = useCallback(() => {
if (selectedDefinition && selectedDefinition.parameterDefinition) {
const name = selectedDefinition.parameterDefinition[0].name;
if (name === 'sqlExpression') {
return (
<Row>
<Col data-testid="sql-editor-container" span={24}>
<Typography.Paragraph className="tw-mb-1.5">
Profile Sample Query
</Typography.Paragraph>
<CodeMirror
data-testid="sql-editor"
options={codeMirrorOption}
value={sqlQuery.value || ''}
onBeforeChange={(_Editor, _EditorChange, value) => {
setSqlQuery((pre) => ({ ...pre, value }));
}}
onChange={(_Editor, _EditorChange, value) => {
setSqlQuery((pre) => ({ ...pre, value }));
}}
/>
</Col>
</Row>
);
}
return <ParameterForm definition={selectedDefinition} />;
}
return;
}, [testCase, selectedDefinition, sqlQuery]);
const fetchTestDefinitionById = async () => {
setIsLoading(true);
try {
const definition = await getTestDefinitionById(
testCase.testDefinition.id || ''
);
setSelectedDefinition(definition);
} catch (error) {
showErrorToast(error as AxiosError);
} finally {
setIsLoading(false);
}
};
const createTestCaseObj = (value: {
testName: string;
params: Record<string, string | { [key: string]: string }[]>;
testTypeId: string;
}) => {
const paramsValue = selectedDefinition?.parameterDefinition?.[0];
const parameterValues =
paramsValue?.name === 'sqlExpression'
? [sqlQuery]
: Object.entries(value.params || {}).map(([key, value]) => ({
name: key,
value:
paramsValue?.dataType === TestDataType.Array
? // need to send array as string formate
JSON.stringify(
(value as { value: string }[]).map((data) => data.value)
)
: value,
}));
return {
parameterValues: parameterValues as TestCaseParameterValue[],
description: markdownRef.current?.getEditorContent(),
};
};
const handleFormSubmit: FormProps['onFinish'] = async (value) => {
const { parameterValues, description } = createTestCaseObj(value);
const updatedTestCase = { ...testCase, parameterValues, description };
const jsonPatch = compare(testCase, updatedTestCase);
if (jsonPatch.length) {
try {
await updateTestCaseById(testCase.id || '', jsonPatch);
onUpdate && onUpdate();
showSuccessToast(
jsonData['api-success-messages']['update-test-case-success']
);
onCancel();
} catch (error) {
showErrorToast(error as AxiosError);
} finally {
form.resetFields();
}
}
};
const getParamsValue = () => {
return testCase?.parameterValues?.reduce(
(acc, curr) => ({
...acc,
[curr.name || '']:
selectedDefinition?.parameterDefinition?.[0].dataType ===
TestDataType.Array
? (JSON.parse(curr.value || '[]') as string[]).map((val) => ({
value: val,
}))
: curr.value,
}),
{}
);
};
useEffect(() => {
if (testCase) {
fetchTestDefinitionById();
form.setFieldsValue({
name: testCase?.name,
testDefinition: testCase?.testDefinition?.name,
params: getParamsValue(),
});
}
}, [testCase]);
return (
<Modal
centered
destroyOnClose
afterClose={() => {
form.resetFields();
onCancel();
}}
title={`Edit ${testCase?.name}`}
visible={visible}
onCancel={onCancel}
onOk={() => form.submit()}>
{isLoading ? (
<Loader />
) : (
<Form
form={form}
layout="vertical"
name="tableTestForm"
onFinish={handleFormSubmit}>
<Form.Item required label="Name:" name="name">
<Input disabled placeholder="Enter test case name" />
</Form.Item>
<Form.Item required label="Test Type:" name="testDefinition">
<Input disabled placeholder="Enter test case name" />
</Form.Item>
{GenerateParamsField()}
<Form.Item label="Description:" name="description">
<RichTextEditor
height="200px"
initialValue={testCase?.description || ''}
ref={markdownRef}
style={{
margin: 0,
}}
/>
</Form.Item>
</Form>
)}
</Modal>
);
};
export default EditTestCaseModal;

115
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/TestSuiteIngestion.tsx
View File

@ -10,14 +10,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Col, Row } from 'antd';
import { Col, Row, Typography } from 'antd';
import { AxiosError } from 'axios';
import { compare } from 'fast-json-patch';
import { camelCase } from 'lodash';
import React, { useMemo, useState } from 'react';
import { useHistory } from 'react-router-dom';
import { useHistory, useParams } from 'react-router-dom';
import {
addIngestionPipeline,
deployIngestionPipelineById,
patchIngestionPipeline,
} from '../../axiosAPIs/ingestionPipelineAPI';
import {
DEPLOYED_PROGRESS_VAL,
@ -32,7 +34,7 @@ import {
} from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
import jsonData from '../../jsons/en';
import { getSettingPath } from '../../utils/RouterUtils';
import { getTestSuitePath } from '../../utils/RouterUtils';
import { showErrorToast } from '../../utils/ToastUtils';
import SuccessScreen from '../common/success-screen/SuccessScreen';
import DeployIngestionLoaderModal from '../Modals/DeployIngestionLoaderModal/DeployIngestionLoaderModal';
@ -40,23 +42,31 @@ import { TestSuiteIngestionProps } from './AddDataQualityTest.interface';
import TestSuiteScheduler from './components/TestSuiteScheduler';
const TestSuiteIngestion: React.FC<TestSuiteIngestionProps> = ({
ingestionPipeline,
testSuite,
onCancel,
}) => {
const { ingestionFQN } = useParams<Record<string, string>>();
const history = useHistory();
const [ingestionData, setIngestionData] = useState<IngestionPipeline>();
const [ingestionData, setIngestionData] = useState<
IngestionPipeline | undefined
>(ingestionPipeline);
const [showDeployModal, setShowDeployModal] = useState(false);
const [showDeployButton, setShowDeployButton] = useState(false);
const [ingestionAction, setIngestionAction] = useState(
IngestionActionMessage.CREATING
ingestionFQN
? IngestionActionMessage.UPDATING
: IngestionActionMessage.CREATING
);
const [isIngestionDeployed, setIsIngestionDeployed] = useState(false);
const [isIngestionCreated, setIsIngestionCreated] = useState(false);
const [ingestionProgress, setIngestionProgress] = useState(0);
const getSuccessMessage = useMemo(() => {
const createMessage = showDeployButton
? 'has been created, but failed to deploy'
: 'has been created and deployed successfully';
? `has been ${ingestionFQN ? 'updated' : 'created'}, but failed to deploy`
: `has been ${
ingestionFQN ? 'updated' : 'created'
} and deployed successfully`;
return (
<span>
@ -71,27 +81,25 @@ const TestSuiteIngestion: React.FC<TestSuiteIngestionProps> = ({
const handleIngestionDeploy = (id?: string) => {
setShowDeployModal(true);
return new Promise<void>((resolve) => {
setIsIngestionCreated(true);
setIngestionProgress(INGESTION_PROGRESS_END_VAL);
setIngestionAction(IngestionActionMessage.DEPLOYING);
setIsIngestionCreated(true);
setIngestionProgress(INGESTION_PROGRESS_END_VAL);
setIngestionAction(IngestionActionMessage.DEPLOYING);
deployIngestionPipelineById(`${id || ingestionData?.id}`)
.then(() => {
setIsIngestionDeployed(true);
setShowDeployButton(false);
setIngestionProgress(DEPLOYED_PROGRESS_VAL);
setIngestionAction(IngestionActionMessage.DEPLOYED);
})
.catch((err: AxiosError) => {
setShowDeployButton(true);
setIngestionAction(IngestionActionMessage.DEPLOYING_ERROR);
showErrorToast(
err || jsonData['api-error-messages']['deploy-ingestion-error']
);
})
.finally(() => resolve());
});
deployIngestionPipelineById(`${id || ingestionData?.id}`)
.then(() => {
setIsIngestionDeployed(true);
setShowDeployButton(false);
setIngestionProgress(DEPLOYED_PROGRESS_VAL);
setIngestionAction(IngestionActionMessage.DEPLOYED);
})
.catch((err: AxiosError) => {
setShowDeployButton(true);
setIngestionAction(IngestionActionMessage.DEPLOYING_ERROR);
showErrorToast(
err || jsonData['api-error-messages']['deploy-ingestion-error']
);
})
.finally(() => setTimeout(() => setShowDeployModal(false), 500));
};
const createIngestionPipeline = async (repeatFrequency: string) => {
@ -115,28 +123,62 @@ const TestSuiteIngestion: React.FC<TestSuiteIngestionProps> = ({
const ingestion = await addIngestionPipeline(ingestionPayload);
setIngestionData(ingestion);
handleIngestionDeploy(ingestion.id).finally(() =>
setShowDeployModal(false)
handleIngestionDeploy(ingestion.id);
};
const updateIngestionPipeline = async (repeatFrequency: string) => {
const updatedPipeline = {
...ingestionPipeline,
airflowConfig: {
...ingestionPipeline?.airflowConfig,
scheduleInterval: repeatFrequency,
},
};
const jsonPatch = compare(
ingestionPipeline as IngestionPipeline,
updatedPipeline
);
if (jsonPatch.length) {
try {
const response = await patchIngestionPipeline(
ingestionPipeline?.id || '',
jsonPatch
);
handleIngestionDeploy(response.id);
} catch (error) {
showErrorToast(
error as AxiosError,
jsonData['api-error-messages']['update-ingestion-error']
);
}
}
};
const handleIngestionSubmit = (repeatFrequency: string) => {
if (ingestionFQN) {
updateIngestionPipeline(repeatFrequency);
} else {
createIngestionPipeline(repeatFrequency);
}
};
const handleViewTestSuiteClick = () => {
history.push(getSettingPath());
history.push(getTestSuitePath(testSuite?.fullyQualifiedName || ''));
};
const handleDeployClick = () => {
setShowDeployModal(true);
handleIngestionDeploy?.().finally(() => {
setTimeout(() => setShowDeployModal(false), 500);
});
handleIngestionDeploy();
};
return (
<Row className="tw-form-container" gutter={[16, 16]}>
<Col span={24}>
<h6 className="tw-heading tw-text-base" data-testid="header">
<Typography.Paragraph
className="tw-heading tw-text-base"
data-testid="header">
Schedule Ingestion
</h6>
</Typography.Paragraph>
</Col>
<Col span={24}>
@ -154,8 +196,9 @@ const TestSuiteIngestion: React.FC<TestSuiteIngestionProps> = ({
/>
) : (
<TestSuiteScheduler
initialData={ingestionPipeline?.airflowConfig.scheduleInterval}
onCancel={onCancel}
onSubmit={createIngestionPipeline}
onSubmit={handleIngestionSubmit}
/>
)}
</Col>

6
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/RightPanel.tsx
View File

@ -18,9 +18,11 @@ import { RightPanelProps } from '../AddDataQualityTest.interface';
const RightPanel: React.FC<RightPanelProps> = ({ data }) => {
return (
<Row>
<h6 className="tw-heading tw-text-base" data-testid="right-panel-header">
<Typography.Paragraph
className="tw-heading tw-text-base"
data-testid="right-panel-header">
{data.title}
</h6>
</Typography.Paragraph>
<Typography.Paragraph>{data.body}</Typography.Paragraph>
</Row>
);

39
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/TestCaseForm.tsx
View File

@ -89,12 +89,14 @@ const TestCaseForm: React.FC<TestCaseFormProps> = ({
}
};
const GenerateParamsField = useCallback(() => {
const getSelectedTestDefinition = () => {
const testType = initialValue?.testDefinition?.id ?? selectedTestType;
const selectedDefinition = testDefinitions.find(
(definition) => definition.id === testType
);
return testDefinitions.find((definition) => definition.id === testType);
};
const GenerateParamsField = useCallback(() => {
const selectedDefinition = getSelectedTestDefinition();
if (selectedDefinition && selectedDefinition.parameterDefinition) {
const name = selectedDefinition.parameterDefinition[0].name;
if (name === 'sqlExpression') {
@ -130,10 +132,7 @@ const TestCaseForm: React.FC<TestCaseFormProps> = ({
params: Record<string, string | { [key: string]: string }[]>;
testTypeId: string;
}) => {
const testType = initialValue?.testDefinition?.id ?? selectedTestType;
const selectedDefinition = testDefinitions.find(
(definition) => definition.id === testType
);
const selectedDefinition = getSelectedTestDefinition();
const paramsValue = selectedDefinition?.parameterDefinition?.[0];
const parameterValues =
@ -172,6 +171,22 @@ const TestCaseForm: React.FC<TestCaseFormProps> = ({
onCancel(createTestCaseObj(data));
};
const getParamsValue = () => {
return initialValue?.parameterValues?.reduce(
(acc, curr) => ({
...acc,
[curr.name || '']:
getSelectedTestDefinition()?.parameterDefinition?.[0].dataType ===
TestDataType.Array
? (JSON.parse(curr.value || '[]') as string[]).map((val) => ({
value: val,
}))
: curr.value,
}),
{}
);
};
useEffect(() => {
if (testDefinitions.length === 0) {
fetchAllTestDefinitions();
@ -187,13 +202,7 @@ const TestCaseForm: React.FC<TestCaseFormProps> = ({
initialValues={{
testName: initialValue?.name,
testTypeId: initialValue?.testDefinition?.id,
params: initialValue?.parameterValues?.reduce(
(acc, curr) => ({
...acc,
[curr.name || '']: curr.value,
}),
{}
),
params: getParamsValue(),
}}
layout="vertical"
name="tableTestForm"

17
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/components/TestSuiteScheduler.tsx
View File

@ -12,15 +12,24 @@
*/
import { Button, Col, Row, Space } from 'antd';
import React, { useState } from 'react';
import React, { useEffect, useState } from 'react';
import CronEditor from '../../common/CronEditor/CronEditor';
import { TestSuiteSchedulerProps } from '../AddDataQualityTest.interface';
const TestSuiteScheduler: React.FC<TestSuiteSchedulerProps> = ({
initialData,
onCancel,
onSubmit,
}) => {
const [repeatFrequency, setRepeatFrequency] = useState('');
const [repeatFrequency, setRepeatFrequency] = useState<string | undefined>(
initialData
);
useEffect(() => {
if (initialData) {
setRepeatFrequency(initialData);
}
}, [initialData]);
return (
<Row gutter={[16, 32]}>
@ -33,7 +42,9 @@ const TestSuiteScheduler: React.FC<TestSuiteSchedulerProps> = ({
<Col span={24}>
<Space className="tw-w-full tw-justify-end" size={16}>
<Button onClick={onCancel}>Back</Button>
<Button type="primary" onClick={() => onSubmit(repeatFrequency)}>
<Button
type="primary"
onClick={() => onSubmit(repeatFrequency || '')}>
Submit
</Button>
</Space>

5
openmetadata-ui/src/main/resources/ui/src/components/Modals/IngestionLogsModal/IngestionLogsModal.tsx
View File

@ -54,6 +54,7 @@ const IngestionLogsModal: FC<IngestionLogsModalProps> = ({
profiler_task?: string;
usage_task?: string;
lineage_task?: string;
test_suite_task?: string;
}>
) => {
switch (pipelineType) {
@ -72,6 +73,10 @@ const IngestionLogsModal: FC<IngestionLogsModalProps> = ({
case PipelineType.Lineage:
setLogs(gzipToStringConverter(res.data?.lineage_task || ''));
break;
case PipelineType.TestSuite:
setLogs(gzipToStringConverter(res.data?.test_suite_task || ''));
break;
default:

10
openmetadata-ui/src/main/resources/ui/src/components/MyAssetStats/MyAssetStats.component.tsx
View File

@ -72,6 +72,16 @@ const MyAssetStats: FunctionComponent<MyAssetStatsProps> = ({
link: getExplorePathWithSearch(undefined, 'mlmodels'),
dataTestId: 'mlmodels',
},
testSuite: {
icon: Icons.TABLE_GREY,
data: 'Test Suite',
count: entityCounts.testSuiteCount,
link: getSettingPath(
GlobalSettingsMenuCategory.DATA_QUALITY,
GlobalSettingOptions.TEST_SUITE
),
dataTestId: 'test-suite',
},
service: {
icon: Icons.SERVICE,
data: 'Services',

5
openmetadata-ui/src/main/resources/ui/src/components/MyAssetStats/MyAssetStats.test.tsx
View File

@ -39,6 +39,7 @@ const mockProp = {
servicesCount: 193,
userCount: 100,
teamCount: 7,
testSuiteCount: 1,
} as EntitiesCount,
};
@ -53,14 +54,14 @@ describe('Test MyDataHeader Component', () => {
expect(myDataHeader).toBeInTheDocument();
});
it('Should have 8 data summary details', () => {
it('Should have 9 data summary details', () => {
const { container } = render(<MyAssetStats {...mockProp} />, {
wrapper: MemoryRouter,
});
const dataSummary = getAllByTestId(container, /-summary$/);
expect(dataSummary.length).toBe(8);
expect(dataSummary.length).toBe(9);
});
it('OnClick it should redirect to respective page', () => {

6
openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/ProfilerDashboard.tsx
View File

@ -69,6 +69,7 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
testCases,
fetchProfilerData,
fetchTestCases,
onTestCaseUpdate,
profilerData,
onTableChange,
}) => {
@ -414,7 +415,10 @@ const ProfilerDashboard: React.FC<ProfilerDashboardProps> = ({
{activeTab === ProfilerDashboardTab.DATA_QUALITY && (
<Col span={24}>
<DataQualityTab testCases={getFilterTestCase()} />
<DataQualityTab
testCases={getFilterTestCase()}
onTestUpdate={onTestCaseUpdate}
/>
</Col>
)}
</Row>

71
openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/DataQualityTab.tsx
View File

@ -11,20 +11,27 @@
* limitations under the License.
*/
import { Button, Space, Table, Tooltip } from 'antd';
import { Button, Row, Space, Table, Tooltip } from 'antd';
import { ColumnsType } from 'antd/lib/table';
import { isUndefined } from 'lodash';
import moment from 'moment';
import React, { useMemo, useState } from 'react';
import { ReactComponent as ArrowDown } from '../../../assets/svg/arrow-down.svg';
import { ReactComponent as ArrowRight } from '../../../assets/svg/arrow-right.svg';
import { TestCase, TestCaseResult } from '../../../generated/tests/testCase';
import SVGIcons, { Icons } from '../../../utils/SvgUtils';
import { getTestResultBadgeIcon } from '../../../utils/TableUtils';
import EditTestCaseModal from '../../AddDataQualityTest/EditTestCaseModal';
import DeleteWidgetModal from '../../common/DeleteWidget/DeleteWidgetModal';
import { DataQualityTabProps } from '../profilerDashboard.interface';
import TestSummary from './TestSummary';
const DataQualityTab: React.FC<DataQualityTabProps> = ({ testCases }) => {
const DataQualityTab: React.FC<DataQualityTabProps> = ({
testCases,
onTestUpdate,
}) => {
const [selectedTestCase, setSelectedTestCase] = useState<TestCase>();
const [editTestCase, setEditTestCase] = useState<TestCase>();
const columns: ColumnsType<TestCase> = useMemo(() => {
return [
{
@ -67,24 +74,47 @@ const DataQualityTab: React.FC<DataQualityTabProps> = ({ testCases }) => {
title: 'Actions',
dataIndex: 'actions',
key: 'actions',
width: 100,
render: (_, record) => (
<Space>
<Row align="middle">
<Tooltip placement="bottom" title="Delete">
<Button
className="flex-center"
icon={
<SVGIcons
alt="Delete"
className="tw-w-4"
className="tw-h-4"
icon={Icons.DELETE}
/>
}
type="text"
onClick={() => {
onClick={(e) => {
// preventing expand/collapse on click of delete button
e.stopPropagation();
setSelectedTestCase(record);
}}
/>
</Tooltip>
</Space>
<Tooltip placement="bottom" title="Edit">
<Button
className="flex-center"
icon={
<SVGIcons
alt="edit"
className="tw-h-4"
icon={Icons.EDIT}
title="Edit"
/>
}
type="text"
onClick={(e) => {
// preventing expand/collapse on click of edit button
e.stopPropagation();
setEditTestCase(record);
}}
/>
</Tooltip>
</Row>
),
},
];
@ -96,12 +126,41 @@ const DataQualityTab: React.FC<DataQualityTabProps> = ({ testCases }) => {
columns={columns}
dataSource={testCases.map((test) => ({ ...test, key: test.name }))}
expandable={{
expandRowByClick: true,
rowExpandable: () => true,
expandedRowRender: (recode) => <TestSummary data={recode} />,
expandIcon: ({ expanded, onExpand, record }) =>
expanded ? (
<ArrowDown
className="mx-auto"
onClick={(e: React.MouseEvent) =>
onExpand(
record,
e as React.MouseEvent<HTMLElement, MouseEvent>
)
}
/>
) : (
<ArrowRight
className="mx-auto"
onClick={(e: React.MouseEvent) =>
onExpand(
record,
e as React.MouseEvent<HTMLElement, MouseEvent>
)
}
/>
),
}}
pagination={false}
size="small"
/>
<EditTestCaseModal
testCase={editTestCase as TestCase}
visible={!isUndefined(editTestCase)}
onCancel={() => setEditTestCase(undefined)}
onUpdate={onTestUpdate}
/>
<DeleteWidgetModal
entityId={selectedTestCase?.id || ''}

2
openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/profilerDashboard.interface.ts
View File

@ -25,6 +25,7 @@ export interface ProfilerDashboardProps {
profilerData: ColumnProfile[];
fetchProfilerData: (tableId: string, days?: number) => void;
fetchTestCases: (fqn: string) => void;
onTestCaseUpdate: () => void;
}
export type MetricChartType = {
@ -79,6 +80,7 @@ export interface ProfilerSummaryCardProps {
export interface DataQualityTabProps {
testCases: TestCase[];
onTestUpdate?: () => void;
}
export interface TestSummaryProps {

69
openmetadata-ui/src/main/resources/ui/src/components/TestCaseCommonTabContainer/TestCaseCommonTabContainer.component.tsx Normal file
View File

@ -0,0 +1,69 @@
import { Button, Col, Row } from 'antd';
import { isUndefined } from 'lodash';
import React from 'react';
import { PAGE_SIZE } from '../../constants/constants';
import { Paging } from '../../generated/type/paging';
import NextPrevious from '../common/next-previous/NextPrevious';
const TestCaseCommonTabContainer = ({
buttonName,
children,
paging,
currentPage,
testCasePageHandler,
onButtonClick,
showButton = true,
isPaging = false,
}: {
buttonName: string;
showButton?: boolean;
children?: JSX.Element;
paging?: Paging;
onButtonClick?: () => void;
currentPage?: number;
testCasePageHandler?: (
cursorValue: string | number,
activePage?: number | undefined
) => void;
isPaging?: boolean;
}) => {
const NextPreviousComponent = () => {
if (
isPaging &&
!isUndefined(paging) &&
paging?.total > PAGE_SIZE &&
!isUndefined(currentPage) &&
testCasePageHandler
) {
return (
<Col span={24}>
<NextPrevious
currentPage={currentPage}
pageSize={PAGE_SIZE}
paging={paging}
pagingHandler={testCasePageHandler}
totalCount={paging.total}
/>
</Col>
);
}
return null;
};
return (
<Row className="tw-mt-4" gutter={[16, 16]}>
{showButton && (
<Col className="tw-flex tw-justify-end" span={24}>
<Button type="primary" onClick={onButtonClick}>
{buttonName}
</Button>
</Col>
)}
{children}
<NextPreviousComponent />
</Row>
);
};
export default TestCaseCommonTabContainer;

58
openmetadata-ui/src/main/resources/ui/src/components/TestCasesTab/TestCasesTab.component.tsx Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Col } from 'antd';
import { orderBy } from 'lodash';
import React from 'react';
import { TestCase } from '../../generated/tests/testCase';
import { Paging } from '../../generated/type/paging';
import DataQualityTab from '../ProfilerDashboard/component/DataQualityTab';
import TestCaseCommonTabContainer from '../TestCaseCommonTabContainer/TestCaseCommonTabContainer.component';
const TestCasesTab = ({
testCases,
testCasesPaging,
currentPage,
onTestUpdate,
testCasePageHandler,
}: {
testCases: Array<TestCase>;
testCasesPaging: Paging;
currentPage: number;
onTestUpdate: () => void;
testCasePageHandler: (
cursorValue: string | number,
activePage?: number | undefined
) => void;
}) => {
const sortedTestCases = orderBy(testCases || [], ['name'], 'asc');
return (
<TestCaseCommonTabContainer
isPaging
buttonName="Add Test"
currentPage={currentPage}
paging={testCasesPaging}
showButton={false}
testCasePageHandler={testCasePageHandler}>
<Col span={24}>
<DataQualityTab
testCases={sortedTestCases}
onTestUpdate={onTestUpdate}
/>
</Col>
</TestCaseCommonTabContainer>
);
};
export default TestCasesTab;

81
openmetadata-ui/src/main/resources/ui/src/components/TestSuiteDetails/TestSuiteDetails.component.tsx Normal file
View File

@ -0,0 +1,81 @@
import { Space } from 'antd';
import React from 'react';
import { IcDeleteColored } from '../../utils/SvgUtils';
import { Button } from '../buttons/Button/Button';
import DeleteWidgetModal from '../common/DeleteWidget/DeleteWidgetModal';
import Description from '../common/description/Description';
import EntitySummaryDetails from '../common/EntitySummaryDetails/EntitySummaryDetails';
import TitleBreadcrumb from '../common/title-breadcrumb/title-breadcrumb.component';
import { TestSuiteDetailsProps } from './TestSuiteDetails.interfaces';
const TestSuiteDetails = ({
extraInfo,
slashedBreadCrumb,
handleDeleteWidgetVisible,
isDeleteWidgetVisible,
isDescriptionEditable,
testSuite,
handleUpdateOwner,
testSuiteDescription,
descriptionHandler,
handleDescriptionUpdate,
}: TestSuiteDetailsProps) => {
return (
<>
<Space
align="center"
className="tw-justify-between"
style={{ width: '100%' }}>
<TitleBreadcrumb
data-testid="test-suite-breadcrumb"
titleLinks={slashedBreadCrumb}
/>
<Button
data-testid="test-suite-delete"
size="small"
theme="primary"
variant="outlined"
onClick={() => handleDeleteWidgetVisible(true)}>
<IcDeleteColored
className="tw-mr-1.5"
height={14}
viewBox="0 0 24 24"
width={14}
/>
<span>Delete</span>
</Button>
<DeleteWidgetModal
allowSoftDelete
entityId={testSuite?.id}
entityName={testSuite?.fullyQualifiedName as string}
entityType="testSuite"
visible={isDeleteWidgetVisible}
onCancel={() => handleDeleteWidgetVisible(false)}
/>
</Space>
<div className="tw-flex tw-gap-1 tw-mb-2 tw-mt-1 tw-flex-wrap">
{extraInfo.map((info, index) => (
<span className="tw-flex" key={index}>
<EntitySummaryDetails data={info} updateOwner={handleUpdateOwner} />
</span>
))}
</div>
<Space>
<Description
hasEditAccess
className="test-suite-description"
description={testSuiteDescription || ''}
entityName={testSuite?.displayName ?? testSuite?.name}
isEdit={isDescriptionEditable}
onCancel={() => descriptionHandler(false)}
onDescriptionEdit={() => descriptionHandler(true)}
onDescriptionUpdate={handleDescriptionUpdate}
/>
</Space>
</>
);
};
export default TestSuiteDetails;

17
openmetadata-ui/src/main/resources/ui/src/components/TestSuiteDetails/TestSuiteDetails.interfaces.ts Normal file
View File

@ -0,0 +1,17 @@
import { ExtraInfo } from 'Models';
import { TestSuite } from '../../generated/tests/testSuite';
import { TitleBreadcrumbProps } from '../common/title-breadcrumb/title-breadcrumb.interface';
export interface TestSuiteDetailsProps {
extraInfo: ExtraInfo[];
slashedBreadCrumb: TitleBreadcrumbProps['titleLinks'];
handleDeleteWidgetVisible: (isVisible: boolean) => void;
isDeleteWidgetVisible: boolean;
isTagEditable?: boolean;
isDescriptionEditable: boolean;
testSuite: TestSuite | undefined;
handleUpdateOwner: (updatedOwner: TestSuite['owner']) => void;
testSuiteDescription: string | undefined;
descriptionHandler: (value: boolean) => void;
handleDescriptionUpdate: (updatedHTML: string) => void;
}

443
openmetadata-ui/src/main/resources/ui/src/components/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx Normal file
View File

@ -0,0 +1,443 @@
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import { Button, Col, Row, Table } from 'antd';
import { ColumnsType } from 'antd/lib/table';
import { AxiosError } from 'axios';
import cronstrue from 'cronstrue';
import React, { Fragment, useEffect, useMemo, useState } from 'react';
import { useHistory, useParams } from 'react-router-dom';
import {
deleteIngestionPipelineById,
deployIngestionPipelineById,
enableDisableIngestionPipelineById,
getIngestionPipelines,
triggerIngestionPipelineById,
} from '../../axiosAPIs/ingestionPipelineAPI';
import { fetchAirflowConfig } from '../../axiosAPIs/miscAPI';
import { TITLE_FOR_NON_ADMIN_ACTION } from '../../constants/constants';
import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
import jsonData from '../../jsons/en';
import { getIngestionStatuses } from '../../utils/CommonUtils';
import { getTestSuiteIngestionPath } from '../../utils/RouterUtils';
import SVGIcons, { Icons } from '../../utils/SvgUtils';
import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
import NonAdminAction from '../common/non-admin-action/NonAdminAction';
import PopOver from '../common/popover/PopOver';
import Loader from '../Loader/Loader';
import EntityDeleteModal from '../Modals/EntityDeleteModal/EntityDeleteModal';
import IngestionLogsModal from '../Modals/IngestionLogsModal/IngestionLogsModal';
import KillIngestionModal from '../Modals/KillIngestionPipelineModal/KillIngestionPipelineModal';
import TestCaseCommonTabContainer from '../TestCaseCommonTabContainer/TestCaseCommonTabContainer.component';
const TestSuitePipelineTab = () => {
const { testSuiteFQN } = useParams<Record<string, string>>();
const history = useHistory();
const [isLoading, setIsLoading] = useState(true);
const [testSuitePipelines, setTestSuitePipelines] = useState<
IngestionPipeline[]
>([]);
const [airFlowEndPoint, setAirFlowEndPoint] = useState('');
const [isLogsModalOpen, setIsLogsModalOpen] = useState(false);
const [selectedPipeline, setSelectedPipeline] = useState<IngestionPipeline>();
const [isKillModalOpen, setIsKillModalOpen] = useState<boolean>(false);
const [deleteSelection, setDeleteSelection] = useState({
id: '',
name: '',
state: '',
});
const [isConfirmationModalOpen, setIsConfirmationModalOpen] = useState(false);
const [currTriggerId, setCurrTriggerId] = useState({ id: '', state: '' });
const [currDeployId, setCurrDeployId] = useState({ id: '', state: '' });
const handleCancelConfirmationModal = () => {
setIsConfirmationModalOpen(false);
setDeleteSelection({
id: '',
name: '',
state: '',
});
};
const getAllIngestionWorkflows = async (paging?: string) => {
try {
setIsLoading(true);
const response = await getIngestionPipelines(
['owner', 'pipelineStatuses'],
testSuiteFQN,
paging
);
setTestSuitePipelines(response.data);
} catch (error) {
showErrorToast(error as AxiosError);
} finally {
setIsLoading(false);
}
};
const handleDelete = async (id: string, displayName: string) => {
setDeleteSelection({ id, name: displayName, state: 'waiting' });
try {
await deleteIngestionPipelineById(id);
setDeleteSelection({ id, name: displayName, state: 'success' });
getAllIngestionWorkflows();
} catch (error) {
showErrorToast(
error as AxiosError,
`${jsonData['api-error-messages']['delete-ingestion-error']} ${displayName}`
);
} finally {
handleCancelConfirmationModal();
}
};
const fetchAirFlowEndPoint = async () => {
try {
const response = await fetchAirflowConfig();
setAirFlowEndPoint(response.apiEndpoint);
} catch {
setAirFlowEndPoint('');
}
};
const handleEnableDisableIngestion = async (id: string) => {
try {
await enableDisableIngestionPipelineById(id);
getAllIngestionWorkflows();
} catch (error) {
showErrorToast(
error as AxiosError,
jsonData['api-error-messages']['unexpected-server-response']
);
}
};
const separator = (
<span className="tw-inline-block tw-text-gray-400 tw-self-center">|</span>
);
const confirmDelete = (id: string, name: string) => {
setDeleteSelection({
id,
name,
state: '',
});
setIsConfirmationModalOpen(true);
};
const handleTriggerIngestion = async (id: string, displayName: string) => {
setCurrTriggerId({ id, state: 'waiting' });
try {
await triggerIngestionPipelineById(id);
setCurrTriggerId({ id, state: 'success' });
setTimeout(() => {
setCurrTriggerId({ id: '', state: '' });
showSuccessToast('Pipeline triggered successfully');
}, 1500);
getAllIngestionWorkflows();
} catch (error) {
showErrorToast(
`${jsonData['api-error-messages']['triggering-ingestion-error']} ${displayName}`
);
setCurrTriggerId({ id: '', state: '' });
}
};
const handleDeployIngestion = async (id: string) => {
setCurrDeployId({ id, state: 'waiting' });
try {
await deployIngestionPipelineById(id);
setCurrDeployId({ id, state: 'success' });
setTimeout(() => setCurrDeployId({ id: '', state: '' }), 1500);
} catch (error) {
setCurrDeployId({ id: '', state: '' });
showErrorToast(
error as AxiosError,
jsonData['api-error-messages']['update-ingestion-error']
);
}
};
const getTriggerDeployButton = (ingestion: IngestionPipeline) => {
if (ingestion.deployed) {
return (
<Button
data-testid="run"
type="link"
onClick={() =>
handleTriggerIngestion(ingestion.id as string, ingestion.name)
}>
{currTriggerId.id === ingestion.id ? (
currTriggerId.state === 'success' ? (
<FontAwesomeIcon icon="check" />
) : (
<Loader size="small" type="default" />
)
) : (
'Run'
)}
</Button>
);
} else {
return (
<Button
data-testid="deploy"
type="link"
onClick={() => handleDeployIngestion(ingestion.id as string)}>
{currDeployId.id === ingestion.id ? (
currDeployId.state === 'success' ? (
<FontAwesomeIcon icon="check" />
) : (
<Loader size="small" type="default" />
)
) : (
'Deploy'
)}
</Button>
);
}
};
useEffect(() => {
getAllIngestionWorkflows();
fetchAirFlowEndPoint();
}, []);
const pipelineColumns = useMemo(() => {
const column: ColumnsType<IngestionPipeline> = [
{
title: 'Name',
dataIndex: 'name',
key: 'name',
render: (name: string) => {
return (
<>
<NonAdminAction
position="bottom"
title={TITLE_FOR_NON_ADMIN_ACTION}>
<a
className="link-text tw-mr-2"
data-testid="airflow-tree-view"
href={`${airFlowEndPoint}`}
rel="noopener noreferrer"
target="_blank">
{name}
<SVGIcons
alt="external-link"
className="tw-align-middle tw-ml-1"
icon={Icons.EXTERNAL_LINK}
width="16px"
/>
</a>
</NonAdminAction>
</>
);
},
},
{
title: 'Type',
dataIndex: 'pipelineType',
key: 'pipelineType',
},
{
title: 'Schedule',
dataIndex: 'airflowConfig',
key: 'airflowEndpoint',
render: (_, record) => {
return (
<>
{record?.airflowConfig.scheduleInterval ? (
<PopOver
html={
<div>
{cronstrue.toString(
record.airflowConfig.scheduleInterval || '',
{
use24HourTimeFormat: true,
verbose: true,
}
)}
</div>
}
position="bottom"
theme="light"
trigger="mouseenter">
<span>{record.airflowConfig.scheduleInterval ?? '--'}</span>
</PopOver>
) : (
<span>--</span>
)}
</>
);
},
},
{
title: 'Recent Runs',
dataIndex: 'pipelineStatuses',
key: 'recentRuns',
render: (_, record) => (
<Row align="middle">{getIngestionStatuses(record)}</Row>
),
},
{
title: 'Actions',
dataIndex: 'actions',
key: 'actions',
render: (_, record) => {
return (
<>
<NonAdminAction
position="bottom"
title={TITLE_FOR_NON_ADMIN_ACTION}>
<div className="tw-flex">
{record.enabled ? (
<Fragment>
{getTriggerDeployButton(record)}
{separator}
<Button
data-testid="pause"
type="link"
onClick={() =>
handleEnableDisableIngestion(record.id || '')
}>
Pause
</Button>
</Fragment>
) : (
<Button
data-testid="unpause"
type="link"
onClick={() =>
handleEnableDisableIngestion(record.id || '')
}>
Unpause
</Button>
)}
{separator}
<Button
data-testid="edit"
type="link"
onClick={() => {
history.push(
getTestSuiteIngestionPath(
testSuiteFQN,
record.fullyQualifiedName
)
);
}}>
Edit
</Button>
{separator}
<Button
data-testid="delete"
type="link"
onClick={() =>
confirmDelete(record.id as string, record.name)
}>
{deleteSelection.id === record.id ? (
deleteSelection.state === 'success' ? (
<FontAwesomeIcon icon="check" />
) : (
<Loader size="small" type="default" />
)
) : (
'Delete'
)}
</Button>
{separator}
<Button
data-testid="kill"
type="link"
onClick={() => {
setIsKillModalOpen(true);
setSelectedPipeline(record);
}}>
Kill
</Button>
{separator}
<Button
data-testid="logs"
type="link"
onClick={() => {
setIsLogsModalOpen(true);
setSelectedPipeline(record);
}}>
Logs
</Button>
</div>
</NonAdminAction>
{isLogsModalOpen &&
selectedPipeline &&
record.id === selectedPipeline?.id && (
<IngestionLogsModal
isModalOpen={isLogsModalOpen}
pipelinName={selectedPipeline.name}
pipelineId={selectedPipeline.id as string}
pipelineType={selectedPipeline.pipelineType}
onClose={() => {
setIsLogsModalOpen(false);
setSelectedPipeline(undefined);
}}
/>
)}
{isKillModalOpen &&
selectedPipeline &&
record.id === selectedPipeline?.id && (
<KillIngestionModal
isModalOpen={isKillModalOpen}
pipelinName={selectedPipeline.name}
pipelineId={selectedPipeline.id as string}
onClose={() => {
setIsKillModalOpen(false);
setSelectedPipeline(undefined);
}}
onIngestionWorkflowsUpdate={getAllIngestionWorkflows}
/>
)}
</>
);
},
},
];
return column;
}, [airFlowEndPoint, isKillModalOpen, isLogsModalOpen, selectedPipeline]);
if (isLoading) {
return <Loader />;
}
return (
<TestCaseCommonTabContainer
buttonName="Add Ingestion"
showButton={testSuitePipelines.length === 0}
onButtonClick={() => {
history.push(getTestSuiteIngestionPath(testSuiteFQN));
}}>
<Col span={24}>
<Table
columns={pipelineColumns}
dataSource={testSuitePipelines.map((test) => ({
...test,
key: test.name,
}))}
pagination={false}
size="small"
/>
{isConfirmationModalOpen && (
<EntityDeleteModal
entityName={deleteSelection.name}
entityType="ingestion"
loadingState={deleteSelection.state}
onCancel={handleCancelConfirmationModal}
onConfirm={() =>
handleDelete(deleteSelection.id, deleteSelection.name)
}
/>
)}
</Col>
</TestCaseCommonTabContainer>
);
};
export default TestSuitePipelineTab;

2
openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx
View File

@ -98,6 +98,8 @@ const DeleteWidgetModal = ({
return `glossaries`;
} else if (entityType === EntityType.POLICY) {
return 'policies';
} else if (entityType === EntityType.TEST_SUITE) {
return entityType;
} else {
return `${entityType}s`;
}

1
openmetadata-ui/src/main/resources/ui/src/components/common/description/Description.interface.ts
View File

@ -16,6 +16,7 @@ import { Table } from '../../../generated/entity/data/table';
import { ThreadType } from '../../../generated/entity/feed/thread';
export interface DescriptionProps {
className?: string;
entityName?: string;
owner?: Table['owner'];
hasEditAccess?: boolean;

5
openmetadata-ui/src/main/resources/ui/src/components/common/description/Description.tsx
View File

@ -32,6 +32,7 @@ import RichTextEditorPreviewer from '../rich-text-editor/RichTextEditorPreviewer
import { DescriptionProps } from './Description.interface';
const Description: FC<DescriptionProps> = ({
className,
hasEditAccess,
onDescriptionEdit,
description = '',
@ -181,8 +182,8 @@ const Description: FC<DescriptionProps> = ({
};
return (
<div className="schema-description tw-relative">
<div className="tw-px-3 tw-py-1 tw-flex">
<div className={`schema-description tw-relative ${className}`}>
<div className="tw-px-3 tw-py-1 tw-flex description-inner-main-container">
<div className="tw-relative">
<div
className="description tw-h-full tw-overflow-y-scroll tw-min-h-12 tw-relative tw-py-1"

6
openmetadata-ui/src/main/resources/ui/src/constants/constants.ts
View File

@ -94,6 +94,7 @@ export const PLACEHOLDER_USER_BOT = ':bot';
export const PLACEHOLDER_WEBHOOK_TYPE = ':webhookType';
export const PLACEHOLDER_RULE_NAME = ':ruleName';
export const PLACEHOLDER_DASHBOARD_TYPE = ':dashboardType';
export const PLACEHOLDER_TEST_SUITE_FQN = ':testSuiteFQN';
export const pagingObject = { after: '', before: '', total: 0 };
@ -236,6 +237,11 @@ export const ROUTES = {
ADD_POLICY: '/settings/access/policies/add-policy',
ADD_POLICY_RULE: `/settings/access/policies/${PLACEHOLDER_ROUTE_FQN}/add-rule`,
EDIT_POLICY_RULE: `/settings/access/policies/${PLACEHOLDER_ROUTE_FQN}/edit-rule/${PLACEHOLDER_RULE_NAME}`,
// test suites
TEST_SUITES: `/test-suites/${PLACEHOLDER_TEST_SUITE_FQN}`,
TEST_SUITES_ADD_INGESTION: `/test-suites/${PLACEHOLDER_TEST_SUITE_FQN}/add-ingestion`,
TEST_SUITES_EDIT_INGESTION: `/test-suites/${PLACEHOLDER_TEST_SUITE_FQN}/edit-ingestion/${PLACEHOLDER_ROUTE_INGESTION_FQN}`,
};
export const SOCKET_EVENTS = {

2
openmetadata-ui/src/main/resources/ui/src/constants/globalSettings.constants.tsx
View File

@ -26,6 +26,7 @@ export enum GlobalSettingsMenuCategory {
ACCESS = 'access',
SERVICES = 'services',
CUSTOM_ATTRIBUTES = 'customAttributes',
DATA_QUALITY = 'dataQuality',
INTEGRATIONS = 'integrations',
}
@ -45,6 +46,7 @@ export enum GlobalSettingOptions {
BOTS = 'bots',
TABLES = 'tables',
MSTEAMS = 'msteams',
TEST_SUITE = 'testSuite',
}
export const GLOBAL_SETTING_PERMISSION_RESOURCES = [

1
openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts
View File

@ -205,5 +205,4 @@ export const codeMirrorOption = {
export const STEPS_FOR_ADD_TEST_CASE: Array<StepperStepType> = [
{ name: 'Select/Add Test Suite', step: 1 },
{ name: 'Configure Test Case', step: 2 },
// { name: 'Schedule Interval', step: 3 },
];

1
openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts
View File

@ -34,6 +34,7 @@ export enum EntityType {
BOT = 'bot',
ROLE = 'role',
POLICY = 'policy',
TEST_SUITE = 'testSuite',
}
export enum AssetsType {

4
openmetadata-ui/src/main/resources/ui/src/jsons/en.ts
View File

@ -112,6 +112,8 @@ const jsonData = {
'fetch-table-profiler-config-error':
'Error while fetching table profiler config!',
'fetch-column-test-error': 'Error while fetching column test case!',
'fetch-test-suite-error': 'Error while fetching test suite',
'fetch-test-cases-error': 'Error while fetching test cases',
'test-connection-error': 'Error while testing connection!',
@ -144,6 +146,7 @@ const jsonData = {
'join-team-error': 'Error while joining the team!',
'leave-team-error': 'Error while leaving the team!',
'update-test-suite-error': 'Error while updating test suite',
},
'api-success-messages': {
'create-conversation': 'Conversation created successfully!',
@ -161,6 +164,7 @@ const jsonData = {
'user-restored-success': 'User restored successfully!',
'update-profile-congif-success': 'Profile config updated successfully!',
'update-test-case-success': 'Test case updated successfully!',
},
'form-error-messages': {
'empty-email': 'Email is required.',

5
openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx
View File

@ -83,6 +83,10 @@ const ProfilerDashboardPage = () => {
}
};
const handleTestCaseUpdate = () => {
fetchTestCases(generateEntityLink(entityTypeFQN));
};
const fetchTableEntity = async () => {
try {
const fqn = isColumnView
@ -155,6 +159,7 @@ const ProfilerDashboardPage = () => {
table={table}
testCases={testCases}
onTableChange={updateTableHandler}
onTestCaseUpdate={handleTestCaseUpdate}
/>
</PageContainerV1>
);

307
openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx Normal file
View File

@ -0,0 +1,307 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Col, Row } from 'antd';
import { AxiosError } from 'axios';
import { compare } from 'fast-json-patch';
import { camelCase, startCase } from 'lodash';
import { ExtraInfo } from 'Models';
import React, { useEffect, useMemo, useState } from 'react';
import { useParams } from 'react-router-dom';
import {
getListTestCase,
getTestSuiteByName,
ListTestCaseParams,
updateTestSuiteById,
} from '../../axiosAPIs/testAPI';
import TabsPane from '../../components/common/TabsPane/TabsPane';
import { TitleBreadcrumbProps } from '../../components/common/title-breadcrumb/title-breadcrumb.interface';
import PageContainer from '../../components/containers/PageContainer';
import Loader from '../../components/Loader/Loader';
import TestCasesTab from '../../components/TestCasesTab/TestCasesTab.component';
import TestSuiteDetails from '../../components/TestSuiteDetails/TestSuiteDetails.component';
import TestSuitePipelineTab from '../../components/TestSuitePipelineTab/TestSuitePipelineTab.component';
import {
getTeamAndUserDetailsPath,
INITIAL_PAGING_VALUE,
PAGE_SIZE,
pagingObject,
} from '../../constants/constants';
import {
GlobalSettingOptions,
GlobalSettingsMenuCategory,
} from '../../constants/globalSettings.constants';
import { OwnerType } from '../../enums/user.enum';
import { TestCase } from '../../generated/tests/testCase';
import { TestSuite } from '../../generated/tests/testSuite';
import { Paging } from '../../generated/type/paging';
import jsonData from '../../jsons/en';
import { getEntityName, getEntityPlaceHolder } from '../../utils/CommonUtils';
import { getSettingPath } from '../../utils/RouterUtils';
import { showErrorToast } from '../../utils/ToastUtils';
import './TestSuiteDetailsPage.styles.less';
const TestSuiteDetailsPage = () => {
const { testSuiteFQN } = useParams<Record<string, string>>();
const [testSuite, setTestSuite] = useState<TestSuite>();
const [isDescriptionEditable, setIsDescriptionEditable] = useState(false);
const [isDeleteWidgetVisible, setIsDeleteWidgetVisible] = useState(false);
const [isTestCaseLoaded, setIsTestCaseLoaded] = useState(false);
const [testCaseResult, setTestCaseResult] = useState<Array<TestCase>>([]);
const [currentPage, setCurrentPage] = useState(INITIAL_PAGING_VALUE);
const [testCasesPaging, setTestCasesPaging] = useState<Paging>(pagingObject);
const [slashedBreadCrumb, setSlashedBreadCrumb] = useState<
TitleBreadcrumbProps['titleLinks']
>([]);
const [activeTab, setActiveTab] = useState<number>(1);
const tabs = [
{
name: 'Test Cases',
isProtected: false,
position: 1,
},
{
name: 'Pipeline',
isProtected: false,
position: 2,
},
];
const { testSuiteDescription, testSuiteId, testOwner } = useMemo(() => {
return {
testOwner: testSuite?.owner,
testSuiteId: testSuite?.id,
testSuiteDescription: testSuite?.description,
};
}, [testSuite]);
const saveAndUpdateTestSuiteData = (updatedData: TestSuite) => {
const jsonPatch = compare(testSuite as TestSuite, updatedData);
return updateTestSuiteById(testSuiteId as string, jsonPatch);
};
const descriptionHandler = (value: boolean) => {
setIsDescriptionEditable(value);
};
const fetchTestCases = async (param?: ListTestCaseParams, limit?: number) => {
setIsTestCaseLoaded(false);
try {
const response = await getListTestCase({
fields: 'testCaseResult,testDefinition',
testSuiteId: testSuiteId,
limit: limit || PAGE_SIZE,
before: param && param.before,
after: param && param.after,
...param,
});
setTestCaseResult(response.data);
setTestCasesPaging(response.paging);
} catch {
setTestCaseResult([]);
showErrorToast(jsonData['api-error-messages']['fetch-test-cases-error']);
} finally {
setIsTestCaseLoaded(true);
}
};
const afterSubmitAction = () => {
fetchTestCases();
};
const fetchTestSuiteByName = async () => {
try {
const response = await getTestSuiteByName(testSuiteFQN, {
fields: 'owner',
});
setSlashedBreadCrumb([
{
name: 'Test Suites',
url: getSettingPath(
GlobalSettingsMenuCategory.DATA_QUALITY,
GlobalSettingOptions.TEST_SUITE
),
},
{
name: startCase(
camelCase(response?.fullyQualifiedName || response?.name)
),
url: '',
},
]);
setTestSuite(response);
fetchTestCases({ testSuiteId: response.id });
} catch (error) {
setTestSuite(undefined);
showErrorToast(
error as AxiosError,
jsonData['api-error-messages']['fetch-test-suite-error']
);
}
};
const onUpdateOwner = (updatedOwner: TestSuite['owner']) => {
if (updatedOwner) {
const updatedTestSuite = {
...testSuite,
owner: {
...testSuite?.owner,
...updatedOwner,
},
} as TestSuite;
saveAndUpdateTestSuiteData(updatedTestSuite)
.then((res) => {
if (res) {
setTestSuite(res);
} else {
showErrorToast(
jsonData['api-error-messages']['unexpected-server-response']
);
}
})
.catch((err: AxiosError) => {
showErrorToast(
err,
jsonData['api-error-messages']['update-owner-error']
);
});
}
};
const onDescriptionUpdate = (updatedHTML: string) => {
if (testSuite?.description !== updatedHTML) {
const updatedTestSuite = { ...testSuite, description: updatedHTML };
saveAndUpdateTestSuiteData(updatedTestSuite as TestSuite)
.then((res) => {
if (res) {
setTestSuite(res);
} else {
throw jsonData['api-error-messages']['unexpected-server-response'];
}
})
.catch((error: AxiosError) => {
showErrorToast(
error,
jsonData['api-error-messages']['update-test-suite-error']
);
})
.finally(() => {
descriptionHandler(false);
});
} else {
descriptionHandler(false);
}
};
const onSetActiveValue = (tabValue: number) => {
setActiveTab(tabValue);
};
const handleDescriptionUpdate = (updatedHTML: string) => {
onDescriptionUpdate(updatedHTML);
};
const handleDeleteWidgetVisible = (isVisible: boolean) => {
setIsDeleteWidgetVisible(isVisible);
};
const handleTestCasePaging = (
cursorValue: string | number,
activePage?: number | undefined
) => {
setCurrentPage(activePage as number);
fetchTestCases({
[cursorValue]: testCasesPaging[cursorValue as keyof Paging] as string,
});
};
const extraInfo: Array<ExtraInfo> = useMemo(
() => [
{
key: 'Owner',
value:
testOwner?.type === 'team'
? getTeamAndUserDetailsPath(testOwner?.name || '')
: getEntityName(testOwner) || '',
placeholderText:
getEntityPlaceHolder(
(testOwner?.displayName as string) || (testOwner?.name as string),
testOwner?.deleted
) || '',
isLink: testOwner?.type === 'team',
openInNewTab: false,
profileName:
testOwner?.type === OwnerType.USER ? testOwner?.name : undefined,
},
],
[testOwner]
);
useEffect(() => {
fetchTestSuiteByName();
}, []);
return (
<PageContainer>
<Row className="tw-px-6 tw-w-full">
<Col span={24}>
<TestSuiteDetails
descriptionHandler={descriptionHandler}
extraInfo={extraInfo}
handleDeleteWidgetVisible={handleDeleteWidgetVisible}
handleDescriptionUpdate={handleDescriptionUpdate}
handleUpdateOwner={onUpdateOwner}
isDeleteWidgetVisible={isDeleteWidgetVisible}
isDescriptionEditable={isDescriptionEditable}
slashedBreadCrumb={slashedBreadCrumb}
testSuite={testSuite}
testSuiteDescription={testSuiteDescription}
/>
</Col>
<Col className="tw-mt-8" span={24}>
<TabsPane
activeTab={activeTab}
setActiveTab={onSetActiveValue}
tabs={tabs}
/>
<div className="tw-mb-4">
{activeTab === 1 && (
<>
{isTestCaseLoaded ? (
<TestCasesTab
currentPage={currentPage}
testCasePageHandler={handleTestCasePaging}
testCases={testCaseResult}
testCasesPaging={testCasesPaging}
onTestUpdate={afterSubmitAction}
/>
) : (
<Loader />
)}
</>
)}
{activeTab === 2 && <TestSuitePipelineTab />}
</div>
</Col>
</Row>
</PageContainer>
);
};
export default TestSuiteDetailsPage;

10
openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.styles.less Normal file
View File

@ -0,0 +1,10 @@
.test-suite-description {
.description-inner-main-container {
padding-left: 0;
padding-right: 0;
}
.rich-text-editor-container {
padding-left: 0;
}
}

141
openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteIngestionPage/TestSuiteIngestionPage.tsx Normal file
View File

@ -0,0 +1,141 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Empty } from 'antd';
import { AxiosError } from 'axios';
import { isUndefined, startCase } from 'lodash';
import React, { useEffect, useState } from 'react';
import { useHistory, useParams } from 'react-router-dom';
import { getIngestionPipelineByFqn } from '../../axiosAPIs/ingestionPipelineAPI';
import { getTestSuiteByName } from '../../axiosAPIs/testAPI';
import RightPanel from '../../components/AddDataQualityTest/components/RightPanel';
import { INGESTION_DATA } from '../../components/AddDataQualityTest/rightPanelData';
import TestSuiteIngestion from '../../components/AddDataQualityTest/TestSuiteIngestion';
import TitleBreadcrumb from '../../components/common/title-breadcrumb/title-breadcrumb.component';
import { TitleBreadcrumbProps } from '../../components/common/title-breadcrumb/title-breadcrumb.interface';
import PageContainerV1 from '../../components/containers/PageContainerV1';
import PageLayout from '../../components/containers/PageLayout';
import Loader from '../../components/Loader/Loader';
import {
GlobalSettingOptions,
GlobalSettingsMenuCategory,
} from '../../constants/globalSettings.constants';
import { PageLayoutType } from '../../enums/layout.enum';
import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
import { TestSuite } from '../../generated/tests/testSuite';
import jsonData from '../../jsons/en';
import { getSettingPath, getTestSuitePath } from '../../utils/RouterUtils';
import { showErrorToast } from '../../utils/ToastUtils';
const TestSuiteIngestionPage = () => {
const { testSuiteFQN, ingestionFQN } = useParams<Record<string, string>>();
const history = useHistory();
const [isLoading, setIsLoading] = useState(true);
const [testSuite, setTestSuite] = useState<TestSuite>();
const [ingestionPipeline, setIngestionPipeline] =
useState<IngestionPipeline>();
const [slashedBreadCrumb, setSlashedBreadCrumb] = useState<
TitleBreadcrumbProps['titleLinks']
>([]);
const fetchIngestionByName = async () => {
setIsLoading(true);
try {
const response = await getIngestionPipelineByFqn(ingestionFQN);
setIngestionPipeline(response);
} catch (error) {
showErrorToast(
error as AxiosError,
jsonData['api-error-messages']['fetch-ingestion-error']
);
} finally {
setIsLoading(false);
}
};
const fetchTestSuiteByName = async () => {
setIsLoading(true);
try {
const response = await getTestSuiteByName(testSuiteFQN, {
fields: 'owner',
});
setSlashedBreadCrumb([
{
name: 'Test Suites',
url: getSettingPath(
GlobalSettingsMenuCategory.DATA_QUALITY,
GlobalSettingOptions.TEST_SUITE
),
},
{
name: startCase(response.displayName || response.name),
url: getTestSuitePath(response.fullyQualifiedName || ''),
},
{
name: `${ingestionFQN ? 'Edit' : 'Add'} Ingestion`,
url: '',
},
]);
setTestSuite(response);
if (ingestionFQN) {
await fetchIngestionByName();
}
} catch (error) {
setTestSuite(undefined);
showErrorToast(
error as AxiosError,
jsonData['api-error-messages']['fetch-test-suite-error']
);
} finally {
setIsLoading(false);
}
};
const handleCancelBtn = () => {
history.push(getTestSuitePath(testSuiteFQN || ''));
};
useEffect(() => {
fetchTestSuiteByName();
}, []);
if (isLoading) {
return <Loader />;
}
if (isUndefined(testSuite)) {
return <Empty description="No Data found" />;
}
return (
<PageContainerV1>
<div className="tw-self-center">
<PageLayout
classes="tw-max-w-full-hd tw-h-full tw-pt-4"
header={<TitleBreadcrumb titleLinks={slashedBreadCrumb} />}
layout={PageLayoutType['2ColRTL']}
rightPanel={<RightPanel data={INGESTION_DATA} />}>
<TestSuiteIngestion
ingestionPipeline={ingestionPipeline}
testSuite={testSuite}
onCancel={handleCancelBtn}
/>
</PageLayout>
</div>
</PageContainerV1>
);
};
export default TestSuiteIngestionPage;

134
openmetadata-ui/src/main/resources/ui/src/pages/TestSuitePage/TestSuitePage.tsx Normal file
View File

@ -0,0 +1,134 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Col, Row, Table, Typography } from 'antd';
import { ColumnsType } from 'antd/lib/table';
import React, { useEffect, useMemo, useState } from 'react';
import { Link } from 'react-router-dom';
import { getListTestSuites } from '../../axiosAPIs/testAPI';
import Ellipses from '../../components/common/Ellipses/Ellipses';
import NextPrevious from '../../components/common/next-previous/NextPrevious';
import {
INITIAL_PAGING_VALUE,
PAGE_SIZE,
pagingObject,
} from '../../constants/constants';
import { TestSuite } from '../../generated/tests/testSuite';
import { Paging } from '../../generated/type/paging';
import { getTestSuitePath } from '../../utils/RouterUtils';
const { Text } = Typography;
const TestSuitePage = () => {
const [testSuites, setTestSuites] = useState<Array<TestSuite>>([]);
const [isLoading, setIsLoading] = useState(false);
const [testSuitePage, setTestSuitePage] = useState(INITIAL_PAGING_VALUE);
const [testSuitePaging, setTestSuitePaging] = useState<Paging>(pagingObject);
const fetchTestSuites = async (param?: Record<string, string>) => {
try {
setIsLoading(true);
const response = await getListTestSuites({
fields: 'owner,tests',
limit: PAGE_SIZE,
before: param && param.before,
after: param && param.after,
});
setTestSuites(response.data);
setTestSuitePaging(response.paging);
} catch (err) {
setTestSuites([]);
setTestSuitePaging(pagingObject);
} finally {
setIsLoading(false);
}
};
const columns = useMemo(() => {
const col: ColumnsType<TestSuite> = [
{
title: 'Name',
dataIndex: 'name',
key: 'name',
render: (_, record) => (
<Link to={getTestSuitePath(record.name)}>{record.name}</Link>
),
},
{
title: 'Description',
dataIndex: 'description',
key: 'description',
width: 300,
render: (_, record) => (
<Ellipses tooltip className="tw-w-11/12">
{record.description}
</Ellipses>
),
},
{
title: 'No. of Test',
dataIndex: 'noOfTests',
key: 'noOfTests',
render: (_, record) => <Text>{record?.tests?.length} Tests</Text>,
},
{
title: 'Owner',
dataIndex: 'owner',
key: 'owner',
render: (_, record) => <span>{record?.owner?.displayName}</span>,
},
];
return col;
}, [testSuites]);
const testSuitePagingHandler = (
cursorValue: string | number,
activePage?: number
) => {
setTestSuitePage(activePage as number);
fetchTestSuites({
[cursorValue]: testSuitePaging[cursorValue as keyof Paging] as string,
});
};
useEffect(() => {
fetchTestSuites();
}, []);
return (
<Row className="tw-w-full">
<Col span={24}>
<Table
columns={columns}
dataSource={testSuites.map((test) => ({ ...test, key: test.name }))}
loading={isLoading}
pagination={false}
size="small"
/>
</Col>
{testSuites.length > PAGE_SIZE && (
<Col span={24}>
<NextPrevious
currentPage={testSuitePage}
pageSize={PAGE_SIZE}
paging={testSuitePaging}
pagingHandler={testSuitePagingHandler}
totalCount={testSuitePaging.total}
/>
</Col>
)}
</Row>
);
};
export default TestSuitePage;

31
openmetadata-ui/src/main/resources/ui/src/router/AuthenticatedAppRouter.tsx
View File

@ -18,7 +18,6 @@ import AppState from '../AppState';
import { usePermissionProvider } from '../components/PermissionProvider/PermissionProvider';
import { ResourceEntity } from '../components/PermissionProvider/PermissionProvider.interface';
import { ROUTES } from '../constants/constants';
import AddDataQualityTestPage from '../pages/AddDataQualityTestPage/AddDataQualityTestPage';
import { Operation } from '../generated/entity/policies/policy';
import { checkPermission } from '../utils/PermissionsUtils';
import AdminProtectedRoute from './AdminProtectedRoute';
@ -34,6 +33,24 @@ const ProfilerDashboardPage = withSuspenseFallback(
)
);
const TestSuiteIngestionPage = withSuspenseFallback(
React.lazy(
() => import('../pages/TestSuiteIngestionPage/TestSuiteIngestionPage')
)
);
const TestSuiteDetailsPage = withSuspenseFallback(
React.lazy(
() => import('../pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component')
)
);
const AddDataQualityTestPage = withSuspenseFallback(
React.lazy(
() => import('../pages/AddDataQualityTestPage/AddDataQualityTestPage')
)
);
const AddCustomProperty = withSuspenseFallback(
React.lazy(
() =>
@ -466,7 +483,17 @@ const AuthenticatedAppRouter: FunctionComponent = () => {
component={GlobalSettingPage}
path={ROUTES.SETTINGS_WITH_TAB_FQN}
/>
<Route exact component={TestSuiteDetailsPage} path={ROUTES.TEST_SUITES} />
<Route
exact
component={TestSuiteIngestionPage}
path={ROUTES.TEST_SUITES_ADD_INGESTION}
/>
<Route
exact
component={TestSuiteIngestionPage}
path={ROUTES.TEST_SUITES_EDIT_INGESTION}
/>
<Redirect to={ROUTES.NOT_FOUND} />
</Switch>
);

11
openmetadata-ui/src/main/resources/ui/src/router/GlobalSettingRouter.tsx
View File

@ -66,6 +66,9 @@ const SlackSettingsPage = withSuspenseFallback(
() => import('../pages/SlackSettingsPage/SlackSettingsPage.component')
)
);
const TestSuitePage = withSuspenseFallback(
React.lazy(() => import('../pages/TestSuitePage/TestSuitePage'))
);
const MsTeamsPage = withSuspenseFallback(
React.lazy(() => import('../pages/MsTeamsPage/MsTeamsPage.component'))
);
@ -100,6 +103,14 @@ const GlobalSettingRouter = () => {
true
)}
/>
<Route
exact
component={TestSuitePage}
path={getSettingPath(
GlobalSettingsMenuCategory.DATA_QUALITY,
GlobalSettingOptions.TEST_SUITE
)}
/>
{/* Roles route start
* Do not change the order of these route
*/}

25
openmetadata-ui/src/main/resources/ui/src/styles/app.less Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// common css utils file
.flex-center {
display: flex;
align-items: center;
justify-content: center;
}
.mx-auto {
margin-right: auto;
margin-left: auto;
}

1
openmetadata-ui/src/main/resources/ui/src/styles/index.js
View File

@ -14,6 +14,7 @@
import 'tailwindcss/tailwind.css';
import '../fonts/Inter/Inter-VariableFont_slnt,wght.ttf';
import './antd-master.less';
import './app.less';
import './components/glossary.less';
import './components/step.less';
import './fonts.css';

56
openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx
View File

@ -28,6 +28,7 @@ import { reactLocalStorage } from 'reactjs-localstorage';
import AppState from '../AppState';
import { getFeedCount } from '../axiosAPIs/feedsAPI';
import { Button } from '../components/buttons/Button/Button';
import PopOver from '../components/common/popover/PopOver';
import { FQN_SEPARATOR_CHAR } from '../constants/char.constants';
import {
imageTypes,
@ -49,6 +50,7 @@ import { Table } from '../generated/entity/data/table';
import { Topic } from '../generated/entity/data/topic';
import { ThreadTaskStatus, ThreadType } from '../generated/entity/feed/thread';
import { Policy } from '../generated/entity/policies/policy';
import { IngestionPipeline } from '../generated/entity/services/ingestionPipelines/ingestionPipeline';
import { Role } from '../generated/entity/teams/role';
import { Team } from '../generated/entity/teams/team';
import { EntityReference, User } from '../generated/entity/teams/user';
@ -823,3 +825,57 @@ export const getTeamsUser = (
return;
};
export const getIngestionStatuses = (ingestion: IngestionPipeline) => {
const lastFiveIngestions = ingestion.pipelineStatuses
?.sort((a, b) => {
// Turn your strings into millis, and then subtract them
// to get a value that is either negative, positive, or zero.
const date1 = new Date(a.startDate || '');
const date2 = new Date(b.startDate || '');
return date1.getTime() - date2.getTime();
})
.slice(Math.max(ingestion.pipelineStatuses.length - 5, 0));
return lastFiveIngestions?.map((r, i) => {
const status =
i === lastFiveIngestions.length - 1 ? (
<p
className={`tw-h-5 tw-w-16 tw-rounded-sm tw-bg-status-${r.state} tw-mr-1 tw-px-1 tw-text-white tw-text-center`}
key={i}>
{capitalize(r.state)}
</p>
) : (
<p
className={`tw-w-4 tw-h-5 tw-rounded-sm tw-bg-status-${r.state} tw-mr-1`}
key={i}
/>
);
return r?.endDate || r?.startDate || r?.timestamp ? (
<PopOver
html={
<div className="tw-text-left">
{r.timestamp ? (
<p>Execution Date: {new Date(r.timestamp).toUTCString()}</p>
) : null}
{r.startDate ? (
<p>Start Date: {new Date(r.startDate).toUTCString()}</p>
) : null}
{r.endDate ? (
<p>End Date: {new Date(r.endDate).toUTCString()}</p>
) : null}
</div>
}
key={i}
position="bottom"
theme="light"
trigger="mouseenter">
{status}
</PopOver>
) : (
status
);
});
};

14
openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsUtils.tsx
View File

@ -152,6 +152,20 @@ export const getGlobalSettingsMenuWithPermission = (
},
],
},
{
category: 'Data Quality',
items: [
{
label: 'Test Suite',
isProtected: checkPermission(
Operation.ViewAll,
ResourceEntity.TEST_SUITE,
permissions
),
icon: <TableIcon className="side-panel-icons" />,
},
],
},
{
category: 'Custom Attributes',
items: [

24
openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts
View File

@ -28,6 +28,7 @@ import {
PLACEHOLDER_RULE_NAME,
PLACEHOLDER_SETTING_CATEGORY,
PLACEHOLDER_TAG_NAME,
PLACEHOLDER_TEST_SUITE_FQN,
ROUTES,
} from '../constants/constants';
import { initialFilterQS } from '../constants/explore.constants';
@ -343,3 +344,26 @@ export const getAddDataQualityTableTestPath = (
return path;
};
export const getTestSuitePath = (testSuiteName: string) => {
let path = ROUTES.TEST_SUITES;
path = path.replace(PLACEHOLDER_TEST_SUITE_FQN, testSuiteName);
return path;
};
export const getTestSuiteIngestionPath = (
testSuiteName: string,
ingestionFQN?: string
) => {
let path = ingestionFQN
? ROUTES.TEST_SUITES_EDIT_INGESTION
: ROUTES.TEST_SUITES_ADD_INGESTION;
path = path.replace(PLACEHOLDER_TEST_SUITE_FQN, testSuiteName);
if (ingestionFQN) {
path = path.replace(PLACEHOLDER_ROUTE_INGESTION_FQN, ingestionFQN);
}
return path;
};