yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-03 03:59:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed issue-3325 Data Quality - columnValuesMissingCountToBeEqual UI, 3338: Add Ingestion Modal are missing Snowflake (#3351)

Browse Source 
* added support to accept multiple value for missingValueMatch in columnValuesMissingCountToBeEqual test
This commit is contained in:
Shailesh Parmar 2022-03-10 20:24:59 +05:30 committed by GitHub
parent 020613a6da
commit f03fb1cafa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 167 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx
View File

@ -95,8 +95,8 @@ const ColumnTestForm = ({
});
const [columnName, setColumnName] = useState(data?.columnName);
const [missingValueMatch, setMissingValueMatch] = useState<string>(
data?.testCase?.config?.missingValueMatch || ''
const [missingValueMatch, setMissingValueMatch] = useState<string[]>(
data?.testCase?.config?.missingValueMatch || ['']
);
const [missingCountValue, setMissingCountValue] = useState<
number | undefined
@ -123,6 +123,22 @@ const ColumnTestForm = ({
setIsShowError({ ...isShowError, values: false });
};
const addMatchFields = () => {
setMissingValueMatch([...missingValueMatch, '']);
};
const removeMatchFields = (i: number) => {
const newFormValues = [...missingValueMatch];
newFormValues.splice(i, 1);
setMissingValueMatch(newFormValues);
};
const handlMatchFieldsChange = (i: number, value: string) => {
const newFormValues = [...missingValueMatch];
newFormValues[i] = value;
setMissingValueMatch(newFormValues);
};
const setAllTestOption = (datatype: string) => {
const newTest = filteredColumnTestOption(datatype);
setTestTypeOptions(newTest);
@ -225,14 +241,18 @@ const ColumnTestForm = ({
maxValue: !isEmpty(maxValue) ? maxValue : undefined,
};
case ColumnTestType.columnValuesMissingCountToBeEqual:
case ColumnTestType.columnValuesMissingCountToBeEqual: {
const filterMatchValue = missingValueMatch.filter(
(value) => !isEmpty(value)
);
return {
missingCountValue: missingCountValue,
missingValueMatch: !isEmpty(missingValueMatch)
missingValueMatch: filterMatchValue.length
? missingValueMatch
: undefined,
};
}
case ColumnTestType.columnValuesToBeNotInSet:
return {
forbiddenValues: forbiddenValues.filter((v) => !isEmpty(v)),
@ -344,11 +364,6 @@ const ColumnTestForm = ({
break;
}
case 'missingValueMatch':
setMissingValueMatch(value);
break;
case 'missingCountValue':
setMissingCountValue(value as unknown as number);
errorMsg.missingCountValue = false;
@ -413,42 +428,67 @@ const ColumnTestForm = ({
const getMissingCountToBeEqualFields = () => {
return (
<Fragment>
<div className="tw-flex tw-gap-4 tw-w-full">
<div className="tw-flex-1">
<label
className="tw-block tw-form-label"
htmlFor="missingCountValue">
{requiredField('Count:')}
</label>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="missingCountValue"
id="missingCountValue"
name="missingCountValue"
placeholder="Missing count value"
type="number"
value={missingCountValue}
onChange={handleValidation}
/>
{isShowError.missingCountValue &&
errorMsg('Count value is required.')}
</div>
<div className="tw-flex-1">
<label
className="tw-block tw-form-label"
htmlFor="missingValueMatch">
Match:
</label>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="missingValueMatch"
id="missingValueMatch"
name="missingValueMatch"
placeholder="Missing value match"
value={missingValueMatch}
onChange={handleValidation}
/>
<Field>
<label className="tw-block tw-form-label" htmlFor="missingCountValue">
{requiredField('Count:')}
</label>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="missingCountValue"
id="missingCountValue"
name="missingCountValue"
placeholder="Missing count value"
type="number"
value={missingCountValue}
onChange={handleValidation}
/>
{isShowError.missingCountValue &&
errorMsg('Count value is required.')}
</Field>
<div data-testid="missing-count-to-be-equal">
<div className="tw-flex tw-items-center tw-mt-6">
<p className="w-form-label tw-mr-3">Match:</p>
<Button
className="tw-h-5 tw-px-2"
size="x-small"
theme="primary"
variant="contained"
onClick={addMatchFields}>
<i aria-hidden="true" className="fa fa-plus" />
</Button>
</div>
{missingValueMatch.map((value, i) => (
<div className="tw-flex tw-items-center" key={i}>
<div className="tw-w-11/12">
<Field>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
id={`value-key-${i}`}
name="key"
placeholder="Missing value to be match"
type="text"
value={value}
onChange={(e) => handlMatchFieldsChange(i, e.target.value)}
/>
</Field>
</div>
<button
className="focus:tw-outline-none tw-mt-3 tw-w-1/12"
onClick={(e) => {
e.preventDefault();
removeMatchFields(i);
}}>
<SVGIcons
alt="delete"
icon="icon-delete"
title="Delete"
width="12px"
/>
</button>
</div>
))}
</div>
</Fragment>
);
@ -552,10 +592,7 @@ const ColumnTestForm = ({
<p className="tw-font-medium tw-px-4">
{isUndefined(data) ? 'Add' : 'Edit'} Column Test
</p>
<form
className="tw-w-screen-sm"
data-testid="form"
onSubmit={(e) => e.preventDefault()}>
<div className="tw-w-screen-sm">
<div className="tw-px-4 tw-mx-auto">
<Field>
<label className="tw-block tw-form-label" htmlFor="columnName">
@ -663,7 +700,7 @@ const ColumnTestForm = ({
</Button>
</Field>
</Field>
</form>
</div>
</div>
);
};

7
openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.tsx
View File

@ -282,10 +282,7 @@ const TableTestForm = ({
{isUndefined(data) ? 'Add' : 'Edit'} Table Test
</p>
<form
className="tw-w-screen-sm"
data-testid="form"
onSubmit={(e) => e.preventDefault()}>
<div className="tw-w-screen-sm">
<div className="tw-px-4 tw-mx-auto">
<Field>
<label className="tw-block tw-form-label" htmlFor="tableTestType">
@ -369,7 +366,7 @@ const TableTestForm = ({
Save
</Button>
</Field>
</form>
</div>
</Fragment>
);
};

3
openmetadata-ui/src/main/resources/ui/src/components/DataQualityTab/DataQualityTab.tsx
View File

@ -36,6 +36,7 @@ type Props = {
tableTestCase: TableTest[];
handleRemoveTableTest: (testType: TableTestType) => void;
selectedColumn: string;
handleSelectedColumn: (value: string | undefined) => void;
handleRemoveColumnTest: (
columnName: string,
testType: ColumnTestType
@ -51,6 +52,7 @@ const DataQualityTab = ({
handleAddColumnTestCase,
handleRemoveTableTest,
handleRemoveColumnTest,
handleSelectedColumn,
testMode,
tableTestCase,
selectedColumn,
@ -61,6 +63,7 @@ const DataQualityTab = ({
const onFormCancel = () => {
handleShowTestForm(false);
setActiveData(undefined);
handleSelectedColumn(undefined);
};
const handleShowDropDown = (value: boolean) => {

2
openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx
View File

@ -116,6 +116,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
handleRemoveTableTest,
handleRemoveColumnTest,
qualityTestFormHandler,
handleSelectedColumn,
selectedColumn,
}: DatasetDetailsProps) => {
const { isAuthDisabled } = useAuthContext();
@ -643,6 +644,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
handleAddTableTestCase={handleAddTableTestCase}
handleRemoveColumnTest={handleRemoveColumnTest}
handleRemoveTableTest={handleRemoveTableTest}
handleSelectedColumn={handleSelectedColumn}
handleShowTestForm={handleShowTestForm}
handleTestModeChange={handleTestModeChange}
selectedColumn={selectedColumn}

1
openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.interface.ts
View File

@ -95,6 +95,7 @@ export interface DatasetDetailsProps {
columnsUpdateHandler: (updatedTable: Table) => void;
descriptionUpdateHandler: (updatedTable: Table) => void;
versionHandler: () => void;
handleSelectedColumn: (value: string | undefined) => void;
loadNodeHandler: (node: EntityReference, pos: LineagePos) => void;
addLineageHandler: (edge: Edge) => Promise<void>;
removeLineageHandler: (data: EdgeData) => void;

1
openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx
View File

@ -102,6 +102,7 @@ const DatasetDetailsProps = {
tableTestCase: [],
selectedColumn: '',
handleAddColumnTestCase: jest.fn(),
handleSelectedColumn: jest.fn(),
createThread: jest.fn(),
handleShowTestForm: jest.fn(),
handleRemoveTableTest: jest.fn(),

2
openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx
View File

@ -441,6 +441,7 @@ const Ingestion: React.FC<Props> = ({
name: s.name,
serviceType: s.serviceType,
}))}
serviceType={serviceType}
type=""
onCancel={() => setIsAdding(false)}
/>
@ -457,6 +458,7 @@ const Ingestion: React.FC<Props> = ({
name: s.name,
serviceType: s.serviceType,
}))}
serviceType={serviceType}
updateIngestion={(data, triggerIngestion) => {
setIsUpdating(false);
handleUpdateIngestion(data, triggerIngestion);

44
openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx
View File

@ -17,6 +17,7 @@ import { isEmpty } from 'lodash';
import { StepperStepType } from 'Models';
import { utc } from 'moment';
import React, { Fragment, ReactNode, useEffect, useState } from 'react';
import { DatabaseServiceType } from '../../generated/entity/services/databaseService';
import {
AirflowPipeline,
ConfigObject,
@ -94,6 +95,7 @@ const getIngestionName = (name: string) => {
const IngestionModal: React.FC<IngestionModalProps> = ({
isUpdating,
header,
serviceType,
service = '',
ingestionTypes,
ingestionList,
@ -150,6 +152,9 @@ const IngestionModal: React.FC<IngestionModalProps> = ({
selectedIngestion?.scheduleInterval || '5 * * * *'
);
const [warehouse, setWarehouse] = useState(pipelineConfig.warehouse);
const [account, setAccount] = useState(pipelineConfig.account);
const [showErrorMsg, setShowErrorMsg] = useState<ValidationErrorMsg>({
selectService: false,
name: false,
@ -284,6 +289,45 @@ const IngestionModal: React.FC<IngestionModalProps> = ({
errorMsg('Ingestion Type is required')}
</Field>
{serviceType === DatabaseServiceType.Snowflake && (
<div className="tw-grid tw-grid-cols-2 tw-gap-x-4 tw-mt-6">
<div>
<label className="tw-block" htmlFor="warehouse">
Warehouse:
</label>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="warehouse"
id="warehouse"
name="warehouse"
placeholder="Warehouse"
type="text"
value={warehouse}
onChange={(e) => {
setWarehouse(e.target.value);
}}
/>
</div>
<div>
<label className="tw-block" htmlFor="account">
Account:
</label>
<input
className="tw-form-inputs tw-px-3 tw-py-1"
data-testid="account"
id="account"
name="account"
placeholder="Account"
type="text"
value={account}
onChange={(e) => {
setAccount(e.target.value);
}}
/>
</div>
</div>
)}
<Field>
{getSeparator('Table Filter Pattern')}
<div className="tw-grid tw-grid-cols-2 tw-gap-x-4 tw-mt-1">

1
openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.interface.ts
View File

@ -35,6 +35,7 @@ export interface IngestionModalProps {
schedule?: string;
connectorConfig?: ConnectorConfig;
selectedIngestion?: AirflowPipeline;
serviceType: string;
addIngestion?: (data: AirflowPipeline, triggerIngestion?: boolean) => void;
updateIngestion?: (data: AirflowPipeline, triggerIngestion?: boolean) => void;
onCancel: () => void;

3
openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.test.tsx
View File

@ -29,6 +29,7 @@ const mockServiceList = [
serviceType: 'Redshift',
},
];
const mockServiceType = 'Redshift';
jest.mock('../IngestionStepper/IngestionStepper.component', () => {
return jest.fn().mockReturnValue(<p>IngestionStepper</p>);
@ -42,6 +43,7 @@ describe('Test Ingestion modal component', () => {
ingestionList={[]}
ingestionTypes={[]}
serviceList={mockServiceList}
serviceType={mockServiceType}
onCancel={mockFunction}
/>,
{
@ -81,6 +83,7 @@ describe('Test Ingestion modal component', () => {
ingestionTypes={['metadata', 'queryUsage'] as PipelineType[]}
service="bigquery_gcp"
serviceList={[]}
serviceType={mockServiceType}
onCancel={mockFunction}
/>,
{

1
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
View File

@ -206,6 +206,7 @@ export enum DatabaseServiceType {
Postgres = 'Postgres',
Presto = 'Presto',
Redshift = 'Redshift',
SQLite = 'SQLite',
SingleStore = 'SingleStore',
Snowflake = 'Snowflake',
Trino = 'Trino',

2
openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts
View File

@ -27,7 +27,7 @@ export interface TestCaseConfigType {
regex?: string;
forbiddenValues?: Array<number | string>;
missingCountValue?: number;
missingValueMatch?: string;
missingValueMatch?: Array<string>;
columnValuesToBeUnique?: boolean;
columnValuesToBeNotNull?: boolean;
}

6
openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx
View File

@ -176,6 +176,10 @@ const DatasetDetailsPage: FunctionComponent = () => {
setShowTestForm(value);
};
const handleSelectedColumn = (value: string | undefined) => {
setSelectedColumn(value);
};
const activeTabHandler = (tabValue: number) => {
const currentTabIndex = tabValue - 1;
if (datasetTableTabs[currentTabIndex].path !== tab) {
@ -686,6 +690,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
});
setColumns(updatedColumns);
handleShowTestForm(false);
setSelectedColumn(undefined);
showToast({
variant: 'success',
body: `Test ${data.testCase.columnTestType} for ${data.columnName} has been added.`,
@ -794,6 +799,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
handleAddTableTestCase={handleAddTableTestCase}
handleRemoveColumnTest={handleRemoveColumnTest}
handleRemoveTableTest={handleRemoveTableTest}
handleSelectedColumn={handleSelectedColumn}
handleShowTestForm={handleShowTestForm}
handleTestModeChange={handleTestModeChange}
isLineageLoading={isLineageLoading}

1
openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx
View File

@ -193,6 +193,7 @@ const TourPage = () => {
handleAddTableTestCase={handleCountChange}
handleRemoveColumnTest={handleCountChange}
handleRemoveTableTest={handleCountChange}
handleSelectedColumn={handleCountChange}
handleShowTestForm={handleCountChange}
handleTestModeChange={handleCountChange}
isNodeLoading={{

83
openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.ts
View File

@ -62,7 +62,7 @@ import {
VERTICA,
} from '../constants/services.const';
import { TabSpecificField } from '../enums/entity.enum';
import { IngestionType, ServiceCategory } from '../enums/service.enum';
import { ServiceCategory } from '../enums/service.enum';
import { DashboardServiceType } from '../generated/entity/services/dashboardService';
import { DatabaseServiceType } from '../generated/entity/services/databaseService';
import { MessagingServiceType } from '../generated/entity/services/messagingService';
@ -318,89 +318,30 @@ export const getTotalEntityCountByService = (buckets: Array<Bucket> = []) => {
return entityCounts;
};
export const getIngestionTypeList = (
serviceType: string,
onlyMetaData = false
): Array<string> | undefined => {
let ingestionType: Array<string> | undefined;
switch (serviceType) {
case DatabaseServiceType.BigQuery:
ingestionType = onlyMetaData
? [IngestionType.BIGQUERY]
: [IngestionType.BIGQUERY, IngestionType.BIGQUERY_USAGE];
break;
case DatabaseServiceType.Hive:
ingestionType = [IngestionType.HIVE];
break;
case DatabaseServiceType.Mssql:
ingestionType = [IngestionType.MSSQL];
break;
case DatabaseServiceType.MySQL:
ingestionType = [IngestionType.MYSQL];
break;
case DatabaseServiceType.Postgres:
ingestionType = [IngestionType.POSTGRES];
break;
case DatabaseServiceType.Redshift:
ingestionType = onlyMetaData
? [IngestionType.REDSHIFT]
: [IngestionType.REDSHIFT, IngestionType.REDSHIFT_USAGE];
break;
case DatabaseServiceType.Trino:
ingestionType = [IngestionType.TRINO];
break;
case DatabaseServiceType.Snowflake:
ingestionType = onlyMetaData
? [IngestionType.SNOWFLAKE]
: [IngestionType.SNOWFLAKE, IngestionType.SNOWFLAKE_USAGE];
break;
case DatabaseServiceType.Vertica:
ingestionType = [IngestionType.VERTICA];
break;
default:
break;
}
return ingestionType;
};
export const getAirflowPipelineTypes = (
serviceType: string,
onlyMetaData = false
): Array<PipelineType> | undefined => {
if (onlyMetaData) {
return [PipelineType.Metadata];
}
switch (serviceType) {
case DatabaseServiceType.Redshift:
case DatabaseServiceType.BigQuery:
case DatabaseServiceType.Snowflake:
return [PipelineType.Metadata, PipelineType.QueryUsage];
case DatabaseServiceType.Hive:
case DatabaseServiceType.ClickHouse:
case DatabaseServiceType.Mssql:
return onlyMetaData
? [PipelineType.Metadata]
: [PipelineType.Metadata, PipelineType.QueryUsage];
// need to add additional config feild to support trino
// case DatabaseServiceType.Trino:
case DatabaseServiceType.Hive:
case DatabaseServiceType.MySQL:
case DatabaseServiceType.Postgres:
case DatabaseServiceType.Trino:
case DatabaseServiceType.Vertica:
case DatabaseServiceType.MariaDB:
case DatabaseServiceType.SingleStore:
case DatabaseServiceType.SQLite:
case DatabaseServiceType.AzureSQL:
return [PipelineType.Metadata];
default: