Fixed issue-3325 Data Quality - columnValuesMissingCountToBeEqual UI, 3338: Add Ingestion Modal are missing Snowflake (#3351)

* added support to accept multiple value for missingValueMatch in columnValuesMissingCountToBeEqual test

openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx

@@ -95,8 +95,8 @@ const ColumnTestForm = ({
   });
 
   const [columnName, setColumnName] = useState(data?.columnName);
-  const [missingValueMatch, setMissingValueMatch] = useState<string>(
+  const [missingValueMatch, setMissingValueMatch] = useState<string[]>(
-    data?.testCase?.config?.missingValueMatch || ''
+    data?.testCase?.config?.missingValueMatch || ['']
   );
   const [missingCountValue, setMissingCountValue] = useState<
     number | undefined
@@ -123,6 +123,22 @@ const ColumnTestForm = ({
     setIsShowError({ ...isShowError, values: false });
   };
 
+  const addMatchFields = () => {
+    setMissingValueMatch([...missingValueMatch, '']);
+  };
+
+  const removeMatchFields = (i: number) => {
+    const newFormValues = [...missingValueMatch];
+    newFormValues.splice(i, 1);
+    setMissingValueMatch(newFormValues);
+  };
+
+  const handlMatchFieldsChange = (i: number, value: string) => {
+    const newFormValues = [...missingValueMatch];
+    newFormValues[i] = value;
+    setMissingValueMatch(newFormValues);
+  };
+
   const setAllTestOption = (datatype: string) => {
     const newTest = filteredColumnTestOption(datatype);
     setTestTypeOptions(newTest);
@@ -225,14 +241,18 @@ const ColumnTestForm = ({
           maxValue: !isEmpty(maxValue) ? maxValue : undefined,
         };
 
-      case ColumnTestType.columnValuesMissingCountToBeEqual:
+      case ColumnTestType.columnValuesMissingCountToBeEqual: {
+        const filterMatchValue = missingValueMatch.filter(
+          (value) => !isEmpty(value)
+        );
+
         return {
           missingCountValue: missingCountValue,
-          missingValueMatch: !isEmpty(missingValueMatch)
+          missingValueMatch: filterMatchValue.length
             ? missingValueMatch
             : undefined,
         };
-
+      }
       case ColumnTestType.columnValuesToBeNotInSet:
         return {
           forbiddenValues: forbiddenValues.filter((v) => !isEmpty(v)),
@@ -344,11 +364,6 @@ const ColumnTestForm = ({
         break;
       }
 
-      case 'missingValueMatch':
-        setMissingValueMatch(value);
-
-        break;
-
       case 'missingCountValue':
         setMissingCountValue(value as unknown as number);
         errorMsg.missingCountValue = false;
@@ -413,42 +428,67 @@ const ColumnTestForm = ({
   const getMissingCountToBeEqualFields = () => {
     return (
       <Fragment>
-        <div className="tw-flex tw-gap-4 tw-w-full">
+        <Field>
-          <div className="tw-flex-1">
+          <label className="tw-block tw-form-label" htmlFor="missingCountValue">
-            <label
+            {requiredField('Count:')}
-              className="tw-block tw-form-label"
+          </label>
-              htmlFor="missingCountValue">
+          <input
-              {requiredField('Count:')}
+            className="tw-form-inputs tw-px-3 tw-py-1"
-            </label>
+            data-testid="missingCountValue"
-            <input
+            id="missingCountValue"
-              className="tw-form-inputs tw-px-3 tw-py-1"
+            name="missingCountValue"
-              data-testid="missingCountValue"
+            placeholder="Missing count value"
-              id="missingCountValue"
+            type="number"
-              name="missingCountValue"
+            value={missingCountValue}
-              placeholder="Missing count value"
+            onChange={handleValidation}
-              type="number"
+          />
-              value={missingCountValue}
+          {isShowError.missingCountValue &&
-              onChange={handleValidation}
+            errorMsg('Count value is required.')}
-            />
+        </Field>
-            {isShowError.missingCountValue &&
+
-              errorMsg('Count value is required.')}
+        <div data-testid="missing-count-to-be-equal">
-          </div>
+          <div className="tw-flex tw-items-center tw-mt-6">
-          <div className="tw-flex-1">
+            <p className="w-form-label tw-mr-3">Match:</p>
-            <label
+            <Button
-              className="tw-block tw-form-label"
+              className="tw-h-5 tw-px-2"
-              htmlFor="missingValueMatch">
+              size="x-small"
-              Match:
+              theme="primary"
-            </label>
+              variant="contained"
-            <input
+              onClick={addMatchFields}>
-              className="tw-form-inputs tw-px-3 tw-py-1"
+              <i aria-hidden="true" className="fa fa-plus" />
-              data-testid="missingValueMatch"
+            </Button>
-              id="missingValueMatch"
-              name="missingValueMatch"
-              placeholder="Missing value match"
-              value={missingValueMatch}
-              onChange={handleValidation}
-            />
           </div>
+
+          {missingValueMatch.map((value, i) => (
+            <div className="tw-flex tw-items-center" key={i}>
+              <div className="tw-w-11/12">
+                <Field>
+                  <input
+                    className="tw-form-inputs tw-px-3 tw-py-1"
+                    id={`value-key-${i}`}
+                    name="key"
+                    placeholder="Missing value to be match"
+                    type="text"
+                    value={value}
+                    onChange={(e) => handlMatchFieldsChange(i, e.target.value)}
+                  />
+                </Field>
+              </div>
+              <button
+                className="focus:tw-outline-none tw-mt-3 tw-w-1/12"
+                onClick={(e) => {
+                  e.preventDefault();
+                  removeMatchFields(i);
+                }}>
+                <SVGIcons
+                  alt="delete"
+                  icon="icon-delete"
+                  title="Delete"
+                  width="12px"
+                />
+              </button>
+            </div>
+          ))}
         </div>
       </Fragment>
     );
@@ -552,10 +592,7 @@ const ColumnTestForm = ({
       <p className="tw-font-medium tw-px-4">
         {isUndefined(data) ? 'Add' : 'Edit'} Column Test
       </p>
-      <form
+      <div className="tw-w-screen-sm">
-        className="tw-w-screen-sm"
-        data-testid="form"
-        onSubmit={(e) => e.preventDefault()}>
         <div className="tw-px-4 tw-mx-auto">
           <Field>
             <label className="tw-block tw-form-label" htmlFor="columnName">
@@ -663,7 +700,7 @@ const ColumnTestForm = ({
             </Button>
           </Field>
         </Field>
-      </form>
+      </div>
     </div>
   );
 };

openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/TableTestForm.tsx

@@ -282,10 +282,7 @@ const TableTestForm = ({
         {isUndefined(data) ? 'Add' : 'Edit'} Table Test
       </p>
 
-      <form
+      <div className="tw-w-screen-sm">
-        className="tw-w-screen-sm"
-        data-testid="form"
-        onSubmit={(e) => e.preventDefault()}>
         <div className="tw-px-4 tw-mx-auto">
           <Field>
             <label className="tw-block tw-form-label" htmlFor="tableTestType">
@@ -369,7 +366,7 @@ const TableTestForm = ({
             Save
           </Button>
         </Field>
-      </form>
+      </div>
     </Fragment>
   );
 };

openmetadata-ui/src/main/resources/ui/src/components/DataQualityTab/DataQualityTab.tsx

@@ -36,6 +36,7 @@ type Props = {
   tableTestCase: TableTest[];
   handleRemoveTableTest: (testType: TableTestType) => void;
   selectedColumn: string;
+  handleSelectedColumn: (value: string | undefined) => void;
   handleRemoveColumnTest: (
     columnName: string,
     testType: ColumnTestType
@@ -51,6 +52,7 @@ const DataQualityTab = ({
   handleAddColumnTestCase,
   handleRemoveTableTest,
   handleRemoveColumnTest,
+  handleSelectedColumn,
   testMode,
   tableTestCase,
   selectedColumn,
@@ -61,6 +63,7 @@ const DataQualityTab = ({
   const onFormCancel = () => {
     handleShowTestForm(false);
     setActiveData(undefined);
+    handleSelectedColumn(undefined);
   };
 
   const handleShowDropDown = (value: boolean) => {

openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx

@@ -116,6 +116,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
   handleRemoveTableTest,
   handleRemoveColumnTest,
   qualityTestFormHandler,
+  handleSelectedColumn,
   selectedColumn,
 }: DatasetDetailsProps) => {
   const { isAuthDisabled } = useAuthContext();
@@ -643,6 +644,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
                 handleAddTableTestCase={handleAddTableTestCase}
                 handleRemoveColumnTest={handleRemoveColumnTest}
                 handleRemoveTableTest={handleRemoveTableTest}
+                handleSelectedColumn={handleSelectedColumn}
                 handleShowTestForm={handleShowTestForm}
                 handleTestModeChange={handleTestModeChange}
                 selectedColumn={selectedColumn}

openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.interface.ts

@@ -95,6 +95,7 @@ export interface DatasetDetailsProps {
   columnsUpdateHandler: (updatedTable: Table) => void;
   descriptionUpdateHandler: (updatedTable: Table) => void;
   versionHandler: () => void;
+  handleSelectedColumn: (value: string | undefined) => void;
   loadNodeHandler: (node: EntityReference, pos: LineagePos) => void;
   addLineageHandler: (edge: Edge) => Promise<void>;
   removeLineageHandler: (data: EdgeData) => void;

openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx

@@ -102,6 +102,7 @@ const DatasetDetailsProps = {
   tableTestCase: [],
   selectedColumn: '',
   handleAddColumnTestCase: jest.fn(),
+  handleSelectedColumn: jest.fn(),
   createThread: jest.fn(),
   handleShowTestForm: jest.fn(),
   handleRemoveTableTest: jest.fn(),

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx

@@ -441,6 +441,7 @@ const Ingestion: React.FC<Props> = ({
             name: s.name,
             serviceType: s.serviceType,
           }))}
+          serviceType={serviceType}
           type=""
           onCancel={() => setIsAdding(false)}
         />
@@ -457,6 +458,7 @@ const Ingestion: React.FC<Props> = ({
             name: s.name,
             serviceType: s.serviceType,
           }))}
+          serviceType={serviceType}
           updateIngestion={(data, triggerIngestion) => {
             setIsUpdating(false);
             handleUpdateIngestion(data, triggerIngestion);

openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx

@@ -17,6 +17,7 @@ import { isEmpty } from 'lodash';
 import { StepperStepType } from 'Models';
 import { utc } from 'moment';
 import React, { Fragment, ReactNode, useEffect, useState } from 'react';
+import { DatabaseServiceType } from '../../generated/entity/services/databaseService';
 import {
   AirflowPipeline,
   ConfigObject,
@@ -94,6 +95,7 @@ const getIngestionName = (name: string) => {
 const IngestionModal: React.FC<IngestionModalProps> = ({
   isUpdating,
   header,
+  serviceType,
   service = '',
   ingestionTypes,
   ingestionList,
@@ -150,6 +152,9 @@ const IngestionModal: React.FC<IngestionModalProps> = ({
     selectedIngestion?.scheduleInterval || '5 * * * *'
   );
 
+  const [warehouse, setWarehouse] = useState(pipelineConfig.warehouse);
+  const [account, setAccount] = useState(pipelineConfig.account);
+
   const [showErrorMsg, setShowErrorMsg] = useState<ValidationErrorMsg>({
     selectService: false,
     name: false,
@@ -284,6 +289,45 @@ const IngestionModal: React.FC<IngestionModalProps> = ({
                 errorMsg('Ingestion Type is required')}
             </Field>
 
+            {serviceType === DatabaseServiceType.Snowflake && (
+              <div className="tw-grid tw-grid-cols-2 tw-gap-x-4 tw-mt-6">
+                <div>
+                  <label className="tw-block" htmlFor="warehouse">
+                    Warehouse:
+                  </label>
+                  <input
+                    className="tw-form-inputs tw-px-3 tw-py-1"
+                    data-testid="warehouse"
+                    id="warehouse"
+                    name="warehouse"
+                    placeholder="Warehouse"
+                    type="text"
+                    value={warehouse}
+                    onChange={(e) => {
+                      setWarehouse(e.target.value);
+                    }}
+                  />
+                </div>
+                <div>
+                  <label className="tw-block" htmlFor="account">
+                    Account:
+                  </label>
+                  <input
+                    className="tw-form-inputs tw-px-3 tw-py-1"
+                    data-testid="account"
+                    id="account"
+                    name="account"
+                    placeholder="Account"
+                    type="text"
+                    value={account}
+                    onChange={(e) => {
+                      setAccount(e.target.value);
+                    }}
+                  />
+                </div>
+              </div>
+            )}
+
             <Field>
               {getSeparator('Table Filter Pattern')}
               <div className="tw-grid tw-grid-cols-2 tw-gap-x-4 tw-mt-1">

openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.interface.ts

@@ -35,6 +35,7 @@ export interface IngestionModalProps {
   schedule?: string;
   connectorConfig?: ConnectorConfig;
   selectedIngestion?: AirflowPipeline;
+  serviceType: string;
   addIngestion?: (data: AirflowPipeline, triggerIngestion?: boolean) => void;
   updateIngestion?: (data: AirflowPipeline, triggerIngestion?: boolean) => void;
   onCancel: () => void;

openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.test.tsx

@@ -29,6 +29,7 @@ const mockServiceList = [
     serviceType: 'Redshift',
   },
 ];
+const mockServiceType = 'Redshift';
 
 jest.mock('../IngestionStepper/IngestionStepper.component', () => {
   return jest.fn().mockReturnValue(<p>IngestionStepper</p>);
@@ -42,6 +43,7 @@ describe('Test Ingestion modal component', () => {
         ingestionList={[]}
         ingestionTypes={[]}
         serviceList={mockServiceList}
+        serviceType={mockServiceType}
         onCancel={mockFunction}
       />,
       {
@@ -81,6 +83,7 @@ describe('Test Ingestion modal component', () => {
         ingestionTypes={['metadata', 'queryUsage'] as PipelineType[]}
         service="bigquery_gcp"
         serviceList={[]}
+        serviceType={mockServiceType}
         onCancel={mockFunction}
       />,
       {

openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts

@@ -206,6 +206,7 @@ export enum DatabaseServiceType {
   Postgres = 'Postgres',
   Presto = 'Presto',
   Redshift = 'Redshift',
+  SQLite = 'SQLite',
   SingleStore = 'SingleStore',
   Snowflake = 'Snowflake',
   Trino = 'Trino',

openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts

@@ -27,7 +27,7 @@ export interface TestCaseConfigType {
   regex?: string;
   forbiddenValues?: Array<number | string>;
   missingCountValue?: number;
-  missingValueMatch?: string;
+  missingValueMatch?: Array<string>;
   columnValuesToBeUnique?: boolean;
   columnValuesToBeNotNull?: boolean;
 }

openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx

@@ -176,6 +176,10 @@ const DatasetDetailsPage: FunctionComponent = () => {
     setShowTestForm(value);
   };
 
+  const handleSelectedColumn = (value: string | undefined) => {
+    setSelectedColumn(value);
+  };
+
   const activeTabHandler = (tabValue: number) => {
     const currentTabIndex = tabValue - 1;
     if (datasetTableTabs[currentTabIndex].path !== tab) {
@@ -686,6 +690,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
         });
         setColumns(updatedColumns);
         handleShowTestForm(false);
+        setSelectedColumn(undefined);
         showToast({
           variant: 'success',
           body: `Test ${data.testCase.columnTestType} for ${data.columnName} has been added.`,
@@ -794,6 +799,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
           handleAddTableTestCase={handleAddTableTestCase}
           handleRemoveColumnTest={handleRemoveColumnTest}
           handleRemoveTableTest={handleRemoveTableTest}
+          handleSelectedColumn={handleSelectedColumn}
           handleShowTestForm={handleShowTestForm}
           handleTestModeChange={handleTestModeChange}
           isLineageLoading={isLineageLoading}

openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx

@@ -193,6 +193,7 @@ const TourPage = () => {
             handleAddTableTestCase={handleCountChange}
             handleRemoveColumnTest={handleCountChange}
             handleRemoveTableTest={handleCountChange}
+            handleSelectedColumn={handleCountChange}
             handleShowTestForm={handleCountChange}
             handleTestModeChange={handleCountChange}
             isNodeLoading={{

openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.ts

@@ -62,7 +62,7 @@ import {
   VERTICA,
 } from '../constants/services.const';
 import { TabSpecificField } from '../enums/entity.enum';
-import { IngestionType, ServiceCategory } from '../enums/service.enum';
+import { ServiceCategory } from '../enums/service.enum';
 import { DashboardServiceType } from '../generated/entity/services/dashboardService';
 import { DatabaseServiceType } from '../generated/entity/services/databaseService';
 import { MessagingServiceType } from '../generated/entity/services/messagingService';
@@ -318,89 +318,30 @@ export const getTotalEntityCountByService = (buckets: Array<Bucket> = []) => {
   return entityCounts;
 };
 
-export const getIngestionTypeList = (
-  serviceType: string,
-  onlyMetaData = false
-): Array<string> | undefined => {
-  let ingestionType: Array<string> | undefined;
-  switch (serviceType) {
-    case DatabaseServiceType.BigQuery:
-      ingestionType = onlyMetaData
-        ? [IngestionType.BIGQUERY]
-        : [IngestionType.BIGQUERY, IngestionType.BIGQUERY_USAGE];
-
-      break;
-    case DatabaseServiceType.Hive:
-      ingestionType = [IngestionType.HIVE];
-
-      break;
-
-    case DatabaseServiceType.Mssql:
-      ingestionType = [IngestionType.MSSQL];
-
-      break;
-
-    case DatabaseServiceType.MySQL:
-      ingestionType = [IngestionType.MYSQL];
-
-      break;
-
-    case DatabaseServiceType.Postgres:
-      ingestionType = [IngestionType.POSTGRES];
-
-      break;
-
-    case DatabaseServiceType.Redshift:
-      ingestionType = onlyMetaData
-        ? [IngestionType.REDSHIFT]
-        : [IngestionType.REDSHIFT, IngestionType.REDSHIFT_USAGE];
-
-      break;
-
-    case DatabaseServiceType.Trino:
-      ingestionType = [IngestionType.TRINO];
-
-      break;
-
-    case DatabaseServiceType.Snowflake:
-      ingestionType = onlyMetaData
-        ? [IngestionType.SNOWFLAKE]
-        : [IngestionType.SNOWFLAKE, IngestionType.SNOWFLAKE_USAGE];
-
-      break;
-
-    case DatabaseServiceType.Vertica:
-      ingestionType = [IngestionType.VERTICA];
-
-      break;
-
-    default:
-      break;
-  }
-
-  return ingestionType;
-};
-
 export const getAirflowPipelineTypes = (
   serviceType: string,
   onlyMetaData = false
 ): Array<PipelineType> | undefined => {
-  if (onlyMetaData) {
-    return [PipelineType.Metadata];
-  }
   switch (serviceType) {
     case DatabaseServiceType.Redshift:
     case DatabaseServiceType.BigQuery:
     case DatabaseServiceType.Snowflake:
-      return [PipelineType.Metadata, PipelineType.QueryUsage];
+    case DatabaseServiceType.ClickHouse:
-
-    case DatabaseServiceType.Hive:
     case DatabaseServiceType.Mssql:
+      return onlyMetaData
+        ? [PipelineType.Metadata]
+        : [PipelineType.Metadata, PipelineType.QueryUsage];
+
+    // need to add additional config feild to support trino
+    // case DatabaseServiceType.Trino:
+    case DatabaseServiceType.Hive:
     case DatabaseServiceType.MySQL:
     case DatabaseServiceType.Postgres:
-    case DatabaseServiceType.Trino:
     case DatabaseServiceType.Vertica:
     case DatabaseServiceType.MariaDB:
+    case DatabaseServiceType.SingleStore:
+    case DatabaseServiceType.SQLite:
+    case DatabaseServiceType.AzureSQL:
       return [PipelineType.Metadata];
 
     default: