Fix: Messaging and Dashboard ingestion from UI (#4890)

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx

@@ -171,7 +171,7 @@ const AddIngestion = ({
       true
   );
   const [enableDebugLog, setEnableDebugLog] = useState(
-    isUndefined(data?.loggerLevel) || data?.loggerLevel === LogLevels.Debug
+    data?.loggerLevel === LogLevels.Debug
   );
   const [dashboardFilterPattern, setDashboardFilterPattern] =
     useState<FilterPattern>(
@@ -351,7 +351,11 @@ const AddIngestion = ({
     setActiveIngestionStep(prevStep);
   };
 
-  const getFilterPatternData = (data: FilterPattern) => {
+  const getFilterPatternData = (data: FilterPattern, isVisible: boolean) => {
+    if (!isVisible) {
+      return undefined;
+    }
+
     const { includes, excludes } = data;
 
     const filterPattern =
@@ -366,6 +370,64 @@ const AddIngestion = ({
     return filterPattern;
   };
 
+  const getMetadataIngestionFields = () => {
+    switch (serviceCategory) {
+      case ServiceCategory.DATABASE_SERVICES: {
+        const DatabaseConfigData = {
+          ...(showDBTConfig
+            ? escapeBackwardSlashChar({ dbtConfigSource } as ConfigClass)
+            : undefined),
+        };
+
+        return {
+          enableDataProfiler: enableDataProfiler,
+          generateSampleData: ingestSampleData,
+          includeViews: includeView,
+          databaseFilterPattern: getFilterPatternData(
+            databaseFilterPattern,
+            showDatabaseFilter
+          ),
+          schemaFilterPattern: getFilterPatternData(
+            schemaFilterPattern,
+            showSchemaFilter
+          ),
+          tableFilterPattern: getFilterPatternData(
+            tableFilterPattern,
+            showTableFilter
+          ),
+          markDeletedTables,
+          ...DatabaseConfigData,
+          type: ConfigType.DatabaseMetadata,
+        };
+      }
+      case ServiceCategory.MESSAGING_SERVICES: {
+        return {
+          topicFilterPattern: getFilterPatternData(
+            topicFilterPattern,
+            showTopicFilter
+          ),
+          type: ConfigType.MessagingMetadata,
+        };
+      }
+      case ServiceCategory.DASHBOARD_SERVICES: {
+        return {
+          chartFilterPattern: getFilterPatternData(
+            chartFilterPattern,
+            showChartFilter
+          ),
+          dashboardFilterPattern: getFilterPatternData(
+            dashboardFilterPattern,
+            showDashboardFilter
+          ),
+          type: ConfigType.DashboardMetadata,
+        };
+      }
+      default: {
+        return {};
+      }
+    }
+  };
+
   const getConfigData = (type: PipelineType): ConfigClass => {
     switch (type) {
       case PipelineType.Usage: {
@@ -378,31 +440,16 @@ const AddIngestion = ({
       }
       case PipelineType.Profiler: {
         return {
-          fqnFilterPattern: getFilterPatternData(fqnFilterPattern),
+          fqnFilterPattern: getFilterPatternData(
+            fqnFilterPattern,
+            showFqnFilter
+          ),
           type: profilerIngestionType,
         };
       }
       case PipelineType.Metadata:
       default: {
-        const DatabaseConfigData = {
-          markDeletedTables: isDatabaseService ? markDeletedTables : undefined,
-          ...(showDBTConfig
-            ? escapeBackwardSlashChar({ dbtConfigSource } as ConfigClass)
-            : undefined),
-        };
-
-        return {
-          enableDataProfiler: enableDataProfiler,
-          generateSampleData: ingestSampleData,
-          includeViews: includeView,
-          databaseFilterPattern: getFilterPatternData(databaseFilterPattern),
-          schemaFilterPattern: getFilterPatternData(schemaFilterPattern),
-          tableFilterPattern: getFilterPatternData(tableFilterPattern),
-          chartFilterPattern: getFilterPatternData(chartFilterPattern),
-          dashboardFilterPattern: getFilterPatternData(dashboardFilterPattern),
-          topicFilterPattern: getFilterPatternData(topicFilterPattern),
-          ...DatabaseConfigData,
-        };
+        return getMetadataIngestionFields();
       }
     }
   };

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx

@@ -70,10 +70,26 @@ const ConfigureIngestion = ({
 }: ConfigureIngestionProps) => {
   const markdownRef = useRef<EditorContentRef>();
 
+  const getDebugLogToggle = () => {
+    return (
+      <Field>
+        <div className="tw-flex tw-gap-1">
+          <label>Enable Debug Log</label>
+          <ToggleSwitchV1
+            checked={enableDebugLog}
+            handleCheck={handleEnableDebugLog}
+            testId="enable-debug-log"
+          />
+        </div>
+        <p className="tw-text-grey-muted tw-mt-3">Enable debug logging</p>
+        {getSeparator('')}
+      </Field>
+    );
+  };
+
   const getDatabaseFieldToggles = () => {
     return (
       <>
-        {getSeparator('')}
         <div>
           <Field>
             <div className="tw-flex tw-gap-1">
@@ -118,18 +134,7 @@ const ConfigureIngestion = ({
             </p>
             {getSeparator('')}
           </Field>
-          <Field>
-            <div className="tw-flex tw-gap-1">
-              <label>Enable Debug Log</label>
-              <ToggleSwitchV1
-                checked={enableDebugLog}
-                handleCheck={handleEnableDebugLog}
-                testId="enable-debug-log"
-              />
-            </div>
-            <p className="tw-text-grey-muted tw-mt-3">Enable debug logging</p>
-            {getSeparator('')}
-          </Field>
+          {getDebugLogToggle()}
           {!isNil(markDeletedTables) && (
             <Field>
               <div className="tw-flex tw-gap-1">
@@ -195,6 +200,7 @@ const ConfigureIngestion = ({
               showSeparator={false}
               type={FilterPatternEnum.TABLE}
             />
+            {getSeparator('')}
             {getDatabaseFieldToggles()}
           </Fragment>
         );
@@ -224,23 +230,29 @@ const ConfigureIngestion = ({
               showSeparator={false}
               type={FilterPatternEnum.CHART}
             />
+            {getSeparator('')}
+            {getDebugLogToggle()}
           </Fragment>
         );
 
       case ServiceCategory.MESSAGING_SERVICES:
         return (
-          <FilterPattern
-            checked={showTopicFilter}
-            excludePattern={topicFilterPattern.excludes ?? []}
-            getExcludeValue={getExcludeValue}
-            getIncludeValue={getIncludeValue}
-            handleChecked={(value) =>
-              handleShowFilter(value, FilterPatternEnum.TOPIC)
-            }
-            includePattern={topicFilterPattern.includes ?? []}
-            showSeparator={false}
-            type={FilterPatternEnum.TOPIC}
-          />
+          <Fragment>
+            <FilterPattern
+              checked={showTopicFilter}
+              excludePattern={topicFilterPattern.excludes ?? []}
+              getExcludeValue={getExcludeValue}
+              getIncludeValue={getIncludeValue}
+              handleChecked={(value) =>
+                handleShowFilter(value, FilterPatternEnum.TOPIC)
+              }
+              includePattern={topicFilterPattern.includes ?? []}
+              showSeparator={false}
+              type={FilterPatternEnum.TOPIC}
+            />
+            {getSeparator('')}
+            {getDebugLogToggle()}
+          </Fragment>
         );
       default:
         return <></>;
@@ -337,6 +349,7 @@ const ConfigureIngestion = ({
           />
           {getSeparator('')}
         </Field>
+        {getDebugLogToggle()}
       </>
     );
   };
@@ -366,6 +379,7 @@ const ConfigureIngestion = ({
         </div>
         <div>{getProfilerFilterPatternField()}</div>
         {getSeparator('')}
+        {getDebugLogToggle()}
         <div>
           <Field>
             <label className="tw-block tw-form-label tw-mb-1" htmlFor="name">

openmetadata-ui/src/main/resources/ui/src/components/ServiceConfig/ConnectionConfigForm.tsx

@@ -37,7 +37,10 @@ import { getDatabaseConfig } from '../../utils/DatabaseServiceUtils';
 import { formatFormDataForSubmit } from '../../utils/JSONSchemaFormUtils';
 import { getMessagingConfig } from '../../utils/MessagingServiceUtils';
 import { getPipelineConfig } from '../../utils/PipelineServiceUtils';
-import { shouldTestConnection } from '../../utils/ServiceUtils';
+import {
+  getTestConnectionType,
+  shouldTestConnection,
+} from '../../utils/ServiceUtils';
 import { showErrorToast } from '../../utils/ToastUtils';
 import FormBuilder from '../common/FormBuilder/FormBuilder';
 
@@ -83,7 +86,7 @@ const ConnectionConfigForm: FunctionComponent<Props> = ({
     const updatedFormData = formatFormDataForSubmit(formData);
 
     return new Promise<void>((resolve, reject) => {
-      TestConnection(updatedFormData, 'Database')
+      TestConnection(updatedFormData, getTestConnectionType(serviceCategory))
         .then((res) => {
           // This api only responds with status 200 on success
           // No data sent on api success

openmetadata-ui/src/main/resources/ui/src/components/common/success-screen/SuccessScreen.tsx

@@ -46,7 +46,7 @@ const SuccessScreen = ({
           alt="success"
           className="tw-w-10 tw-h-10"
           data-testid="success-icon"
-          icon={Icons.CIRCLE_CHECKBOX}
+          icon={Icons.SUCCESS_BADGE}
         />
       </div>
       <p className="tw-mb-7" data-testid="success-line">

openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx

@@ -164,6 +164,7 @@ const EditIngestionPage = () => {
       airflowConfig,
       description,
       displayName,
+      loggerLevel,
       name,
       owner,
       pipelineType,
@@ -174,6 +175,7 @@ const EditIngestionPage = () => {
       airflowConfig,
       description,
       displayName,
+      loggerLevel,
       name,
       owner,
       pipelineType,

openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx

@@ -74,6 +74,7 @@ import {
   VERTICA,
 } from '../constants/services.const';
 import { ServiceCategory } from '../enums/service.enum';
+import { ConnectionType } from '../generated/api/services/ingestionPipelines/testServiceConnection';
 import { DashboardServiceType } from '../generated/entity/services/dashboardService';
 import { DatabaseServiceType } from '../generated/entity/services/databaseService';
 import { PipelineType as IngestionPipelineType } from '../generated/entity/services/ingestionPipelines/ingestionPipeline';
@@ -594,3 +595,15 @@ export const shouldTestConnection = (
     serviceCategory !== ServiceCategory.PIPELINE_SERVICES
   );
 };
+
+export const getTestConnectionType = (serviceCat: ServiceCategory) => {
+  switch (serviceCat) {
+    case ServiceCategory.MESSAGING_SERVICES:
+      return ConnectionType.Messaging;
+    case ServiceCategory.DASHBOARD_SERVICES:
+      return ConnectionType.Dashboard;
+    case ServiceCategory.DATABASE_SERVICES:
+    default:
+      return ConnectionType.Database;
+  }
+};