Fix #5116: Update UI connection for Pipeline Services (#5408)

catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json

@@ -22,7 +22,7 @@
     "connectionType": {
       "description": "Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres...",
       "type": "string",
-      "enum": ["Database", "Dashboard", "Messaging"],
+      "enum": ["Database", "Dashboard", "Messaging", "Pipeline"],
       "javaEnums": [
         {
           "name": "Database"
@@ -32,6 +32,9 @@
         },
         {
           "name": "Messaging"
+        },
+        {
+          "name": "Pipeline"
         }
       ]
     }

openmetadata-ui/src/main/resources/ui/parseConnectionSchema.js

@@ -13,7 +13,7 @@ const rootDir = 'connTemp';
 const srcDir = 'schema/entity/services/connections';
 const destDir = 'src/jsons/connectionSchemas/connections';
 
-const playDir = `${rootDir}/${srcDir}/${rootDir}`;
+const playDir = `${rootDir}/${srcDir}`;
 
 const globalParserOptions = {
   continueOnError: true,
@@ -24,8 +24,11 @@ const globalParserOptions = {
 
 async function parseSchema(filePath, destPath) {
   try {
+    const fileDir = `${cwd}/${path.dirname(filePath)}`;
+    const fileName = path.basename(filePath);
+    process.chdir(fileDir);
     const parser = new $RefParser(globalParserOptions);
-    const schema = await parser.parse(filePath);
+    const schema = await parser.parse(fileName);
     const api = await parser.bundle(schema);
     const dirname = `${cwd}/${path.dirname(destPath)}`;
     if (!fs.existsSync(dirname)) {
@@ -38,54 +41,44 @@ async function parseSchema(filePath, destPath) {
     fs.writeFileSync(`${cwd}/${destPath}`, JSON.stringify(api, null, 2));
   } catch (err) {
     console.log(err);
+  } finally {
+    process.chdir(cwd);
   }
 }
 
-function traverseDirectory(Directory) {
+async function traverseDirectory(Directory) {
-  fs.readdirSync(Directory).forEach((File) => {
+  const Files = fs.readdirSync(Directory);
+  for (const File of Files) {
     const Absolute = path.join(Directory, File);
     if (fs.statSync(Absolute).isDirectory()) {
-      return traverseDirectory(Absolute);
+      await traverseDirectory(Absolute);
     } else {
-      const name = Absolute.replace(srcDir, destDir);
+      const name = Absolute.replace(playDir, destDir);
-      return parseSchema(Absolute, name);
+      await parseSchema(Absolute, name);
+    }
   }
-  });
 }
 
 function copySourceFiles() {
   try {
     fse.copySync(schemaDir, `${rootDir}/schema`);
-    fse.copySync(schemaDir, `${playDir}/schema`);
   } catch (err) {
     console.error(err);
   }
 }
 
-function main() {
+async function main() {
   try {
     if (fs.existsSync(destDir)) {
       fs.rmSync(destDir, { recursive: true });
     }
     fs.mkdirSync(destDir, { recursive: true });
     copySourceFiles();
+
+    await traverseDirectory(`${playDir}`);
   } catch (err) {
     console.log(err);
   }
-
-  process.chdir(`${cwd}/${playDir}`);
-
-  fs.readdir(srcDir, (err, Files) => {
-    if (err) console.log(err);
-    else {
-      Files.forEach((File, index) => {
-        const Absolute = path.join(srcDir, File);
-        if (fs.statSync(Absolute).isDirectory()) {
-          traverseDirectory(Absolute);
-        }
-      });
-    }
-  });
 }
 
 main();

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx

@@ -122,6 +122,11 @@ const AddIngestion = ({
   const [showChartFilter, setShowChartFilter] = useState(
     !isUndefined((data?.sourceConfig.config as ConfigClass)?.chartFilterPattern)
   );
+  const [showPipelineFilter, setShowPipelineFilter] = useState(
+    !isUndefined(
+      (data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern
+    )
+  );
   const [showFqnFilter, setShowFqnFilter] = useState(
     !isUndefined((data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern)
   );
@@ -155,6 +160,9 @@ const AddIngestion = ({
   const [includeView, setIncludeView] = useState(
     Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews)
   );
+  const [includeLineage, setIncludeLineage] = useState(
+    Boolean((data?.sourceConfig.config as ConfigClass)?.includeLineage ?? true)
+  );
   const [enableDebugLog, setEnableDebugLog] = useState(
     data?.loggerLevel === LogLevels.Debug
   );
@@ -184,6 +192,11 @@ const AddIngestion = ({
     (data?.sourceConfig.config as ConfigClass)?.chartFilterPattern ??
       INITIAL_FILTER_PATTERN
   );
+  const [pipelineFilterPattern, setPipelineFilterPattern] =
+    useState<FilterPattern>(
+      (data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern ??
+        INITIAL_FILTER_PATTERN
+    );
   const [fqnFilterPattern, setFqnFilterPattern] = useState<FilterPattern>(
     (data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern ??
       INITIAL_FILTER_PATTERN
@@ -237,12 +250,16 @@ const AddIngestion = ({
 
         break;
       case FilterPatternEnum.CHART:
-        setChartFilterPattern({ ...topicFilterPattern, includes: value });
+        setChartFilterPattern({ ...chartFilterPattern, includes: value });
 
         break;
       case FilterPatternEnum.FQN:
         setFqnFilterPattern({ ...fqnFilterPattern, includes: value });
 
+        break;
+      case FilterPatternEnum.PIPELINE:
+        setPipelineFilterPattern({ ...pipelineFilterPattern, includes: value });
+
         break;
     }
   };
@@ -272,12 +289,16 @@ const AddIngestion = ({
 
         break;
       case FilterPatternEnum.CHART:
-        setChartFilterPattern({ ...topicFilterPattern, excludes: value });
+        setChartFilterPattern({ ...chartFilterPattern, excludes: value });
 
         break;
       case FilterPatternEnum.FQN:
         setFqnFilterPattern({ ...fqnFilterPattern, excludes: value });
 
+        break;
+      case FilterPatternEnum.PIPELINE:
+        setPipelineFilterPattern({ ...pipelineFilterPattern, excludes: value });
+
         break;
     }
   };
@@ -311,6 +332,10 @@ const AddIngestion = ({
       case FilterPatternEnum.FQN:
         setShowFqnFilter(value);
 
+        break;
+      case FilterPatternEnum.PIPELINE:
+        setShowPipelineFilter(value);
+
         break;
     }
   };
@@ -404,6 +429,16 @@ const AddIngestion = ({
           type: ConfigType.DashboardMetadata,
         };
       }
+      case ServiceCategory.PIPELINE_SERVICES: {
+        return {
+          includeLineage: includeLineage,
+          pipelineFilterPattern: getFilterPatternData(
+            pipelineFilterPattern,
+            showPipelineFilter
+          ),
+          type: ConfigType.PipelineMetadata,
+        };
+      }
       default: {
         return {};
       }
@@ -580,6 +615,7 @@ const AddIngestion = ({
             getIncludeValue={getIncludeValue}
             handleDescription={(val) => setDescription(val)}
             handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)}
+            handleIncludeLineage={() => setIncludeLineage((pre) => !pre)}
             handleIncludeView={() => setIncludeView((pre) => !pre)}
             handleIngestionName={(val) => setIngestionName(val)}
             handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)}
@@ -587,9 +623,11 @@ const AddIngestion = ({
             handleResultLimit={(val) => setResultLimit(val)}
             handleShowFilter={handleShowFilter}
             handleStageFileLocation={(val) => setStageFileLocation(val)}
+            includeLineage={includeLineage}
             includeView={includeView}
             ingestionName={ingestionName}
             markDeletedTables={markDeletedTables}
+            pipelineFilterPattern={pipelineFilterPattern}
             pipelineType={pipelineType}
             queryLogDuration={queryLogDuration}
             resultLimit={resultLimit}
@@ -599,6 +637,7 @@ const AddIngestion = ({
             showDashboardFilter={showDashboardFilter}
             showDatabaseFilter={showDatabaseFilter}
             showFqnFilter={showFqnFilter}
+            showPipelineFilter={showPipelineFilter}
             showSchemaFilter={showSchemaFilter}
             showTableFilter={showTableFilter}
             showTopicFilter={showTopicFilter}

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx

@@ -54,10 +54,15 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
     includes: [],
     excludes: [],
   },
+  pipelineFilterPattern: {
+    includes: [],
+    excludes: [],
+  },
   fqnFilterPattern: {
     includes: [],
     excludes: [],
   },
+  includeLineage: false,
   includeView: false,
   pipelineType: PipelineType.Metadata,
   queryLogDuration: 1,
@@ -70,7 +75,9 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
   showTableFilter: false,
   showTopicFilter: false,
   showChartFilter: false,
+  showPipelineFilter: false,
   showFqnFilter: false,
+  handleIncludeLineage: jest.fn(),
   handleIncludeView: jest.fn(),
   handleIngestionName: jest.fn(),
   handleMarkDeletedTables: jest.fn(),

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx

@@ -34,7 +34,9 @@ const ConfigureIngestion = ({
   tableFilterPattern,
   topicFilterPattern,
   chartFilterPattern,
+  pipelineFilterPattern,
   fqnFilterPattern,
+  includeLineage,
   includeView,
   markDeletedTables,
   serviceCategory,
@@ -45,6 +47,7 @@ const ConfigureIngestion = ({
   showTableFilter,
   showTopicFilter,
   showChartFilter,
+  showPipelineFilter,
   showFqnFilter,
   queryLogDuration,
   stageFileLocation,
@@ -56,6 +59,7 @@ const ConfigureIngestion = ({
   handleIngestionName,
   handleDescription,
   handleShowFilter,
+  handleIncludeLineage,
   handleIncludeView,
   handleMarkDeletedTables,
   handleQueryLogDuration,
@@ -128,6 +132,28 @@ const ConfigureIngestion = ({
     );
   };
 
+  const getPipelineFieldToggles = () => {
+    return (
+      <div>
+        <Field>
+          <div className="tw-flex tw-gap-1">
+            <label>Include lineage</label>
+            <ToggleSwitchV1
+              checked={includeLineage}
+              handleCheck={handleIncludeLineage}
+              testId="include-lineage"
+            />
+          </div>
+          <p className="tw-text-grey-muted tw-mt-3">
+            Configuration to turn off fetching lineage from pipelines.
+          </p>
+          {getSeparator('')}
+        </Field>
+        {getDebugLogToggle()}
+      </div>
+    );
+  };
+
   const getMetadataFilterPatternField = () => {
     switch (serviceCategory) {
       case ServiceCategory.DATABASE_SERVICES:
@@ -221,6 +247,25 @@ const ConfigureIngestion = ({
             {getDebugLogToggle()}
           </Fragment>
         );
+      case ServiceCategory.PIPELINE_SERVICES:
+        return (
+          <Fragment>
+            <FilterPattern
+              checked={showPipelineFilter}
+              excludePattern={pipelineFilterPattern.excludes ?? []}
+              getExcludeValue={getExcludeValue}
+              getIncludeValue={getIncludeValue}
+              handleChecked={(value) =>
+                handleShowFilter(value, FilterPatternEnum.PIPELINE)
+              }
+              includePattern={pipelineFilterPattern.includes ?? []}
+              showSeparator={false}
+              type={FilterPatternEnum.PIPELINE}
+            />
+            {getSeparator('')}
+            {getPipelineFieldToggles()}
+          </Fragment>
+        );
       default:
         return <></>;
     }

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts

@@ -64,7 +64,9 @@ export interface ConfigureIngestionProps {
   tableFilterPattern: FilterPattern;
   topicFilterPattern: FilterPattern;
   chartFilterPattern: FilterPattern;
+  pipelineFilterPattern: FilterPattern;
   fqnFilterPattern: FilterPattern;
+  includeLineage: boolean;
   includeView: boolean;
   markDeletedTables?: boolean;
   enableDebugLog: boolean;
@@ -75,12 +77,14 @@ export interface ConfigureIngestionProps {
   showTableFilter: boolean;
   showTopicFilter: boolean;
   showChartFilter: boolean;
+  showPipelineFilter: boolean;
   showFqnFilter: boolean;
   queryLogDuration: number;
   stageFileLocation: string;
   resultLimit: number;
   handleIngestionName: (value: string) => void;
   handleDescription?: (value: string) => void;
+  handleIncludeLineage: () => void;
   handleIncludeView: () => void;
   handleMarkDeletedTables?: () => void;
   handleEnableDebugLog: () => void;

openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx

@@ -21,17 +21,12 @@ import { FormSubmitType } from '../../enums/form.enum';
 import { PageLayoutType } from '../../enums/layout.enum';
 import { ServiceCategory } from '../../enums/service.enum';
 import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
-import {
+import { ConfigData, DataObj } from '../../interface/service.interface';
-  ConfigData,
-  DataObj,
-  DataService,
-} from '../../interface/service.interface';
 import { getCurrentUserId } from '../../utils/CommonUtils';
 import { getAddServicePath } from '../../utils/RouterUtils';
 import {
   getServiceCreatedLabel,
   getServiceIngestionStepGuide,
-  isIngestionSupported,
 } from '../../utils/ServiceUtils';
 import AddIngestion from '../AddIngestion/AddIngestion.component';
 import SuccessScreen from '../common/success-screen/SuccessScreen';
@@ -129,10 +124,7 @@ const AddService = ({
     });
   };
 
-  const handleConfigUpdate = (
+  const handleConfigUpdate = (oData: ConfigData) => {
-    oData: ConfigData,
-    serviceCat: ServiceCategory
-  ) => {
     const data = {
       name: serviceName,
       serviceType: selectServiceType,
@@ -142,10 +134,7 @@ const AddService = ({
         type: 'user',
       },
     };
-    const configData =
+    const configData = {
-      serviceCat === ServiceCategory.PIPELINE_SERVICES
-        ? { ...data, pipelineUrl: oData.pipelineUrl }
-        : {
       ...data,
       connection: {
         config: oData,
@@ -229,30 +218,23 @@ const AddService = ({
           {activeServiceStep === 3 && (
             <ConnectionConfigForm
               cancelText="Back"
-              data={
-                (serviceCategory !== ServiceCategory.PIPELINE_SERVICES
-                  ? {
-                      connection: { config: { type: selectServiceType } },
-                    }
-                  : {}) as DataService
-              }
               serviceCategory={serviceCategory}
               serviceType={selectServiceType}
               status={saveServiceState}
               onCancel={handleConnectionDetailsBackClick}
               onSave={(e) => {
-                handleConfigUpdate(e.formData, serviceCategory);
+                handleConfigUpdate(e.formData);
               }}
             />
           )}
 
           {activeServiceStep > 3 && (
             <SuccessScreen
+              showIngestionButton
               handleIngestionClick={() => handleAddIngestion(true)}
               handleViewServiceClick={handleViewServiceClick}
               isAirflowSetup={isAirflowRunning}
               name={serviceName}
-              showIngestionButton={isIngestionSupported(serviceCategory)}
               state={FormSubmitType.ADD}
               suffix={getServiceCreatedLabel(serviceCategory)}
               onCheckAirflowStatus={onAirflowStatusCheck}

openmetadata-ui/src/main/resources/ui/src/components/ServiceConfig/ConnectionConfigForm.tsx

@@ -18,19 +18,22 @@ import React, { Fragment, FunctionComponent, useMemo } from 'react';
 import { TestConnection } from '../../axiosAPIs/serviceAPI';
 import { ServiceCategory } from '../../enums/service.enum';
 import {
-  DashboardConnection,
   DashboardService,
+  DashboardServiceType,
 } from '../../generated/entity/services/dashboardService';
 import {
-  DatabaseConnection,
   DatabaseService,
+  DatabaseServiceType,
 } from '../../generated/entity/services/databaseService';
 import {
-  MessagingConnection,
   MessagingService,
+  MessagingServiceType,
 } from '../../generated/entity/services/messagingService';
-import { PipelineService } from '../../generated/entity/services/pipelineService';
+import {
-import { ConfigData } from '../../interface/service.interface';
+  PipelineService,
+  PipelineServiceType,
+} from '../../generated/entity/services/pipelineService';
+import { ConfigData, DataService } from '../../interface/service.interface';
 import jsonData from '../../jsons/en';
 import { getDashboardConfig } from '../../utils/DashboardServiceUtils';
 import { getDatabaseConfig } from '../../utils/DatabaseServiceUtils';
@@ -45,7 +48,11 @@ import { showErrorToast } from '../../utils/ToastUtils';
 import FormBuilder from '../common/FormBuilder/FormBuilder';
 
 interface Props {
-  data: DatabaseService | MessagingService | DashboardService | PipelineService;
+  data?:
+    | DatabaseService
+    | MessagingService
+    | DashboardService
+    | PipelineService;
   okText?: string;
   cancelText?: string;
   serviceType: string;
@@ -66,17 +73,11 @@ const ConnectionConfigForm: FunctionComponent<Props> = ({
   onSave,
 }: Props) => {
   const allowTestConn = useMemo(() => {
-    return shouldTestConnection(serviceType, serviceCategory);
+    return shouldTestConnection(serviceType);
-  }, [serviceType, serviceCategory]);
+  }, [serviceType]);
 
   const config = !isNil(data)
-    ? /* eslint-disable-next-line no-prototype-builtins */
+    ? ((data as DataService).connection.config as ConfigData)
-      data.hasOwnProperty('connection')
-      ? ((data as DatabaseService | MessagingService | DashboardService)
-          .connection.config as ConfigData)
-      : ({
-          pipelineUrl: (data as PipelineService).connection.config?.hostPort,
-        } as ConfigData)
     : ({} as ConfigData);
 
   const handleSave = (data: ISubmitEvent<ConfigData>) => {
@@ -124,28 +125,22 @@ const ConnectionConfigForm: FunctionComponent<Props> = ({
 
     switch (serviceCategory) {
       case ServiceCategory.DATABASE_SERVICES: {
-        connSch = getDatabaseConfig(
+        connSch = getDatabaseConfig(serviceType as DatabaseServiceType);
-          validConfig as DatabaseConnection['config']
-        );
 
         break;
       }
       case ServiceCategory.MESSAGING_SERVICES: {
-        connSch = getMessagingConfig(
+        connSch = getMessagingConfig(serviceType as MessagingServiceType);
-          validConfig as MessagingConnection['config']
-        );
 
         break;
       }
       case ServiceCategory.DASHBOARD_SERVICES: {
-        connSch = getDashboardConfig(
+        connSch = getDashboardConfig(serviceType as DashboardServiceType);
-          validConfig as DashboardConnection['config']
-        );
 
         break;
       }
       case ServiceCategory.PIPELINE_SERVICES: {
-        connSch = getPipelineConfig();
+        connSch = getPipelineConfig(serviceType as PipelineServiceType);
 
         break;
       }

openmetadata-ui/src/main/resources/ui/src/constants/services.const.ts

@@ -131,10 +131,17 @@ export const STEPS_FOR_ADD_SERVICE: Array<StepperStepType> = [
   { name: 'Connection Details', step: 3 },
 ];
 
-export const COMMON_UI_SCHEMA = {
+const DEF_UI_SCHEMA = {
   supportsMetadataExtraction: { 'ui:widget': 'hidden', 'ui:hideError': true },
   supportsUsageExtraction: { 'ui:widget': 'hidden', 'ui:hideError': true },
   supportsProfiler: { 'ui:widget': 'hidden', 'ui:hideError': true },
   supportsDatabase: { 'ui:widget': 'hidden', 'ui:hideError': true },
   type: { 'ui:widget': 'hidden' },
 };
+
+export const COMMON_UI_SCHEMA = {
+  ...DEF_UI_SCHEMA,
+  connection: {
+    ...DEF_UI_SCHEMA,
+  },
+};

openmetadata-ui/src/main/resources/ui/src/enums/filterPattern.enum.ts

@@ -19,4 +19,5 @@ export enum FilterPatternEnum {
   DASHBOARD = 'dashboard',
   TOPIC = 'topic',
   FQN = 'fqn',
+  PIPELINE = 'pipeline',
 }

openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts

@@ -75,6 +75,5 @@ export interface ServiceResponse {
 
 export type ConfigData = Partial<DatabaseService['connection']> &
   Partial<MessagingService['connection']> &
-  Partial<DashboardService['connection']> & {
+  Partial<DashboardService['connection']> &
-    pipelineUrl: string;
+  Partial<PipelineService['connection']>;
-  };

openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx

@@ -63,7 +63,7 @@ import { IngestionPipeline } from '../../generated/entity/services/ingestionPipe
 import { EntityReference } from '../../generated/type/entityReference';
 import { Paging } from '../../generated/type/paging';
 import { useAuth } from '../../hooks/authHooks';
-import { ServiceDataObj } from '../../interface/service.interface';
+import { ConfigData, ServiceDataObj } from '../../interface/service.interface';
 import jsonData from '../../jsons/en';
 import {
   getEntityDeleteMessage,
@@ -77,7 +77,6 @@ import { getInfoElements } from '../../utils/EntityUtils';
 import { getServicesWithTabPath } from '../../utils/RouterUtils';
 import {
   getCurrentServiceTab,
-  getIsIngestionEnable,
   getServiceCategoryFromType,
   servicePageTabs,
   serviceTypeLogo,
@@ -96,9 +95,6 @@ const ServicePage: FunctionComponent = () => {
   const [serviceName, setServiceName] = useState(
     serviceCategory || getServiceCategoryFromType(serviceType)
   );
-  const [isIngestionEnable] = useState(
-    getIsIngestionEnable(serviceName as ServiceCategory)
-  );
   const [slashedTableName, setSlashedTableName] = useState<
     TitleBreadcrumbProps['titleLinks']
   >([]);
@@ -155,7 +151,6 @@ const ServicePage: FunctionComponent = () => {
         title: 'Sample Data',
         selectedName: 'sample-data-color',
       },
-      isHidden: !isIngestionEnable,
       isProtected: false,
       position: 2,
       count: ingestions.length,
@@ -340,24 +335,8 @@ const ServicePage: FunctionComponent = () => {
     }).finally(() => setIsloading(false));
   };
 
-  const handleConfigUpdate = (
+  const handleConfigUpdate = (updatedData: ConfigData) => {
-    updatedData: ServicesData,
+    const configData = {
-    serviceCategory: ServiceCategory
-  ) => {
-    const configData =
-      serviceCategory === ServiceCategory.PIPELINE_SERVICES
-        ? {
-            databaseConnection: updatedData.databaseConnection,
-            name: updatedData.name,
-            serviceType: updatedData.serviceType,
-            brokers: updatedData.brokers,
-            schemaRegistry: updatedData.schemaRegistry,
-            dashboardUrl: updatedData.dashboardUrl,
-            username: updatedData.username,
-            password: updatedData.password,
-            pipelineUrl: updatedData.pipelineUrl,
-          }
-        : {
       name: serviceDetails?.name,
       serviceType: serviceDetails?.serviceType,
       description: serviceDetails?.description,
@@ -735,7 +714,6 @@ const ServicePage: FunctionComponent = () => {
       activeTabHandler(1);
     }
 
-    if (isIngestionEnable) {
     getAirflowStatus()
       .then(() => {
         setIsAirflowRunning(true);
@@ -744,7 +722,6 @@ const ServicePage: FunctionComponent = () => {
       .catch(() => {
         setIsAirflowRunning(false);
       });
-    }
   }, []);
 
   const onCancel = () => {

openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts

@@ -30,10 +30,10 @@ export const getDashboardURL = (config: DashboardConnection['config']) => {
     : '--';
 };
 
-export const getDashboardConfig = (config?: DashboardConnection['config']) => {
+export const getDashboardConfig = (type: DashboardServiceType) => {
   let schema = {};
   const uiSchema = { ...COMMON_UI_SCHEMA };
-  switch (config?.type) {
+  switch (type) {
     case DashboardServiceType.Looker: {
       schema = lookerConnection;
 

openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts

@@ -13,10 +13,7 @@
 
 import { cloneDeep } from 'lodash';
 import { COMMON_UI_SCHEMA } from '../constants/services.const';
-import {
+import { DatabaseServiceType } from '../generated/entity/services/databaseService';
-  DatabaseConnection,
-  DatabaseServiceType,
-} from '../generated/entity/services/databaseService';
 import athenaConnection from '../jsons/connectionSchemas/connections/database/athenaConnection.json';
 import azureSQLConnection from '../jsons/connectionSchemas/connections/database/azureSQLConnection.json';
 import bigQueryConnection from '../jsons/connectionSchemas/connections/database/bigQueryConnection.json';
@@ -43,10 +40,10 @@ import sqliteConnection from '../jsons/connectionSchemas/connections/database/sq
 import trinoConnection from '../jsons/connectionSchemas/connections/database/trinoConnection.json';
 import verticaConnection from '../jsons/connectionSchemas/connections/database/verticaConnection.json';
 
-export const getDatabaseConfig = (config?: DatabaseConnection['config']) => {
+export const getDatabaseConfig = (type: DatabaseServiceType) => {
   let schema = {};
   const uiSchema = { ...COMMON_UI_SCHEMA };
-  switch (config?.type as unknown as DatabaseServiceType) {
+  switch (type as unknown as DatabaseServiceType) {
     case DatabaseServiceType.Athena: {
       schema = athenaConnection;
 

openmetadata-ui/src/main/resources/ui/src/utils/MessagingServiceUtils.ts

@@ -30,10 +30,10 @@ export const getBrokers = (config: MessagingConnection['config']) => {
   return !isUndefined(retVal) ? retVal : '--';
 };
 
-export const getMessagingConfig = (config: MessagingConnection['config']) => {
+export const getMessagingConfig = (type: MessagingServiceType) => {
   let schema = {};
   const uiSchema = { ...COMMON_UI_SCHEMA };
-  if (config?.type === MessagingServiceType.Kafka) {
+  if (type === MessagingServiceType.Kafka) {
     schema = kafkaConnection;
   }
 

openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts

@@ -11,25 +11,27 @@
  *  limitations under the License.
  */
 
-import { JSONSchema7 } from 'json-schema';
 import { cloneDeep } from 'lodash';
+import { COMMON_UI_SCHEMA } from '../constants/services.const';
+import { PipelineServiceType } from '../generated/entity/services/pipelineService';
+import airflowConnection from '../jsons/connectionSchemas/connections/pipeline/airflowConnection.json';
+import glueConnection from '../jsons/connectionSchemas/connections/pipeline/glueConnection.json';
 
-export const getPipelineConfig = () => {
+export const getPipelineConfig = (type: PipelineServiceType) => {
-  const schema = {
+  let schema = {};
-    $schema: 'http://json-schema.org/draft-07/schema#',
+  const uiSchema = { ...COMMON_UI_SCHEMA };
-    title: 'PipelineConnection',
+  switch (type) {
-    description: 'Pipeline Connection Config',
+    case PipelineServiceType.Airflow: {
-    type: 'object',
+      schema = airflowConnection;
-    properties: {
+
-      pipelineUrl: {
+      break;
-        description: 'Pipeline Service Management/UI URL.',
+    }
-        type: 'string',
+    case PipelineServiceType.Glue: {
-        format: 'uri',
+      schema = glueConnection;
-      },
+
-    },
+      break;
-    additionalProperties: false,
+    }
-  } as JSONSchema7;
+  }
-  const uiSchema = {};
 
   return cloneDeep({ schema, uiSchema });
 };

openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx

@@ -339,23 +339,6 @@ export const getTotalEntityCountByService = (buckets: Array<Bucket> = []) => {
   return entityCounts;
 };
 
-export const getIsIngestionEnable = (serviceCategory: ServiceCategory) => {
-  switch (serviceCategory) {
-    case ServiceCategory.DATABASE_SERVICES:
-    case ServiceCategory.MESSAGING_SERVICES:
-    case ServiceCategory.DASHBOARD_SERVICES:
-      return true;
-
-    case ServiceCategory.PIPELINE_SERVICES:
-    default:
-      return false;
-  }
-};
-
-export const isIngestionSupported = (serviceCategory: ServiceCategory) => {
-  return getIsIngestionEnable(serviceCategory);
-};
-
 export const getKeyValuePair = (obj: DynamicObj) => {
   return Object.entries(obj).map((v) => {
     return {
@@ -558,14 +541,8 @@ export const getIngestionName = (
   }
 };
 
-export const shouldTestConnection = (
+export const shouldTestConnection = (serviceType: string) => {
-  serviceType: string,
+  return serviceType !== DatabaseServiceType.SampleData;
-  serviceCategory: ServiceCategory
-) => {
-  return (
-    serviceType !== DatabaseServiceType.SampleData &&
-    serviceCategory !== ServiceCategory.PIPELINE_SERVICES
-  );
 };
 
 export const getTestConnectionType = (serviceCat: ServiceCategory) => {
@@ -574,6 +551,8 @@ export const getTestConnectionType = (serviceCat: ServiceCategory) => {
       return ConnectionType.Messaging;
     case ServiceCategory.DASHBOARD_SERVICES:
       return ConnectionType.Dashboard;
+    case ServiceCategory.PIPELINE_SERVICES:
+      return ConnectionType.Pipeline;
     case ServiceCategory.DATABASE_SERVICES:
     default:
       return ConnectionType.Database;