From ce149c86bee304873c40ec0bda608b63a8698cc1 Mon Sep 17 00:00:00 2001 From: darth-coder00 <86726556+darth-coder00@users.noreply.github.com> Date: Sat, 11 Jun 2022 10:32:06 +0530 Subject: [PATCH] Fix #5116: Update UI connection for Pipeline Services (#5408) --- .../testServiceConnection.json | 5 +- .../resources/ui/parseConnectionSchema.js | 41 ++++++------- .../AddIngestion/AddIngestion.component.tsx | 43 ++++++++++++- .../Steps/ConfigureIngestion.test.tsx | 7 +++ .../AddIngestion/Steps/ConfigureIngestion.tsx | 45 ++++++++++++++ .../AddIngestion/addIngestion.interface.ts | 4 ++ .../AddService/AddService.component.tsx | 38 +++--------- .../ServiceConfig/ConnectionConfigForm.tsx | 45 ++++++-------- .../ui/src/constants/services.const.ts | 9 ++- .../ui/src/enums/filterPattern.enum.ts | 1 + .../ui/src/interface/service.interface.ts | 5 +- .../resources/ui/src/pages/service/index.tsx | 61 ++++++------------- .../ui/src/utils/DashboardServiceUtils.ts | 4 +- .../ui/src/utils/DatabaseServiceUtils.ts | 9 +-- .../ui/src/utils/MessagingServiceUtils.ts | 4 +- .../ui/src/utils/PipelineServiceUtils.ts | 36 +++++------ .../resources/ui/src/utils/ServiceUtils.tsx | 29 ++------- 17 files changed, 208 insertions(+), 178 deletions(-) diff --git a/catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json b/catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json index b3cb3428617..790bcfaf36f 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/services/ingestionPipelines/testServiceConnection.json @@ -22,7 +22,7 @@ "connectionType": { "description": "Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres...", "type": "string", - "enum": ["Database", "Dashboard", "Messaging"], + "enum": ["Database", "Dashboard", "Messaging", "Pipeline"], "javaEnums": [ { "name": "Database" @@ -32,6 +32,9 @@ }, { "name": "Messaging" + }, + { + "name": "Pipeline" } ] } diff --git a/openmetadata-ui/src/main/resources/ui/parseConnectionSchema.js b/openmetadata-ui/src/main/resources/ui/parseConnectionSchema.js index 25b850ceded..0e035d05f04 100644 --- a/openmetadata-ui/src/main/resources/ui/parseConnectionSchema.js +++ b/openmetadata-ui/src/main/resources/ui/parseConnectionSchema.js @@ -13,7 +13,7 @@ const rootDir = 'connTemp'; const srcDir = 'schema/entity/services/connections'; const destDir = 'src/jsons/connectionSchemas/connections'; -const playDir = `${rootDir}/${srcDir}/${rootDir}`; +const playDir = `${rootDir}/${srcDir}`; const globalParserOptions = { continueOnError: true, @@ -24,8 +24,11 @@ const globalParserOptions = { async function parseSchema(filePath, destPath) { try { + const fileDir = `${cwd}/${path.dirname(filePath)}`; + const fileName = path.basename(filePath); + process.chdir(fileDir); const parser = new $RefParser(globalParserOptions); - const schema = await parser.parse(filePath); + const schema = await parser.parse(fileName); const api = await parser.bundle(schema); const dirname = `${cwd}/${path.dirname(destPath)}`; if (!fs.existsSync(dirname)) { @@ -38,54 +41,44 @@ async function parseSchema(filePath, destPath) { fs.writeFileSync(`${cwd}/${destPath}`, JSON.stringify(api, null, 2)); } catch (err) { console.log(err); + } finally { + process.chdir(cwd); } } -function traverseDirectory(Directory) { - fs.readdirSync(Directory).forEach((File) => { +async function traverseDirectory(Directory) { + const Files = fs.readdirSync(Directory); + for (const File of Files) { const Absolute = path.join(Directory, File); if (fs.statSync(Absolute).isDirectory()) { - return traverseDirectory(Absolute); + await traverseDirectory(Absolute); } else { - const name = Absolute.replace(srcDir, destDir); - return parseSchema(Absolute, name); + const name = Absolute.replace(playDir, destDir); + await parseSchema(Absolute, name); } - }); + } } function copySourceFiles() { try { fse.copySync(schemaDir, `${rootDir}/schema`); - fse.copySync(schemaDir, `${playDir}/schema`); } catch (err) { console.error(err); } } -function main() { +async function main() { try { if (fs.existsSync(destDir)) { fs.rmSync(destDir, { recursive: true }); } fs.mkdirSync(destDir, { recursive: true }); copySourceFiles(); + + await traverseDirectory(`${playDir}`); } catch (err) { console.log(err); } - - process.chdir(`${cwd}/${playDir}`); - - fs.readdir(srcDir, (err, Files) => { - if (err) console.log(err); - else { - Files.forEach((File, index) => { - const Absolute = path.join(srcDir, File); - if (fs.statSync(Absolute).isDirectory()) { - traverseDirectory(Absolute); - } - }); - } - }); } main(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 6fb6e44e892..0c9f116a0e8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -122,6 +122,11 @@ const AddIngestion = ({ const [showChartFilter, setShowChartFilter] = useState( !isUndefined((data?.sourceConfig.config as ConfigClass)?.chartFilterPattern) ); + const [showPipelineFilter, setShowPipelineFilter] = useState( + !isUndefined( + (data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern + ) + ); const [showFqnFilter, setShowFqnFilter] = useState( !isUndefined((data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern) ); @@ -155,6 +160,9 @@ const AddIngestion = ({ const [includeView, setIncludeView] = useState( Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews) ); + const [includeLineage, setIncludeLineage] = useState( + Boolean((data?.sourceConfig.config as ConfigClass)?.includeLineage ?? true) + ); const [enableDebugLog, setEnableDebugLog] = useState( data?.loggerLevel === LogLevels.Debug ); @@ -184,6 +192,11 @@ const AddIngestion = ({ (data?.sourceConfig.config as ConfigClass)?.chartFilterPattern ?? INITIAL_FILTER_PATTERN ); + const [pipelineFilterPattern, setPipelineFilterPattern] = + useState( + (data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern ?? + INITIAL_FILTER_PATTERN + ); const [fqnFilterPattern, setFqnFilterPattern] = useState( (data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern ?? INITIAL_FILTER_PATTERN @@ -237,12 +250,16 @@ const AddIngestion = ({ break; case FilterPatternEnum.CHART: - setChartFilterPattern({ ...topicFilterPattern, includes: value }); + setChartFilterPattern({ ...chartFilterPattern, includes: value }); break; case FilterPatternEnum.FQN: setFqnFilterPattern({ ...fqnFilterPattern, includes: value }); + break; + case FilterPatternEnum.PIPELINE: + setPipelineFilterPattern({ ...pipelineFilterPattern, includes: value }); + break; } }; @@ -272,12 +289,16 @@ const AddIngestion = ({ break; case FilterPatternEnum.CHART: - setChartFilterPattern({ ...topicFilterPattern, excludes: value }); + setChartFilterPattern({ ...chartFilterPattern, excludes: value }); break; case FilterPatternEnum.FQN: setFqnFilterPattern({ ...fqnFilterPattern, excludes: value }); + break; + case FilterPatternEnum.PIPELINE: + setPipelineFilterPattern({ ...pipelineFilterPattern, excludes: value }); + break; } }; @@ -311,6 +332,10 @@ const AddIngestion = ({ case FilterPatternEnum.FQN: setShowFqnFilter(value); + break; + case FilterPatternEnum.PIPELINE: + setShowPipelineFilter(value); + break; } }; @@ -404,6 +429,16 @@ const AddIngestion = ({ type: ConfigType.DashboardMetadata, }; } + case ServiceCategory.PIPELINE_SERVICES: { + return { + includeLineage: includeLineage, + pipelineFilterPattern: getFilterPatternData( + pipelineFilterPattern, + showPipelineFilter + ), + type: ConfigType.PipelineMetadata, + }; + } default: { return {}; } @@ -580,6 +615,7 @@ const AddIngestion = ({ getIncludeValue={getIncludeValue} handleDescription={(val) => setDescription(val)} handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)} + handleIncludeLineage={() => setIncludeLineage((pre) => !pre)} handleIncludeView={() => setIncludeView((pre) => !pre)} handleIngestionName={(val) => setIngestionName(val)} handleMarkDeletedTables={() => setMarkDeletedTables((pre) => !pre)} @@ -587,9 +623,11 @@ const AddIngestion = ({ handleResultLimit={(val) => setResultLimit(val)} handleShowFilter={handleShowFilter} handleStageFileLocation={(val) => setStageFileLocation(val)} + includeLineage={includeLineage} includeView={includeView} ingestionName={ingestionName} markDeletedTables={markDeletedTables} + pipelineFilterPattern={pipelineFilterPattern} pipelineType={pipelineType} queryLogDuration={queryLogDuration} resultLimit={resultLimit} @@ -599,6 +637,7 @@ const AddIngestion = ({ showDashboardFilter={showDashboardFilter} showDatabaseFilter={showDatabaseFilter} showFqnFilter={showFqnFilter} + showPipelineFilter={showPipelineFilter} showSchemaFilter={showSchemaFilter} showTableFilter={showTableFilter} showTopicFilter={showTopicFilter} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx index 8a9b577247a..5c367ab8d77 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx @@ -54,10 +54,15 @@ const mockConfigureIngestion: ConfigureIngestionProps = { includes: [], excludes: [], }, + pipelineFilterPattern: { + includes: [], + excludes: [], + }, fqnFilterPattern: { includes: [], excludes: [], }, + includeLineage: false, includeView: false, pipelineType: PipelineType.Metadata, queryLogDuration: 1, @@ -70,7 +75,9 @@ const mockConfigureIngestion: ConfigureIngestionProps = { showTableFilter: false, showTopicFilter: false, showChartFilter: false, + showPipelineFilter: false, showFqnFilter: false, + handleIncludeLineage: jest.fn(), handleIncludeView: jest.fn(), handleIngestionName: jest.fn(), handleMarkDeletedTables: jest.fn(), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index 72642c829a1..7dbef1a68ca 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -34,7 +34,9 @@ const ConfigureIngestion = ({ tableFilterPattern, topicFilterPattern, chartFilterPattern, + pipelineFilterPattern, fqnFilterPattern, + includeLineage, includeView, markDeletedTables, serviceCategory, @@ -45,6 +47,7 @@ const ConfigureIngestion = ({ showTableFilter, showTopicFilter, showChartFilter, + showPipelineFilter, showFqnFilter, queryLogDuration, stageFileLocation, @@ -56,6 +59,7 @@ const ConfigureIngestion = ({ handleIngestionName, handleDescription, handleShowFilter, + handleIncludeLineage, handleIncludeView, handleMarkDeletedTables, handleQueryLogDuration, @@ -128,6 +132,28 @@ const ConfigureIngestion = ({ ); }; + const getPipelineFieldToggles = () => { + return ( +
+ +
+ + +
+

+ Configuration to turn off fetching lineage from pipelines. +

+ {getSeparator('')} + + {getDebugLogToggle()} +
+ ); + }; + const getMetadataFilterPatternField = () => { switch (serviceCategory) { case ServiceCategory.DATABASE_SERVICES: @@ -221,6 +247,25 @@ const ConfigureIngestion = ({ {getDebugLogToggle()} ); + case ServiceCategory.PIPELINE_SERVICES: + return ( + + + handleShowFilter(value, FilterPatternEnum.PIPELINE) + } + includePattern={pipelineFilterPattern.includes ?? []} + showSeparator={false} + type={FilterPatternEnum.PIPELINE} + /> + {getSeparator('')} + {getPipelineFieldToggles()} + + ); default: return <>; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts index b801f577d31..1483a50db1e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts @@ -64,7 +64,9 @@ export interface ConfigureIngestionProps { tableFilterPattern: FilterPattern; topicFilterPattern: FilterPattern; chartFilterPattern: FilterPattern; + pipelineFilterPattern: FilterPattern; fqnFilterPattern: FilterPattern; + includeLineage: boolean; includeView: boolean; markDeletedTables?: boolean; enableDebugLog: boolean; @@ -75,12 +77,14 @@ export interface ConfigureIngestionProps { showTableFilter: boolean; showTopicFilter: boolean; showChartFilter: boolean; + showPipelineFilter: boolean; showFqnFilter: boolean; queryLogDuration: number; stageFileLocation: string; resultLimit: number; handleIngestionName: (value: string) => void; handleDescription?: (value: string) => void; + handleIncludeLineage: () => void; handleIncludeView: () => void; handleMarkDeletedTables?: () => void; handleEnableDebugLog: () => void; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx index 524cddc9cf7..ed13311825f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx @@ -21,17 +21,12 @@ import { FormSubmitType } from '../../enums/form.enum'; import { PageLayoutType } from '../../enums/layout.enum'; import { ServiceCategory } from '../../enums/service.enum'; import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; -import { - ConfigData, - DataObj, - DataService, -} from '../../interface/service.interface'; +import { ConfigData, DataObj } from '../../interface/service.interface'; import { getCurrentUserId } from '../../utils/CommonUtils'; import { getAddServicePath } from '../../utils/RouterUtils'; import { getServiceCreatedLabel, getServiceIngestionStepGuide, - isIngestionSupported, } from '../../utils/ServiceUtils'; import AddIngestion from '../AddIngestion/AddIngestion.component'; import SuccessScreen from '../common/success-screen/SuccessScreen'; @@ -129,10 +124,7 @@ const AddService = ({ }); }; - const handleConfigUpdate = ( - oData: ConfigData, - serviceCat: ServiceCategory - ) => { + const handleConfigUpdate = (oData: ConfigData) => { const data = { name: serviceName, serviceType: selectServiceType, @@ -142,15 +134,12 @@ const AddService = ({ type: 'user', }, }; - const configData = - serviceCat === ServiceCategory.PIPELINE_SERVICES - ? { ...data, pipelineUrl: oData.pipelineUrl } - : { - ...data, - connection: { - config: oData, - }, - }; + const configData = { + ...data, + connection: { + config: oData, + }, + }; return new Promise((resolve, reject) => { setSaveServiceState('waiting'); @@ -229,30 +218,23 @@ const AddService = ({ {activeServiceStep === 3 && ( { - handleConfigUpdate(e.formData, serviceCategory); + handleConfigUpdate(e.formData); }} /> )} {activeServiceStep > 3 && ( handleAddIngestion(true)} handleViewServiceClick={handleViewServiceClick} isAirflowSetup={isAirflowRunning} name={serviceName} - showIngestionButton={isIngestionSupported(serviceCategory)} state={FormSubmitType.ADD} suffix={getServiceCreatedLabel(serviceCategory)} onCheckAirflowStatus={onAirflowStatusCheck} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ServiceConfig/ConnectionConfigForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ServiceConfig/ConnectionConfigForm.tsx index 7aa7c43f139..79380b8ff47 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ServiceConfig/ConnectionConfigForm.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ServiceConfig/ConnectionConfigForm.tsx @@ -18,19 +18,22 @@ import React, { Fragment, FunctionComponent, useMemo } from 'react'; import { TestConnection } from '../../axiosAPIs/serviceAPI'; import { ServiceCategory } from '../../enums/service.enum'; import { - DashboardConnection, DashboardService, + DashboardServiceType, } from '../../generated/entity/services/dashboardService'; import { - DatabaseConnection, DatabaseService, + DatabaseServiceType, } from '../../generated/entity/services/databaseService'; import { - MessagingConnection, MessagingService, + MessagingServiceType, } from '../../generated/entity/services/messagingService'; -import { PipelineService } from '../../generated/entity/services/pipelineService'; -import { ConfigData } from '../../interface/service.interface'; +import { + PipelineService, + PipelineServiceType, +} from '../../generated/entity/services/pipelineService'; +import { ConfigData, DataService } from '../../interface/service.interface'; import jsonData from '../../jsons/en'; import { getDashboardConfig } from '../../utils/DashboardServiceUtils'; import { getDatabaseConfig } from '../../utils/DatabaseServiceUtils'; @@ -45,7 +48,11 @@ import { showErrorToast } from '../../utils/ToastUtils'; import FormBuilder from '../common/FormBuilder/FormBuilder'; interface Props { - data: DatabaseService | MessagingService | DashboardService | PipelineService; + data?: + | DatabaseService + | MessagingService + | DashboardService + | PipelineService; okText?: string; cancelText?: string; serviceType: string; @@ -66,17 +73,11 @@ const ConnectionConfigForm: FunctionComponent = ({ onSave, }: Props) => { const allowTestConn = useMemo(() => { - return shouldTestConnection(serviceType, serviceCategory); - }, [serviceType, serviceCategory]); + return shouldTestConnection(serviceType); + }, [serviceType]); const config = !isNil(data) - ? /* eslint-disable-next-line no-prototype-builtins */ - data.hasOwnProperty('connection') - ? ((data as DatabaseService | MessagingService | DashboardService) - .connection.config as ConfigData) - : ({ - pipelineUrl: (data as PipelineService).connection.config?.hostPort, - } as ConfigData) + ? ((data as DataService).connection.config as ConfigData) : ({} as ConfigData); const handleSave = (data: ISubmitEvent) => { @@ -124,28 +125,22 @@ const ConnectionConfigForm: FunctionComponent = ({ switch (serviceCategory) { case ServiceCategory.DATABASE_SERVICES: { - connSch = getDatabaseConfig( - validConfig as DatabaseConnection['config'] - ); + connSch = getDatabaseConfig(serviceType as DatabaseServiceType); break; } case ServiceCategory.MESSAGING_SERVICES: { - connSch = getMessagingConfig( - validConfig as MessagingConnection['config'] - ); + connSch = getMessagingConfig(serviceType as MessagingServiceType); break; } case ServiceCategory.DASHBOARD_SERVICES: { - connSch = getDashboardConfig( - validConfig as DashboardConnection['config'] - ); + connSch = getDashboardConfig(serviceType as DashboardServiceType); break; } case ServiceCategory.PIPELINE_SERVICES: { - connSch = getPipelineConfig(); + connSch = getPipelineConfig(serviceType as PipelineServiceType); break; } diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/services.const.ts b/openmetadata-ui/src/main/resources/ui/src/constants/services.const.ts index a32f2286b8d..eee30272466 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/services.const.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/services.const.ts @@ -131,10 +131,17 @@ export const STEPS_FOR_ADD_SERVICE: Array = [ { name: 'Connection Details', step: 3 }, ]; -export const COMMON_UI_SCHEMA = { +const DEF_UI_SCHEMA = { supportsMetadataExtraction: { 'ui:widget': 'hidden', 'ui:hideError': true }, supportsUsageExtraction: { 'ui:widget': 'hidden', 'ui:hideError': true }, supportsProfiler: { 'ui:widget': 'hidden', 'ui:hideError': true }, supportsDatabase: { 'ui:widget': 'hidden', 'ui:hideError': true }, type: { 'ui:widget': 'hidden' }, }; + +export const COMMON_UI_SCHEMA = { + ...DEF_UI_SCHEMA, + connection: { + ...DEF_UI_SCHEMA, + }, +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/filterPattern.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/filterPattern.enum.ts index 364904d633f..d631706a377 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/filterPattern.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/filterPattern.enum.ts @@ -19,4 +19,5 @@ export enum FilterPatternEnum { DASHBOARD = 'dashboard', TOPIC = 'topic', FQN = 'fqn', + PIPELINE = 'pipeline', } diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts index 86109fcf700..b0a592fb16f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts @@ -75,6 +75,5 @@ export interface ServiceResponse { export type ConfigData = Partial & Partial & - Partial & { - pipelineUrl: string; - }; + Partial & + Partial; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx index 4642d4fc46f..548b6cf902a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx @@ -63,7 +63,7 @@ import { IngestionPipeline } from '../../generated/entity/services/ingestionPipe import { EntityReference } from '../../generated/type/entityReference'; import { Paging } from '../../generated/type/paging'; import { useAuth } from '../../hooks/authHooks'; -import { ServiceDataObj } from '../../interface/service.interface'; +import { ConfigData, ServiceDataObj } from '../../interface/service.interface'; import jsonData from '../../jsons/en'; import { getEntityDeleteMessage, @@ -77,7 +77,6 @@ import { getInfoElements } from '../../utils/EntityUtils'; import { getServicesWithTabPath } from '../../utils/RouterUtils'; import { getCurrentServiceTab, - getIsIngestionEnable, getServiceCategoryFromType, servicePageTabs, serviceTypeLogo, @@ -96,9 +95,6 @@ const ServicePage: FunctionComponent = () => { const [serviceName, setServiceName] = useState( serviceCategory || getServiceCategoryFromType(serviceType) ); - const [isIngestionEnable] = useState( - getIsIngestionEnable(serviceName as ServiceCategory) - ); const [slashedTableName, setSlashedTableName] = useState< TitleBreadcrumbProps['titleLinks'] >([]); @@ -155,7 +151,6 @@ const ServicePage: FunctionComponent = () => { title: 'Sample Data', selectedName: 'sample-data-color', }, - isHidden: !isIngestionEnable, isProtected: false, position: 2, count: ingestions.length, @@ -340,32 +335,16 @@ const ServicePage: FunctionComponent = () => { }).finally(() => setIsloading(false)); }; - const handleConfigUpdate = ( - updatedData: ServicesData, - serviceCategory: ServiceCategory - ) => { - const configData = - serviceCategory === ServiceCategory.PIPELINE_SERVICES - ? { - databaseConnection: updatedData.databaseConnection, - name: updatedData.name, - serviceType: updatedData.serviceType, - brokers: updatedData.brokers, - schemaRegistry: updatedData.schemaRegistry, - dashboardUrl: updatedData.dashboardUrl, - username: updatedData.username, - password: updatedData.password, - pipelineUrl: updatedData.pipelineUrl, - } - : { - name: serviceDetails?.name, - serviceType: serviceDetails?.serviceType, - description: serviceDetails?.description, - owner: serviceDetails?.owner, - connection: { - config: updatedData, - }, - }; + const handleConfigUpdate = (updatedData: ConfigData) => { + const configData = { + name: serviceDetails?.name, + serviceType: serviceDetails?.serviceType, + description: serviceDetails?.description, + owner: serviceDetails?.owner, + connection: { + config: updatedData, + }, + }; return new Promise((resolve, reject) => { updateService(serviceName, serviceDetails?.id, configData) @@ -735,16 +714,14 @@ const ServicePage: FunctionComponent = () => { activeTabHandler(1); } - if (isIngestionEnable) { - getAirflowStatus() - .then(() => { - setIsAirflowRunning(true); - getAllIngestionWorkflows(); - }) - .catch(() => { - setIsAirflowRunning(false); - }); - } + getAirflowStatus() + .then(() => { + setIsAirflowRunning(true); + getAllIngestionWorkflows(); + }) + .catch(() => { + setIsAirflowRunning(false); + }); }, []); const onCancel = () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts index c8f19eae869..ea0c7926c9b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DashboardServiceUtils.ts @@ -30,10 +30,10 @@ export const getDashboardURL = (config: DashboardConnection['config']) => { : '--'; }; -export const getDashboardConfig = (config?: DashboardConnection['config']) => { +export const getDashboardConfig = (type: DashboardServiceType) => { let schema = {}; const uiSchema = { ...COMMON_UI_SCHEMA }; - switch (config?.type) { + switch (type) { case DashboardServiceType.Looker: { schema = lookerConnection; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts index 1cc0ea4553a..1cb568ffccf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts @@ -13,10 +13,7 @@ import { cloneDeep } from 'lodash'; import { COMMON_UI_SCHEMA } from '../constants/services.const'; -import { - DatabaseConnection, - DatabaseServiceType, -} from '../generated/entity/services/databaseService'; +import { DatabaseServiceType } from '../generated/entity/services/databaseService'; import athenaConnection from '../jsons/connectionSchemas/connections/database/athenaConnection.json'; import azureSQLConnection from '../jsons/connectionSchemas/connections/database/azureSQLConnection.json'; import bigQueryConnection from '../jsons/connectionSchemas/connections/database/bigQueryConnection.json'; @@ -43,10 +40,10 @@ import sqliteConnection from '../jsons/connectionSchemas/connections/database/sq import trinoConnection from '../jsons/connectionSchemas/connections/database/trinoConnection.json'; import verticaConnection from '../jsons/connectionSchemas/connections/database/verticaConnection.json'; -export const getDatabaseConfig = (config?: DatabaseConnection['config']) => { +export const getDatabaseConfig = (type: DatabaseServiceType) => { let schema = {}; const uiSchema = { ...COMMON_UI_SCHEMA }; - switch (config?.type as unknown as DatabaseServiceType) { + switch (type as unknown as DatabaseServiceType) { case DatabaseServiceType.Athena: { schema = athenaConnection; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/MessagingServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/MessagingServiceUtils.ts index 38348f873b2..a6de8ae9368 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/MessagingServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/MessagingServiceUtils.ts @@ -30,10 +30,10 @@ export const getBrokers = (config: MessagingConnection['config']) => { return !isUndefined(retVal) ? retVal : '--'; }; -export const getMessagingConfig = (config: MessagingConnection['config']) => { +export const getMessagingConfig = (type: MessagingServiceType) => { let schema = {}; const uiSchema = { ...COMMON_UI_SCHEMA }; - if (config?.type === MessagingServiceType.Kafka) { + if (type === MessagingServiceType.Kafka) { schema = kafkaConnection; } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts index 984b742d0b5..d2e43e45a7d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts @@ -11,25 +11,27 @@ * limitations under the License. */ -import { JSONSchema7 } from 'json-schema'; import { cloneDeep } from 'lodash'; +import { COMMON_UI_SCHEMA } from '../constants/services.const'; +import { PipelineServiceType } from '../generated/entity/services/pipelineService'; +import airflowConnection from '../jsons/connectionSchemas/connections/pipeline/airflowConnection.json'; +import glueConnection from '../jsons/connectionSchemas/connections/pipeline/glueConnection.json'; -export const getPipelineConfig = () => { - const schema = { - $schema: 'http://json-schema.org/draft-07/schema#', - title: 'PipelineConnection', - description: 'Pipeline Connection Config', - type: 'object', - properties: { - pipelineUrl: { - description: 'Pipeline Service Management/UI URL.', - type: 'string', - format: 'uri', - }, - }, - additionalProperties: false, - } as JSONSchema7; - const uiSchema = {}; +export const getPipelineConfig = (type: PipelineServiceType) => { + let schema = {}; + const uiSchema = { ...COMMON_UI_SCHEMA }; + switch (type) { + case PipelineServiceType.Airflow: { + schema = airflowConnection; + + break; + } + case PipelineServiceType.Glue: { + schema = glueConnection; + + break; + } + } return cloneDeep({ schema, uiSchema }); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx index 763af9ac577..e98d8d7977f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx @@ -339,23 +339,6 @@ export const getTotalEntityCountByService = (buckets: Array = []) => { return entityCounts; }; -export const getIsIngestionEnable = (serviceCategory: ServiceCategory) => { - switch (serviceCategory) { - case ServiceCategory.DATABASE_SERVICES: - case ServiceCategory.MESSAGING_SERVICES: - case ServiceCategory.DASHBOARD_SERVICES: - return true; - - case ServiceCategory.PIPELINE_SERVICES: - default: - return false; - } -}; - -export const isIngestionSupported = (serviceCategory: ServiceCategory) => { - return getIsIngestionEnable(serviceCategory); -}; - export const getKeyValuePair = (obj: DynamicObj) => { return Object.entries(obj).map((v) => { return { @@ -558,14 +541,8 @@ export const getIngestionName = ( } }; -export const shouldTestConnection = ( - serviceType: string, - serviceCategory: ServiceCategory -) => { - return ( - serviceType !== DatabaseServiceType.SampleData && - serviceCategory !== ServiceCategory.PIPELINE_SERVICES - ); +export const shouldTestConnection = (serviceType: string) => { + return serviceType !== DatabaseServiceType.SampleData; }; export const getTestConnectionType = (serviceCat: ServiceCategory) => { @@ -574,6 +551,8 @@ export const getTestConnectionType = (serviceCat: ServiceCategory) => { return ConnectionType.Messaging; case ServiceCategory.DASHBOARD_SERVICES: return ConnectionType.Dashboard; + case ServiceCategory.PIPELINE_SERVICES: + return ConnectionType.Pipeline; case ServiceCategory.DATABASE_SERVICES: default: return ConnectionType.Database;