From a6424b271362eabc4cd938f03f76b1c1bc79b113 Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Fri, 15 Apr 2022 03:26:15 +0530 Subject: [PATCH] Added new API integration for service page (#4158) --- .../ui/src/axiosAPIs/ingestionPipelineAPI.ts | 44 ++ .../AddIngestion/AddIngestion.component.tsx | 197 ++++++--- .../AddIngestion/AddIngestion.test.tsx | 7 +- .../Steps/ConfigureIngestion.test.tsx | 20 +- .../AddIngestion/Steps/ConfigureIngestion.tsx | 20 +- .../AddIngestion/addIngestion.interface.ts | 38 +- .../AddService/AddService.component.tsx | 9 +- .../Ingestion/Ingestion.component.tsx | 383 ++++++++---------- .../components/Ingestion/Ingestion.mock.ts | 153 ++++--- .../components/Ingestion/Ingestion.test.tsx | 67 +-- .../Ingestion/ingestion.interface.ts | 27 +- .../ui/src/constants/ingestion.constant.ts | 8 +- .../ui/src/pages/service/index.test.tsx | 14 +- .../resources/ui/src/pages/service/index.tsx | 139 +++---- 14 files changed, 634 insertions(+), 492 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts index 150418ee1cd..77bf1fbb48a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts @@ -12,7 +12,9 @@ */ import { AxiosResponse } from 'axios'; +import { Operation } from 'fast-json-patch'; import { CreateIngestionPipeline } from '../generated/api/services/ingestionPipelines/createIngestionPipeline'; +import { getURLWithQueryFields } from '../utils/APIUtils'; import APIClient from './index'; export const addIngestionPipeline = ( @@ -20,3 +22,45 @@ export const addIngestionPipeline = ( ): Promise => { return APIClient.post('/services/ingestionPipelines', data); }; + +export const getIngestionPipelines = ( + arrQueryFields: Array, + serviceFilter?: string, + paging?: string +): Promise => { + const service = serviceFilter ? `service=${serviceFilter}` : ''; + const url = `${getURLWithQueryFields( + '/services/ingestionPipelines', + arrQueryFields, + service + )}${paging ? paging : ''}`; + + return APIClient.get(url); +}; + +export const triggerIngestionPipelineById = ( + id: string +): Promise => { + return APIClient.post(`/services/ingestionPipelines/trigger/${id}`); +}; + +export const deleteIngestionPipelineById = ( + id: string +): Promise => { + return APIClient.delete(`/services/ingestionPipelines/${id}?hardDelete=true`); +}; + +export const updateIngestionPipeline = ( + id: string, + patch: Operation[] +): Promise => { + const configOptions = { + headers: { 'Content-type': 'application/json-patch+json' }, + }; + + return APIClient.patch( + `/services/ingestionPipelines/${id}`, + patch, + configOptions + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 8c05e38e17d..2826de13546 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -11,6 +11,7 @@ * limitations under the License. */ +import { isUndefined } from 'lodash'; import React, { useState } from 'react'; import { INGESTION_SCHEDULER_INITIAL_VALUE, @@ -18,55 +19,104 @@ import { STEPS_FOR_ADD_INGESTION, } from '../../constants/ingestion.constant'; import { FilterPatternEnum } from '../../enums/filterPattern.enum'; +import { FormSubmitType } from '../../enums/form.enum'; import { + ConfigClass, CreateIngestionPipeline, - PipelineType, } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; +import { + FilterPattern, + IngestionPipeline, +} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { getCurrentDate, getCurrentUserId } from '../../utils/CommonUtils'; import SuccessScreen from '../common/success-screen/SuccessScreen'; import IngestionStepper from '../IngestionStepper/IngestionStepper.component'; -import { AddIngestionProps, PatternType } from './addIngestion.interface'; +import { AddIngestionProps } from './addIngestion.interface'; import ConfigureIngestion from './Steps/ConfigureIngestion'; import ScheduleInterval from './Steps/ScheduleInterval'; const AddIngestion = ({ + heading, + status, + pipelineType, + data, serviceData, serviceCategory, + showSuccessScreen = true, + onUpdateIngestion, + onSuccessSave, onAddIngestionSave, - handleAddIngestion, + handleCancelClick, handleViewServiceClick, }: AddIngestionProps) => { const [activeStepperStep, setActiveStepperStep] = useState(1); const [ingestionName] = useState( - `${serviceData.name}_${PipelineType.Metadata}` + data?.name ?? `${serviceData.name}_${pipelineType}` ); const [repeatFrequency, setRepeatFrequency] = useState( - INGESTION_SCHEDULER_INITIAL_VALUE + data?.airflowConfig.scheduleInterval ?? INGESTION_SCHEDULER_INITIAL_VALUE ); - const [startDate, setStartDate] = useState(getCurrentDate()); - const [endDate, setEndDate] = useState(''); + const [startDate, setStartDate] = useState( + data?.airflowConfig.startDate ?? getCurrentDate() + ); + const [endDate, setEndDate] = useState(data?.airflowConfig?.endDate ?? ''); - const [showDashboardFilter, setShowDashboardFilter] = useState(false); - const [showSchemaFilter, setShowSchemaFilter] = useState(false); - const [showTableFilter, setShowTableFilter] = useState(false); - const [showTopicFilter, setShowTopicFilter] = useState(false); - const [showChartFilter, setShowChartFilter] = useState(false); - const [includeView, setIncludeView] = useState(false); - const [enableDataProfiler, setEnableDataProfiler] = useState(true); - const [ingestSampleData, setIngestSampleData] = useState(true); + const [showDashboardFilter, setShowDashboardFilter] = useState( + !isUndefined( + (data?.source.sourceConfig.config as ConfigClass)?.dashboardFilterPattern + ) + ); + const [showSchemaFilter, setShowSchemaFilter] = useState( + !isUndefined( + (data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern + ) + ); + const [showTableFilter, setShowTableFilter] = useState( + !isUndefined( + (data?.source.sourceConfig.config as ConfigClass)?.tableFilterPattern + ) + ); + const [showTopicFilter, setShowTopicFilter] = useState( + !isUndefined( + (data?.source.sourceConfig.config as ConfigClass)?.topicFilterPattern + ) + ); + const [showChartFilter, setShowChartFilter] = useState( + !isUndefined( + (data?.source.sourceConfig.config as ConfigClass)?.chartFilterPattern + ) + ); + const [includeView, setIncludeView] = useState( + (data?.source.sourceConfig.config as ConfigClass)?.includeViews ?? false + ); + const [enableDataProfiler, setEnableDataProfiler] = useState( + (data?.source.sourceConfig.config as ConfigClass)?.enableDataProfiler ?? + true + ); + const [ingestSampleData, setIngestSampleData] = useState( + (data?.source.sourceConfig.config as ConfigClass)?.generateSampleData ?? + true + ); const [dashboardFilterPattern, setDashboardFilterPattern] = - useState(INITIAL_FILTER_PATTERN); - const [schemaFilterPattern, setSchemaFilterPattern] = useState( - INITIAL_FILTER_PATTERN + useState( + (data?.source.sourceConfig.config as ConfigClass) + ?.dashboardFilterPattern ?? INITIAL_FILTER_PATTERN + ); + const [schemaFilterPattern, setSchemaFilterPattern] = useState( + (data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern ?? + INITIAL_FILTER_PATTERN ); - const [tableFilterPattern, setTableFilterPattern] = useState( - INITIAL_FILTER_PATTERN + const [tableFilterPattern, setTableFilterPattern] = useState( + (data?.source.sourceConfig.config as ConfigClass)?.tableFilterPattern ?? + INITIAL_FILTER_PATTERN ); - const [topicFilterPattern, setTopicFilterPattern] = useState( - INITIAL_FILTER_PATTERN + const [topicFilterPattern, setTopicFilterPattern] = useState( + (data?.source.sourceConfig.config as ConfigClass)?.topicFilterPattern ?? + INITIAL_FILTER_PATTERN ); - const [chartFilterPattern, setChartFilterPattern] = useState( - INITIAL_FILTER_PATTERN + const [chartFilterPattern, setChartFilterPattern] = useState( + (data?.source.sourceConfig.config as ConfigClass)?.chartFilterPattern ?? + INITIAL_FILTER_PATTERN ); const getIncludeValue = (value: Array, type: FilterPatternEnum) => { @@ -74,24 +124,24 @@ const AddIngestion = ({ case FilterPatternEnum.DASHBOARD: setDashboardFilterPattern({ ...dashboardFilterPattern, - include: value, + includes: value, }); break; case FilterPatternEnum.SCHEMA: - setSchemaFilterPattern({ ...schemaFilterPattern, include: value }); + setSchemaFilterPattern({ ...schemaFilterPattern, includes: value }); break; case FilterPatternEnum.TABLE: - setTableFilterPattern({ ...tableFilterPattern, include: value }); + setTableFilterPattern({ ...tableFilterPattern, includes: value }); break; case FilterPatternEnum.TOPIC: - setTopicFilterPattern({ ...topicFilterPattern, include: value }); + setTopicFilterPattern({ ...topicFilterPattern, includes: value }); break; case FilterPatternEnum.CHART: - setChartFilterPattern({ ...topicFilterPattern, include: value }); + setChartFilterPattern({ ...topicFilterPattern, includes: value }); break; } @@ -101,24 +151,24 @@ const AddIngestion = ({ case FilterPatternEnum.DASHBOARD: setDashboardFilterPattern({ ...dashboardFilterPattern, - exclude: value, + excludes: value, }); break; case FilterPatternEnum.SCHEMA: - setSchemaFilterPattern({ ...schemaFilterPattern, exclude: value }); + setSchemaFilterPattern({ ...schemaFilterPattern, excludes: value }); break; case FilterPatternEnum.TABLE: - setTableFilterPattern({ ...tableFilterPattern, exclude: value }); + setTableFilterPattern({ ...tableFilterPattern, excludes: value }); break; case FilterPatternEnum.TOPIC: - setTopicFilterPattern({ ...topicFilterPattern, exclude: value }); + setTopicFilterPattern({ ...topicFilterPattern, excludes: value }); break; case FilterPatternEnum.CHART: - setChartFilterPattern({ ...topicFilterPattern, exclude: value }); + setChartFilterPattern({ ...topicFilterPattern, excludes: value }); break; } @@ -150,7 +200,7 @@ const AddIngestion = ({ }; const handleConfigureIngestionCancelClick = () => { - handleAddIngestion(false); + handleCancelClick(); }; const handleConfigureIngestionNextClick = () => { @@ -161,22 +211,22 @@ const AddIngestion = ({ setActiveStepperStep(1); }; - const getFilterPatternData = (data: PatternType) => { - const { include, exclude } = data; + const getFilterPatternData = (data: FilterPattern) => { + const { includes, excludes } = data; - return include.length === 0 && exclude.length === 0 + return isUndefined(includes) && isUndefined(excludes) ? undefined : { - includes: include.length > 0 ? include : undefined, - excludes: exclude.length > 0 ? exclude : undefined, + includes: includes && includes.length > 0 ? includes : undefined, + excludes: excludes && excludes.length > 0 ? excludes : undefined, }; }; - const handleScheduleIntervalDeployClick = () => { + const createNewIngestion = () => { const ingestionDetails: CreateIngestionPipeline = { airflowConfig: { startDate: startDate as unknown as Date, - endDate: startDate as unknown as Date, + endDate: endDate as unknown as Date, scheduleInterval: repeatFrequency, forceDeploy: true, }, @@ -186,7 +236,7 @@ const AddIngestion = ({ id: getCurrentUserId(), type: 'user', }, - pipelineType: PipelineType.Metadata, + pipelineType: pipelineType, service: { id: serviceData.id as string, type: serviceCategory.slice(0, -1), @@ -206,13 +256,64 @@ const AddIngestion = ({ }; onAddIngestionSave(ingestionDetails).then(() => { - setActiveStepperStep(3); + if (showSuccessScreen) { + setActiveStepperStep(3); + } else { + onSuccessSave?.(); + } }); }; + const updateIngestion = () => { + if (data) { + const updatedData: IngestionPipeline = { + ...data, + airflowConfig: { + ...data.airflowConfig, + startDate: startDate as unknown as Date, + endDate: endDate as unknown as Date, + scheduleInterval: repeatFrequency, + }, + source: { + ...data.source, + sourceConfig: { + config: { + ...(data.source.sourceConfig.config as ConfigClass), + enableDataProfiler: enableDataProfiler, + generateSampleData: ingestSampleData, + includeViews: includeView, + schemaFilterPattern: getFilterPatternData(schemaFilterPattern), + tableFilterPattern: getFilterPatternData(tableFilterPattern), + chartFilterPattern: getFilterPatternData(chartFilterPattern), + dashboardFilterPattern: getFilterPatternData( + dashboardFilterPattern + ), + topicFilterPattern: getFilterPatternData(topicFilterPattern), + }, + }, + }, + }; + + onUpdateIngestion && + onUpdateIngestion(updatedData, data, data.id as string, data.name).then( + () => { + onSuccessSave?.(); + } + ); + } + }; + + const handleScheduleIntervalDeployClick = () => { + if (status === FormSubmitType.ADD) { + createNewIngestion(); + } else { + updateIngestion(); + } + }; + return (
-
Add New Ingestion
+
{heading}
setEndDate(value)} handleRepeatFrequencyChange={(value: string) => setRepeatFrequency(value) } handleStartDateChange={(value: string) => setStartDate(value)} repeatFrequency={repeatFrequency} - startDate={startDate} + startDate={startDate as string} onBack={handleScheduleIntervalBackClick} onDeloy={handleScheduleIntervalDeployClick} /> )} - {activeStepperStep > 2 && ( + {activeStepperStep > 2 && handleViewServiceClick && ( { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx index 7ac504ac145..a4c8439ded3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx @@ -30,24 +30,24 @@ jest.mock('../../common/toggle-switch/ToggleSwitchV1', () => { const mockConfigureIngestion: ConfigureIngestionProps = { ingestionName: '', dashboardFilterPattern: { - include: [], - exclude: [], + includes: [], + excludes: [], }, chartFilterPattern: { - include: [], - exclude: [], + includes: [], + excludes: [], }, schemaFilterPattern: { - include: [], - exclude: [], + includes: [], + excludes: [], }, tableFilterPattern: { - include: [], - exclude: [], + includes: [], + excludes: [], }, topicFilterPattern: { - include: [], - exclude: [], + includes: [], + excludes: [], }, includeView: false, enableDataProfiler: false, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index cc307e10542..02eb18a3b55 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -52,24 +52,24 @@ const ConfigureIngestion = ({ handleShowFilter(value, FilterPatternEnum.SCHEMA) } - includePattern={schemaFilterPattern.include} + includePattern={schemaFilterPattern?.includes ?? []} type={FilterPatternEnum.SCHEMA} /> handleShowFilter(value, FilterPatternEnum.TABLE) } - includePattern={tableFilterPattern.include} + includePattern={tableFilterPattern?.includes ?? []} showSeparator={false} type={FilterPatternEnum.TABLE} /> @@ -80,24 +80,24 @@ const ConfigureIngestion = ({ handleShowFilter(value, FilterPatternEnum.DASHBOARD) } - includePattern={dashboardFilterPattern.include} + includePattern={dashboardFilterPattern.includes ?? []} type={FilterPatternEnum.DASHBOARD} /> handleShowFilter(value, FilterPatternEnum.CHART) } - includePattern={chartFilterPattern.include} + includePattern={chartFilterPattern.includes ?? []} showSeparator={false} type={FilterPatternEnum.CHART} /> @@ -108,13 +108,13 @@ const ConfigureIngestion = ({ return ( handleShowFilter(value, FilterPatternEnum.TOPIC) } - includePattern={topicFilterPattern.include} + includePattern={topicFilterPattern.includes ?? []} showSeparator={false} type={FilterPatternEnum.TOPIC} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts index 49053e6e58b..a6e9957a6d5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts @@ -12,31 +12,45 @@ */ import { FilterPatternEnum } from '../../enums/filterPattern.enum'; +import { FormSubmitType } from '../../enums/form.enum'; import { ServiceCategory } from '../../enums/service.enum'; import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; +import { + FilterPattern, + IngestionPipeline, + PipelineType, +} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { DataObj } from '../../interface/service.interface'; export interface AddIngestionProps { + pipelineType: PipelineType; + heading: string; + status: FormSubmitType; + data?: IngestionPipeline; serviceCategory: ServiceCategory; serviceData: DataObj; - handleAddIngestion: (value: boolean) => void; + showSuccessScreen?: boolean; + handleCancelClick: () => void; onAddIngestionSave: (ingestion: CreateIngestionPipeline) => Promise; - handleViewServiceClick: () => void; + onUpdateIngestion?: ( + data: IngestionPipeline, + oldData: IngestionPipeline, + id: string, + displayName: string, + triggerIngestion?: boolean + ) => Promise; + onSuccessSave?: () => void; + handleViewServiceClick?: () => void; } -export type PatternType = { - include: Array; - exclude: Array; -}; - export interface ConfigureIngestionProps { ingestionName: string; serviceCategory: ServiceCategory; - dashboardFilterPattern: PatternType; - schemaFilterPattern: PatternType; - tableFilterPattern: PatternType; - topicFilterPattern: PatternType; - chartFilterPattern: PatternType; + dashboardFilterPattern: FilterPattern; + schemaFilterPattern: FilterPattern; + tableFilterPattern: FilterPattern; + topicFilterPattern: FilterPattern; + chartFilterPattern: FilterPattern; includeView: boolean; enableDataProfiler: boolean; ingestSampleData: boolean; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx index 46bfc574944..f7b2d27366b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx @@ -11,14 +11,16 @@ * limitations under the License. */ -import { isUndefined } from 'lodash'; +import { capitalize, isUndefined } from 'lodash'; import { LoadingState } from 'Models'; import React, { useState } from 'react'; import { useHistory } from 'react-router-dom'; import { getServiceDetailsPath, ROUTES } from '../../constants/constants'; import { STEPS_FOR_ADD_SERVICE } from '../../constants/services.const'; +import { FormSubmitType } from '../../enums/form.enum'; import { PageLayoutType } from '../../enums/layout.enum'; import { ServiceCategory } from '../../enums/service.enum'; +import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { ConfigData, DataObj, @@ -248,10 +250,13 @@ const AddService = ({
{addIngestion ? ( handleAddIngestion(false)} handleViewServiceClick={handleViewServiceClick} + heading={`Add ${capitalize(PipelineType.Metadata)} Ingestion`} + pipelineType={PipelineType.Metadata} serviceCategory={serviceCategory} serviceData={newServiceData as DataObj} + status={FormSubmitType.ADD} onAddIngestionSave={onAddIngestionSave} /> ) : ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx index 3ecc42959d4..432ae1cc763 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx @@ -15,37 +15,36 @@ import { faExclamationCircle } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import classNames from 'classnames'; import cronstrue from 'cronstrue'; -import { capitalize, isNil, lowerCase } from 'lodash'; -import React, { Fragment, useCallback, useState } from 'react'; +import { capitalize, isNil, isUndefined, lowerCase } from 'lodash'; +import React, { useCallback, useState } from 'react'; import { useAuthContext } from '../../authentication/auth-provider/AuthProvider'; import { PAGE_SIZE, TITLE_FOR_NON_ADMIN_ACTION, } from '../../constants/constants'; +import { FormSubmitType } from '../../enums/form.enum'; import { - AirflowPipeline, - ConfigClass, + IngestionPipeline, PipelineType, -} from '../../generated/operations/pipelines/airflowPipeline'; +} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { useAuth } from '../../hooks/authHooks'; import { isEven } from '../../utils/CommonUtils'; -import { getAirflowPipelineTypes } from '../../utils/ServiceUtils'; import { showInfoToast } from '../../utils/ToastUtils'; +import AddIngestion from '../AddIngestion/AddIngestion.component'; import { Button } from '../buttons/Button/Button'; import NextPrevious from '../common/next-previous/NextPrevious'; import NonAdminAction from '../common/non-admin-action/NonAdminAction'; import PopOver from '../common/popover/PopOver'; import Searchbar from '../common/searchbar/Searchbar'; -import IngestionModal from '../IngestionModal/IngestionModal.component'; import Loader from '../Loader/Loader'; -import ConfirmationModal from '../Modals/ConfirmationModal/ConfirmationModal'; -import { Props } from './ingestion.interface'; +import EntityDeleteModal from '../Modals/EntityDeleteModal/EntityDeleteModal'; +import { IngestionProps, ModifiedConfig } from './ingestion.interface'; -const Ingestion: React.FC = ({ - serviceType = '', +const Ingestion: React.FC = ({ + serviceDetails, serviceName, + serviceCategory, ingestionList, - serviceList, isRequiredDetailsAvailable, deleteIngestion, triggerIngestion, @@ -54,42 +53,45 @@ const Ingestion: React.FC = ({ paging, pagingHandler, currrentPage, -}: Props) => { +}: IngestionProps) => { const { isAdminUser } = useAuth(); const { isAuthDisabled } = useAuthContext(); const [searchText, setSearchText] = useState(''); const [currTriggerId, setCurrTriggerId] = useState({ id: '', state: '' }); - const [isAdding, setIsAdding] = useState(false); - const [isUpdating, setIsUpdating] = useState(false); + const [showIngestionForm, setShowIngestionForm] = useState(false); const [isConfirmationModalOpen, setIsConfirmationModalOpen] = useState(false); const [deleteSelection, setDeleteSelection] = useState({ id: '', name: '', state: '', }); - const [updateSelection, setUpdateSelection] = useState({ - id: '', - name: '', - state: '', - ingestion: {} as AirflowPipeline, - }); + const [updateSelection, setUpdateSelection] = useState(); const noConnectionMsg = `${serviceName} doesn't have connection details filled in. Please add the details before scheduling an ingestion job.`; const handleSearchAction = (searchValue: string) => { setSearchText(searchValue); }; - const getAirflowPipelineTypeOption = (): PipelineType[] => { - const types = getAirflowPipelineTypes(serviceType) || []; + const getIngestionPipelineTypeOption = (): PipelineType[] => { + if (ingestionList.length > 0) { + const ingestion = ingestionList[0]?.source?.serviceConnection + ?.config as ModifiedConfig; + const pipelineType = []; + ingestion?.supportsMetadataExtraction && + pipelineType.push(PipelineType.Metadata); + ingestion?.supportsUsageExtraction && + pipelineType.push(PipelineType.Usage); - return ingestionList.reduce((prev, curr) => { - const index = prev.indexOf(curr.pipelineType); - if (index > -1) { - prev.splice(index, 1); - } + return pipelineType.reduce((prev, curr) => { + if (ingestionList.find((d) => d.pipelineType === curr)) { + return prev; + } else { + return [...prev, curr]; + } + }, [] as PipelineType[]); + } - return prev; - }, types); + return [PipelineType.Metadata, PipelineType.Usage]; }; const handleTriggerIngestion = (id: string, displayName: string) => { @@ -111,59 +113,14 @@ const Ingestion: React.FC = ({ }); }; - const handleUpdate = (ingestion: AirflowPipeline) => { - setUpdateSelection({ - id: ingestion.id as string, - name: ingestion.name, - state: '', - ingestion: ingestion, - }); - setIsUpdating(true); + const handleUpdate = (ingestion: IngestionPipeline) => { + setUpdateSelection(ingestion); + setShowIngestionForm(true); }; const handleCancelUpdate = () => { - setUpdateSelection({ - id: '', - name: '', - state: '', - ingestion: {} as AirflowPipeline, - }); - setIsUpdating(false); - }; - const handleUpdateIngestion = ( - data: AirflowPipeline, - triggerIngestion?: boolean - ) => { - const { pipelineConfig } = updateSelection.ingestion; - const updatedData: AirflowPipeline = { - ...updateSelection.ingestion, - pipelineConfig: { - ...pipelineConfig, - config: { - ...(pipelineConfig.config as ConfigClass), - - ...(data.pipelineConfig.config as ConfigClass), - }, - }, - scheduleInterval: data.scheduleInterval, - }; - setUpdateSelection((prev) => ({ ...prev, state: 'waiting' })); - updateIngestion( - updatedData as AirflowPipeline, - updateSelection.ingestion, - updateSelection.id, - updateSelection.name, - triggerIngestion - ) - .then(() => { - setTimeout(() => { - setUpdateSelection((prev) => ({ ...prev, state: 'success' })); - handleCancelUpdate(); - }, 500); - }) - .catch(() => { - handleCancelUpdate(); - }); + setUpdateSelection(undefined); + setShowIngestionForm(false); }; const handleDelete = (id: string, displayName: string) => { @@ -190,12 +147,12 @@ const Ingestion: React.FC = ({ }; const handleAddIngestionClick = () => { - if (!getAirflowPipelineTypeOption().length) { + if (!getIngestionPipelineTypeOption().length) { showInfoToast( `${serviceName} already has all the supported ingestion jobs added.` ); } else { - setIsAdding(true); + setShowIngestionForm(true); } }; @@ -211,53 +168,59 @@ const Ingestion: React.FC = ({ : ingestionList; }, [searchText, ingestionList]); - const getStatuses = (ingestion: AirflowPipeline) => { - const lastFiveIngestions = ingestion.pipelineStatuses - ?.sort((a, b) => { - // Turn your strings into millis, and then subtract them - // to get a value that is either negative, positive, or zero. - const date1 = new Date(a.startDate || ''); - const date2 = new Date(b.startDate || ''); + /* eslint-disable max-len */ + // TODO:- once api support status we need below function + // const getStatuses = (ingestion: AirflowPipeline) => { + // const lastFiveIngestions = ingestion.pipelineStatuses + // ?.sort((a, b) => { + // // Turn your strings into millis, and then subtract them + // // to get a value that is either negative, positive, or zero. + // const date1 = new Date(a.startDate || ''); + // const date2 = new Date(b.startDate || ''); - return date1.getTime() - date2.getTime(); - }) - .slice(Math.max(ingestion.pipelineStatuses.length - 5, 0)); + // return date1.getTime() - date2.getTime(); + // }) + // .slice(Math.max(ingestion.pipelineStatuses.length - 5, 0)); - return lastFiveIngestions?.map((r, i) => { - return ( - - {r.startDate ? ( -

Start Date: {new Date(r.startDate).toUTCString()}

- ) : null} - {r.endDate ? ( -

End Date: {new Date(r.endDate).toUTCString()}

- ) : null} -
- } - key={i} - position="bottom" - theme="light" - trigger="mouseenter"> - {i === lastFiveIngestions.length - 1 ? ( -

- {capitalize(r.state)} -

- ) : ( -

- )} - - ); - }); - }; + // return lastFiveIngestions?.map((r, i) => { + // return ( + // + // {r.startDate ? ( + //

Start Date: {new Date(r.startDate).toUTCString()}

+ // ) : null} + // {r.endDate ? ( + //

End Date: {new Date(r.endDate).toUTCString()}

+ // ) : null} + //
+ // } + // key={i} + // position="bottom" + // theme="light" + // trigger="mouseenter"> + // {i === lastFiveIngestions.length - 1 ? ( + //

+ // {capitalize(r.state)} + //

+ // ) : ( + //

+ // )} + // + // ); + // }); + // }; - return ( - -

+ /* eslint-enable max-len */ + + const getIngestionTab = () => { + return ( +
{!isRequiredDetailsAvailable && (
@@ -283,10 +246,12 @@ const Ingestion: React.FC = ({ position="bottom" title={TITLE_FOR_NON_ADMIN_ACTION}>
- } - position="bottom" - theme="light" - trigger="mouseenter"> - {ingestion.scheduleInterval} - + {ingestion.airflowConfig?.scheduleInterval ? ( + + {cronstrue.toString( + ingestion.airflowConfig.scheduleInterval || '', + { + use24HourTimeFormat: true, + verbose: true, + } + )} +
+ } + position="bottom" + theme="light" + trigger="mouseenter"> + + {ingestion.airflowConfig.scheduleInterval ?? '--'} + + + ) : ( + -- + )} -
{getStatuses(ingestion)}
+ {/* TODO:- update this once api support pipeline status */} + {/*
{getStatuses(ingestion)}
*/} - {/* - {ingestion.nextExecutionDate || '--'} - */} + = ({ data-testid="edit" disabled={!isRequiredDetailsAvailable} onClick={() => handleUpdate(ingestion)}> - {updateSelection.id === ingestion.id ? ( - updateSelection.state === 'success' ? ( - - ) : ( - - ) - ) : ( - 'Edit' - )} + Edit
)}
- {isAdding ? ( - { - setIsAdding(false); - addIngestion(data, triggerIngestion); - }} - header="Add Ingestion" - ingestionList={ingestionList} - ingestionTypes={getAirflowPipelineTypeOption()} - name="" - service={serviceName} - serviceList={serviceList.map((s) => ({ - name: s.name, - serviceType: s.serviceType, - }))} - serviceType={serviceType} - type="" - onCancel={() => setIsAdding(false)} - /> - ) : null} - {isUpdating ? ( - {`Edit ${updateSelection.name}`}} - ingestionList={ingestionList} - ingestionTypes={getAirflowPipelineTypes(serviceType) || []} - selectedIngestion={updateSelection.ingestion} - service={serviceName} - serviceList={serviceList.map((s) => ({ - name: s.name, - serviceType: s.serviceType, - }))} - serviceType={serviceType} - updateIngestion={(data, triggerIngestion) => { - setIsUpdating(false); - handleUpdateIngestion(data, triggerIngestion); - }} - onCancel={() => handleCancelUpdate()} - /> - ) : null} + ); + }; + + const getIngestionForm = () => { + const type = getIngestionPipelineTypeOption(); + let heading = ''; + + if (isUndefined(updateSelection)) { + heading = `Add ${capitalize(type[0])} Ingestion`; + } else { + heading = `Edit ${capitalize(updateSelection.pipelineType)} Ingestion`; + } + + return ( +
+
+ +
+
+ ); + }; + + return ( +
+ {showIngestionForm ? getIngestionForm() : getIngestionTab()} + {isConfirmationModalOpen && ( - - ) : deleteSelection.state === 'success' ? ( - - ) : ( - 'Delete' - ) - } - header="Are you sure?" + handleDelete(deleteSelection.id, deleteSelection.name) } /> )} - +
); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts index da3516e37a8..09cc17bfd1a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts @@ -15,51 +15,82 @@ export const mockIngestionWorkFlow = { data: { data: [ { - id: '3dae41fd-0469-483b-9d48-622577f2e075', - name: 'test1', - displayName: 'Test1', + id: 'c804ec51-8fcf-4040-b830-5d967c4cbf49', + name: 'test3_metadata', + displayName: 'test3_metadata', + pipelineType: 'metadata', owner: { - id: '360d5fd9-ba6b-4205-a92c-8eb98286c1c5', + id: 'fd96fdc7-a159-4802-84be-33c68d8b7e07', type: 'user', - name: 'Aaron Johnson', - href: 'http://localhost:8585/api/v1/users/360d5fd9-ba6b-4205-a92c-8eb98286c1c5', + name: 'anonymous', + fullyQualifiedName: 'anonymous', + deleted: false, + href: 'http://localhost:8585/api/v1/users/fd96fdc7-a159-4802-84be-33c68d8b7e07', }, - fullyQualifiedName: 'bigquery.test1', - ingestionType: 'bigquery', - tags: [], - forceDeploy: true, - pauseWorkflow: false, - concurrency: 1, - startDate: '2021-11-24', - endDate: '2022-11-25', - workflowTimezone: 'UTC', - retries: 1, - retryDelay: 300, - workflowCatchup: false, - scheduleInterval: '0 12 * * *', - workflowTimeout: 60, - connectorConfig: { - username: 'test', - password: 'test', - host: 'http://localhost:3000/ingestion', - database: 'mysql', - includeViews: true, - enableDataProfiler: false, - includeFilterPattern: [], - excludeFilterPattern: [], + fullyQualifiedName: 'test3.test3_metadata', + source: { + type: 'bigquery', + serviceName: 'test3', + serviceConnection: { + config: { + type: 'BigQuery', + scheme: 'bigquery', + hostPort: 'bigquery.googleapis.com', + partitionField: '_PARTITIONTIME', + partitionQuery: 'select * from {}.{} WHERE {} = "{}" LIMIT 1000', + tagCategoryName: 'BigqueryPolicyTags', + connectionOptions: {}, + connectionArguments: {}, + enablePolicyTagImport: true, + partitionQueryDuration: 1, + supportsUsageExtraction: true, + supportsMetadataExtraction: true, + }, + }, + sourceConfig: { + config: { + includeViews: false, + enableDataProfiler: true, + generateSampleData: true, + }, + }, + }, + openMetadataServerConnection: { + hostPort: 'http://localhost:8585/api', + authProvider: 'no-auth', + apiVersion: 'v1', + }, + airflowConfig: { + forceDeploy: true, + pausePipeline: false, + concurrency: 1, + startDate: '2022-04-14', + endDate: '2022-04-14', + pipelineTimezone: 'UTC', + retries: 3, + retryDelay: 300, + pipelineCatchup: false, + scheduleInterval: '5 * * * *', + pipelineTimeout: 60, + maxActiveRuns: 1, + workflowTimeout: 60, + workflowDefaultView: 'tree', + workflowDefaultViewOrientation: 'LR', }, - ingestionStatuses: [], service: { - id: 'e7e34bc7-fc12-40d6-9478-a6297cdefe7a', + id: 'c68e904a-4262-4b58-84c1-8a986b4aa47d', type: 'databaseService', - name: 'bigquery', - description: 'BigQuery service used for shopify data', - href: 'http://localhost:8585/api/v1/services/databaseServices/e7e34bc7-fc12-40d6-9478-a6297cdefe7a', + name: 'test3', + fullyQualifiedName: 'test3', + description: '', + deleted: false, + href: 'http://localhost:8585/api/v1/services/databaseServices/c68e904a-4262-4b58-84c1-8a986b4aa47d', }, - href: 'http://localhost:8585/api/ingestion/3dae41fd-0469-483b-9d48-622577f2e075', + href: 'http://localhost:8585/api/v1/services/ingestionPipelines/c804ec51-8fcf-4040-b830-5d967c4cbf49', version: 0.1, - updatedAt: 1637736180218, + updatedAt: 1649941364738, updatedBy: 'anonymous', + deleted: false, }, ], paging: { @@ -69,25 +100,37 @@ export const mockIngestionWorkFlow = { }; export const mockService = { - data: { - data: [ - { - id: 'e7e34bc7-fc12-40d6-9478-a6297cdefe7a', - name: 'bigquery', - serviceType: 'BigQuery', - description: 'BigQuery service used for shopify data', - version: 0.1, - updatedAt: 1637734235276, - updatedBy: 'anonymous', - href: 'http://localhost:8585/api/v1/services/databaseServices/e7e34bc7-fc12-40d6-9478-a6297cdefe7a', - jdbc: { - driverClass: 'jdbc', - connectionUrl: 'jdbc://localhost', - }, - }, - ], - paging: { - total: 1, + id: 'c68e904a-4262-4b58-84c1-8a986b4aa47d', + name: 'test3', + serviceType: 'BigQuery', + description: '', + connection: { + config: { + type: 'BigQuery', + scheme: 'bigquery', + hostPort: 'bigquery.googleapis.com', + partitionField: '_PARTITIONTIME', + partitionQuery: 'select * from {}.{} WHERE {} = "{}" LIMIT 1000', + tagCategoryName: 'BigqueryPolicyTags', + connectionOptions: {}, + connectionArguments: {}, + enablePolicyTagImport: true, + partitionQueryDuration: 1, + supportsUsageExtraction: true, + supportsMetadataExtraction: true, }, }, + version: 0.1, + updatedAt: 1649941355557, + updatedBy: 'anonymous', + owner: { + id: 'fd96fdc7-a159-4802-84be-33c68d8b7e07', + type: 'user', + name: 'anonymous', + fullyQualifiedName: 'anonymous', + deleted: false, + href: 'http://localhost:8585/api/v1/users/fd96fdc7-a159-4802-84be-33c68d8b7e07', + }, + href: 'http://localhost:8585/api/v1/services/databaseServices/c68e904a-4262-4b58-84c1-8a986b4aa47d', + deleted: false, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx index 2765aacd4aa..da491d683e1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx @@ -20,14 +20,15 @@ import { } from '@testing-library/react'; import React from 'react'; import { MemoryRouter } from 'react-router'; -import { AirflowPipeline } from '../../generated/operations/pipelines/airflowPipeline'; +import { ServiceCategory } from '../../enums/service.enum'; +import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import Ingestion from './Ingestion.component'; -import { mockIngestionWorkFlow } from './Ingestion.mock'; +import { mockIngestionWorkFlow, mockService } from './Ingestion.mock'; jest.mock('../../authentication/auth-provider/AuthProvider', () => { return { useAuthContext: jest.fn(() => ({ - isAuthDisabled: false, + isAuthDisabled: true, isAuthenticated: true, isProtectedRoute: jest.fn().mockReturnValue(true), isTourRoute: jest.fn().mockReturnValue(false), @@ -73,16 +74,16 @@ jest.mock('../common/next-previous/NextPrevious', () => { return jest.fn().mockImplementation(() =>
NextPrevious
); }); -jest.mock('../IngestionModal/IngestionModal.component', () => { +jest.mock('../AddIngestion/AddIngestion.component', () => { return jest .fn() .mockImplementation(() => ( -
IngestionModal
+
AddIngestion
)); }); -jest.mock('../Modals/ConfirmationModal/ConfirmationModal', () => { - return jest.fn().mockImplementation(() =>
ConfirmationModal
); +jest.mock('../Modals/EntityDeleteModal/EntityDeleteModal', () => { + return jest.fn().mockImplementation(() =>
EntityDeleteModal
); }); describe('Test Ingestion page', () => { @@ -94,10 +95,12 @@ describe('Test Ingestion page', () => { currrentPage={1} deleteIngestion={mockDeleteIngestion} ingestionList={ - mockIngestionWorkFlow.data.data as unknown as AirflowPipeline[] + mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[] } paging={mockPaging} pagingHandler={mockPaginghandler} + serviceCategory={ServiceCategory.DASHBOARD_SERVICES} + serviceDetails={mockService} serviceList={[]} triggerIngestion={mockTriggerIngestion} updateIngestion={mockFunction} @@ -127,15 +130,17 @@ describe('Test Ingestion page', () => { it('Table should render necessary fields', async () => { const { container } = render( { }; const { container } = render( { currrentPage={1} deleteIngestion={mockDeleteIngestion} ingestionList={ - mockIngestionWorkFlow.data.data as unknown as AirflowPipeline[] + mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[] } paging={mockPagingAfter} pagingHandler={mockPaginghandler} + serviceCategory={ServiceCategory.DASHBOARD_SERVICES} + serviceDetails={mockService} serviceList={[]} - serviceType="BigQuery" triggerIngestion={mockTriggerIngestion} updateIngestion={mockFunction} />, @@ -234,7 +242,7 @@ describe('Test Ingestion page', () => { const editButton = await findByTestId(container, 'edit'); fireEvent.click(editButton); - const ingestionModal = await findByTestId(container, 'ingestion-modal'); + const ingestionModal = await findByTestId(container, 'ingestion-form'); expect(ingestionModal).toBeInTheDocument(); }); @@ -253,12 +261,13 @@ describe('Test Ingestion page', () => { currrentPage={1} deleteIngestion={mockDeleteIngestion} ingestionList={ - mockIngestionWorkFlow.data.data as unknown as AirflowPipeline[] + mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[] } paging={mockPagingAfter} pagingHandler={mockPaginghandler} + serviceCategory={ServiceCategory.DASHBOARD_SERVICES} + serviceDetails={mockService} serviceList={[]} - serviceType="BigQuery" triggerIngestion={mockTriggerIngestion} updateIngestion={mockFunction} />, @@ -267,17 +276,6 @@ describe('Test Ingestion page', () => { } ); - // on click of add ingestion - const addIngestionButton = await findByTestId( - container, - 'add-new-ingestion-button' - ); - fireEvent.click(addIngestionButton); - - const ingestionModal = await findByTestId(container, 'ingestion-modal'); - - expect(ingestionModal).toBeInTheDocument(); - // on click of run button await act(async () => { @@ -293,7 +291,18 @@ describe('Test Ingestion page', () => { fireEvent.click(deleteButton); expect( - await findByText(container, /ConfirmationModal/i) + await findByText(container, /EntityDeleteModal/i) ).toBeInTheDocument(); + + // on click of add ingestion + const addIngestionButton = await findByTestId( + container, + 'add-new-ingestion-button' + ); + fireEvent.click(addIngestionButton); + + const ingestionModal = await findByTestId(container, 'ingestion-form'); + + expect(ingestionModal).toBeInTheDocument(); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts index 25d3d27c74e..a36c8b615d9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts @@ -11,11 +11,16 @@ * limitations under the License. */ -import { IngestionType } from '../../enums/service.enum'; +import { IngestionType, ServiceCategory } from '../../enums/service.enum'; +import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; import { DatabaseService } from '../../generated/entity/services/databaseService'; -import { AirflowPipeline } from '../../generated/operations/pipelines/airflowPipeline'; +import { + Connection, + IngestionPipeline, +} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { EntityReference } from '../../generated/type/entityReference'; import { Paging } from '../../generated/type/paging'; +import { DataObj } from '../../interface/service.interface'; export interface ConnectorConfig { username: string; @@ -48,23 +53,29 @@ export interface IngestionData { endDate?: string; } -export interface Props { - serviceType?: string; +export interface IngestionProps { + serviceDetails: DataObj; serviceName?: string; + serviceCategory: ServiceCategory; isRequiredDetailsAvailable: boolean; paging: Paging; - ingestionList: Array; + ingestionList: Array; serviceList: Array; currrentPage: number; pagingHandler: (value: string | number, activePage?: number) => void; deleteIngestion: (id: string, displayName: string) => Promise; triggerIngestion: (id: string, displayName: string) => Promise; - addIngestion: (data: AirflowPipeline, triggerIngestion?: boolean) => void; + addIngestion: (data: CreateIngestionPipeline) => Promise; updateIngestion: ( - data: AirflowPipeline, - oldData: AirflowPipeline, + data: IngestionPipeline, + oldData: IngestionPipeline, id: string, displayName: string, triggerIngestion?: boolean ) => Promise; } + +export interface ModifiedConfig extends Connection { + supportsMetadataExtraction: boolean; + supportsUsageExtraction: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/ingestion.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/ingestion.constant.ts index 6c1bfd79a66..0c2a0b6f81f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/ingestion.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/ingestion.constant.ts @@ -12,7 +12,7 @@ */ import { StepperStepType } from 'Models'; -import { PatternType } from '../components/AddIngestion/addIngestion.interface'; +import { FilterPattern } from '../generated/entity/services/ingestionPipelines/ingestionPipeline'; export const STEPS_FOR_ADD_INGESTION: Array = [ { name: 'Configure Ingestion', step: 1 }, @@ -21,7 +21,7 @@ export const STEPS_FOR_ADD_INGESTION: Array = [ export const INGESTION_SCHEDULER_INITIAL_VALUE = '5 * * * *'; -export const INITIAL_FILTER_PATTERN: PatternType = { - include: [], - exclude: [], +export const INITIAL_FILTER_PATTERN: FilterPattern = { + includes: [], + excludes: [], }; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.test.tsx index 0f227d7bdcc..466356b5026 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.test.tsx @@ -26,7 +26,7 @@ import ServicePage from './index'; jest.mock('../../authentication/auth-provider/AuthProvider', () => { return { useAuthContext: jest.fn(() => ({ - isAuthDisabled: false, + isAuthDisabled: true, isAuthenticated: true, isProtectedRoute: jest.fn().mockReturnValue(true), isTourRoute: jest.fn().mockReturnValue(false), @@ -77,8 +77,8 @@ const mockDatabase = { }, }; -jest.mock('../../axiosAPIs/airflowPipelineAPI', () => ({ - getAirflowPipelines: jest.fn().mockImplementation(() => +jest.mock('../../axiosAPIs/ingestionPipelineAPI', () => ({ + getIngestionPipelines: jest.fn().mockImplementation(() => Promise.resolve({ data: { data: [], @@ -86,10 +86,10 @@ jest.mock('../../axiosAPIs/airflowPipelineAPI', () => ({ }, }) ), - deleteAirflowPipelineById: jest.fn(), - addAirflowPipeline: jest.fn(), - triggerAirflowPipelineById: jest.fn(), - updateAirflowPipeline: jest.fn(), + deleteIngestionPipelineById: jest.fn(), + addIngestionPipeline: jest.fn(), + triggerIngestionPipelineById: jest.fn(), + updateIngestionPipeline: jest.fn(), })); jest.mock('../../axiosAPIs/serviceAPI', () => ({ diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx index 7bd338d2ea9..ec6d08216ff 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx @@ -25,13 +25,6 @@ import React, { Fragment, FunctionComponent, useEffect, useState } from 'react'; import { Link, useHistory, useParams } from 'react-router-dom'; import AppState from '../../AppState'; import { useAuthContext } from '../../authentication/auth-provider/AuthProvider'; -import { - addAirflowPipeline, - deleteAirflowPipelineById, - getAirflowPipelines, - triggerAirflowPipelineById, - updateAirflowPipeline, -} from '../../axiosAPIs/airflowPipelineAPI'; import { getDashboards } from '../../axiosAPIs/dashboardAPI'; import { getDatabases } from '../../axiosAPIs/databaseAPI'; import { @@ -40,6 +33,13 @@ import { postFeedById, postThread, } from '../../axiosAPIs/feedsAPI'; +import { + addIngestionPipeline, + deleteIngestionPipelineById, + getIngestionPipelines, + triggerIngestionPipelineById, + updateIngestionPipeline, +} from '../../axiosAPIs/ingestionPipelineAPI'; import { getPipelines } from '../../axiosAPIs/pipelineAPI'; import { getServiceByFQN, updateService } from '../../axiosAPIs/serviceAPI'; import { getTopics } from '../../axiosAPIs/topicsAPI'; @@ -70,20 +70,17 @@ import { TabSpecificField } from '../../enums/entity.enum'; import { SearchIndex } from '../../enums/search.enum'; import { ServiceCategory } from '../../enums/service.enum'; import { CreateThread } from '../../generated/api/feed/createThread'; +import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline'; import { Dashboard } from '../../generated/entity/data/dashboard'; import { Database } from '../../generated/entity/data/database'; import { Pipeline } from '../../generated/entity/data/pipeline'; import { Topic } from '../../generated/entity/data/topic'; import { DatabaseService } from '../../generated/entity/services/databaseService'; -import { - AirflowPipeline, - PipelineType, - Schema, -} from '../../generated/operations/pipelines/airflowPipeline'; +import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { EntityReference } from '../../generated/type/entityReference'; import { Paging } from '../../generated/type/paging'; import { useAuth } from '../../hooks/authHooks'; -import { ServiceDataObj } from '../../interface/service.interface'; +import { DataObj, ServiceDataObj } from '../../interface/service.interface'; import jsonData from '../../jsons/en'; import { getEntityMissingError, @@ -101,7 +98,6 @@ import { getCurrentServiceTab, getIsIngestionEnable, getServiceCategoryFromType, - isRequiredDetailsAvailableForIngestion, servicePageTabs, serviceTypeLogo, } from '../../utils/ServiceUtils'; @@ -137,7 +133,7 @@ const ServicePage: FunctionComponent = () => { const [isConnectionAvailable, setConnectionAvailable] = useState(true); const [isError, setIsError] = useState(false); - const [ingestions, setIngestions] = useState([]); + const [ingestions, setIngestions] = useState([]); const [serviceList] = useState>([]); const [ingestionPaging, setIngestionPaging] = useState({} as Paging); const [entityThread, setEntityThread] = useState([]); @@ -298,22 +294,9 @@ const ServicePage: FunctionComponent = () => { }); }; - const getSchemaFromType = (type: AirflowPipeline['pipelineType']) => { - switch (type) { - case PipelineType.Metadata: - return Schema.DatabaseServiceMetadataPipeline; - - case PipelineType.QueryUsage: - return Schema.DatabaseServiceQueryUsagePipeline; - - default: - return; - } - }; - const getAllIngestionWorkflows = (paging?: string) => { setIsloading(true); - getAirflowPipelines(['owner', 'pipelineStatuses'], serviceFQN, '', paging) + getIngestionPipelines(['owner'], serviceFQN, paging) .then((res) => { if (res.data.data) { setIngestions(res.data.data); @@ -339,7 +322,7 @@ const ServicePage: FunctionComponent = () => { displayName: string ): Promise => { return new Promise((resolve, reject) => { - triggerAirflowPipelineById(id) + triggerIngestionPipelineById(id) .then((res) => { if (res.data) { resolve(); @@ -367,7 +350,7 @@ const ServicePage: FunctionComponent = () => { displayName: string ): Promise => { return new Promise((resolve, reject) => { - deleteAirflowPipelineById(id) + deleteIngestionPipelineById(id) .then(() => { resolve(); getAllIngestionWorkflows(); @@ -383,8 +366,8 @@ const ServicePage: FunctionComponent = () => { }; const updateIngestion = ( - data: AirflowPipeline, - oldData: AirflowPipeline, + data: IngestionPipeline, + oldData: IngestionPipeline, id: string, displayName: string, triggerIngestion?: boolean @@ -392,7 +375,7 @@ const ServicePage: FunctionComponent = () => { const jsonPatch = compare(oldData, data); return new Promise((resolve, reject) => { - updateAirflowPipeline(id, jsonPatch) + updateIngestionPipeline(id, jsonPatch) .then(() => { resolve(); getAllIngestionWorkflows(); @@ -415,55 +398,33 @@ const ServicePage: FunctionComponent = () => { }); }; - const addIngestionWorkflowHandler = ( - data: AirflowPipeline, - triggerIngestion?: boolean - ) => { - setIsloading(true); - - const ingestionData: AirflowPipeline = { - ...data, - pipelineConfig: { - ...data.pipelineConfig, - schema: getSchemaFromType(data.pipelineType), - }, - service: { - id: serviceDetails?.id, - type: 'databaseService', - name: data.service.name, - } as EntityReference, - }; - - addAirflowPipeline(ingestionData) - .then((res: AxiosResponse) => { - if (res.data) { - const { id, displayName } = res.data; - setIsloading(false); - getAllIngestionWorkflows(); - if (triggerIngestion) { - triggerIngestionById(id, displayName).catch((error: AxiosError) => { - showErrorToast( - error, - `${jsonData['api-error-messages']['triggering-ingestion-error']} ${displayName}` - ); - }); + const onAddIngestionSave = (data: CreateIngestionPipeline) => { + return new Promise((resolve, reject) => { + return addIngestionPipeline(data) + .then((res: AxiosResponse) => { + if (res.data) { + getAllIngestionWorkflows(); + resolve(); + } else { + showErrorToast( + jsonData['api-error-messages']['create-ingestion-error'] + ); + reject(); } - } else { - showErrorToast(jsonData['api-error-messages']['add-ingestion-error']); - } - }) - .catch((error: AxiosError) => { - const message = getErrorText( - error, - jsonData['api-error-messages']['add-ingestion-error'] - ); - if (message.includes('Connection refused')) { - setConnectionAvailable(false); - } else { - showErrorToast(message); - } - setIsloading(false); - }); + }) + .catch((error: AxiosError) => { + const message = getErrorText( + error, + jsonData['api-error-messages']['create-ingestion-error'] + ); + if (message.includes('Connection refused')) { + setConnectionAvailable(false); + } else { + showErrorToast(message); + } + reject(); + }); + }); }; const handleConfigUpdate = ( @@ -1208,24 +1169,20 @@ const ServicePage: FunctionComponent = () => { )} {activeTab === 3 && ( -
+
{isConnectionAvailable ? (