mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-13 17:58:36 +00:00
Added new API integration for service page (#4158)
This commit is contained in:
Shailesh Parmar
GitHub
parent
e57ad12a36
commit
a6424b2713
@ -12,7 +12,9 @@
|
||||
*/
|
||||
|
||||
import { AxiosResponse } from 'axios';
|
||||
import { Operation } from 'fast-json-patch';
|
||||
import { CreateIngestionPipeline } from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
|
||||
import { getURLWithQueryFields } from '../utils/APIUtils';
|
||||
import APIClient from './index';
|
||||
|
||||
export const addIngestionPipeline = (
|
||||
@ -20,3 +22,45 @@ export const addIngestionPipeline = (
|
||||
): Promise<AxiosResponse> => {
|
||||
return APIClient.post('/services/ingestionPipelines', data);
|
||||
};
|
||||
|
||||
export const getIngestionPipelines = (
|
||||
arrQueryFields: Array<string>,
|
||||
serviceFilter?: string,
|
||||
paging?: string
|
||||
): Promise<AxiosResponse> => {
|
||||
const service = serviceFilter ? `service=${serviceFilter}` : '';
|
||||
const url = `${getURLWithQueryFields(
|
||||
'/services/ingestionPipelines',
|
||||
arrQueryFields,
|
||||
service
|
||||
)}${paging ? paging : ''}`;
|
||||
|
||||
return APIClient.get(url);
|
||||
};
|
||||
|
||||
export const triggerIngestionPipelineById = (
|
||||
id: string
|
||||
): Promise<AxiosResponse> => {
|
||||
return APIClient.post(`/services/ingestionPipelines/trigger/${id}`);
|
||||
};
|
||||
|
||||
export const deleteIngestionPipelineById = (
|
||||
id: string
|
||||
): Promise<AxiosResponse> => {
|
||||
return APIClient.delete(`/services/ingestionPipelines/${id}?hardDelete=true`);
|
||||
};
|
||||
|
||||
export const updateIngestionPipeline = (
|
||||
id: string,
|
||||
patch: Operation[]
|
||||
): Promise<AxiosResponse> => {
|
||||
const configOptions = {
|
||||
headers: { 'Content-type': 'application/json-patch+json' },
|
||||
};
|
||||
|
||||
return APIClient.patch(
|
||||
`/services/ingestionPipelines/${id}`,
|
||||
patch,
|
||||
configOptions
|
||||
);
|
||||
};
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { isUndefined } from 'lodash';
|
||||
import React, { useState } from 'react';
|
||||
import {
|
||||
INGESTION_SCHEDULER_INITIAL_VALUE,
|
||||
@ -18,55 +19,104 @@ import {
|
||||
STEPS_FOR_ADD_INGESTION,
|
||||
} from '../../constants/ingestion.constant';
|
||||
import { FilterPatternEnum } from '../../enums/filterPattern.enum';
|
||||
import { FormSubmitType } from '../../enums/form.enum';
|
||||
import {
|
||||
ConfigClass,
|
||||
CreateIngestionPipeline,
|
||||
PipelineType,
|
||||
} from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
|
||||
import {
|
||||
FilterPattern,
|
||||
IngestionPipeline,
|
||||
} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import { getCurrentDate, getCurrentUserId } from '../../utils/CommonUtils';
|
||||
import SuccessScreen from '../common/success-screen/SuccessScreen';
|
||||
import IngestionStepper from '../IngestionStepper/IngestionStepper.component';
|
||||
import { AddIngestionProps, PatternType } from './addIngestion.interface';
|
||||
import { AddIngestionProps } from './addIngestion.interface';
|
||||
import ConfigureIngestion from './Steps/ConfigureIngestion';
|
||||
import ScheduleInterval from './Steps/ScheduleInterval';
|
||||
|
||||
const AddIngestion = ({
|
||||
heading,
|
||||
status,
|
||||
pipelineType,
|
||||
data,
|
||||
serviceData,
|
||||
serviceCategory,
|
||||
showSuccessScreen = true,
|
||||
onUpdateIngestion,
|
||||
onSuccessSave,
|
||||
onAddIngestionSave,
|
||||
handleAddIngestion,
|
||||
handleCancelClick,
|
||||
handleViewServiceClick,
|
||||
}: AddIngestionProps) => {
|
||||
const [activeStepperStep, setActiveStepperStep] = useState(1);
|
||||
const [ingestionName] = useState(
|
||||
`${serviceData.name}_${PipelineType.Metadata}`
|
||||
data?.name ?? `${serviceData.name}_${pipelineType}`
|
||||
);
|
||||
const [repeatFrequency, setRepeatFrequency] = useState(
|
||||
INGESTION_SCHEDULER_INITIAL_VALUE
|
||||
data?.airflowConfig.scheduleInterval ?? INGESTION_SCHEDULER_INITIAL_VALUE
|
||||
);
|
||||
const [startDate, setStartDate] = useState(getCurrentDate());
|
||||
const [endDate, setEndDate] = useState('');
|
||||
const [startDate, setStartDate] = useState(
|
||||
data?.airflowConfig.startDate ?? getCurrentDate()
|
||||
);
|
||||
const [endDate, setEndDate] = useState(data?.airflowConfig?.endDate ?? '');
|
||||
|
||||
const [showDashboardFilter, setShowDashboardFilter] = useState(false);
|
||||
const [showSchemaFilter, setShowSchemaFilter] = useState(false);
|
||||
const [showTableFilter, setShowTableFilter] = useState(false);
|
||||
const [showTopicFilter, setShowTopicFilter] = useState(false);
|
||||
const [showChartFilter, setShowChartFilter] = useState(false);
|
||||
const [includeView, setIncludeView] = useState(false);
|
||||
const [enableDataProfiler, setEnableDataProfiler] = useState(true);
|
||||
const [ingestSampleData, setIngestSampleData] = useState(true);
|
||||
const [showDashboardFilter, setShowDashboardFilter] = useState(
|
||||
!isUndefined(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.dashboardFilterPattern
|
||||
)
|
||||
);
|
||||
const [showSchemaFilter, setShowSchemaFilter] = useState(
|
||||
!isUndefined(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern
|
||||
)
|
||||
);
|
||||
const [showTableFilter, setShowTableFilter] = useState(
|
||||
!isUndefined(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.tableFilterPattern
|
||||
)
|
||||
);
|
||||
const [showTopicFilter, setShowTopicFilter] = useState(
|
||||
!isUndefined(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.topicFilterPattern
|
||||
)
|
||||
);
|
||||
const [showChartFilter, setShowChartFilter] = useState(
|
||||
!isUndefined(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.chartFilterPattern
|
||||
)
|
||||
);
|
||||
const [includeView, setIncludeView] = useState(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.includeViews ?? false
|
||||
);
|
||||
const [enableDataProfiler, setEnableDataProfiler] = useState(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.enableDataProfiler ??
|
||||
true
|
||||
);
|
||||
const [ingestSampleData, setIngestSampleData] = useState(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.generateSampleData ??
|
||||
true
|
||||
);
|
||||
const [dashboardFilterPattern, setDashboardFilterPattern] =
|
||||
useState<PatternType>(INITIAL_FILTER_PATTERN);
|
||||
const [schemaFilterPattern, setSchemaFilterPattern] = useState<PatternType>(
|
||||
INITIAL_FILTER_PATTERN
|
||||
useState<FilterPattern>(
|
||||
(data?.source.sourceConfig.config as ConfigClass)
|
||||
?.dashboardFilterPattern ?? INITIAL_FILTER_PATTERN
|
||||
);
|
||||
const [schemaFilterPattern, setSchemaFilterPattern] = useState<FilterPattern>(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.schemaFilterPattern ??
|
||||
INITIAL_FILTER_PATTERN
|
||||
);
|
||||
const [tableFilterPattern, setTableFilterPattern] = useState<PatternType>(
|
||||
INITIAL_FILTER_PATTERN
|
||||
const [tableFilterPattern, setTableFilterPattern] = useState<FilterPattern>(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.tableFilterPattern ??
|
||||
INITIAL_FILTER_PATTERN
|
||||
);
|
||||
const [topicFilterPattern, setTopicFilterPattern] = useState<PatternType>(
|
||||
INITIAL_FILTER_PATTERN
|
||||
const [topicFilterPattern, setTopicFilterPattern] = useState<FilterPattern>(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.topicFilterPattern ??
|
||||
INITIAL_FILTER_PATTERN
|
||||
);
|
||||
const [chartFilterPattern, setChartFilterPattern] = useState<PatternType>(
|
||||
INITIAL_FILTER_PATTERN
|
||||
const [chartFilterPattern, setChartFilterPattern] = useState<FilterPattern>(
|
||||
(data?.source.sourceConfig.config as ConfigClass)?.chartFilterPattern ??
|
||||
INITIAL_FILTER_PATTERN
|
||||
);
|
||||
|
||||
const getIncludeValue = (value: Array<string>, type: FilterPatternEnum) => {
|
||||
@ -74,24 +124,24 @@ const AddIngestion = ({
|
||||
case FilterPatternEnum.DASHBOARD:
|
||||
setDashboardFilterPattern({
|
||||
...dashboardFilterPattern,
|
||||
include: value,
|
||||
includes: value,
|
||||
});
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.SCHEMA:
|
||||
setSchemaFilterPattern({ ...schemaFilterPattern, include: value });
|
||||
setSchemaFilterPattern({ ...schemaFilterPattern, includes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.TABLE:
|
||||
setTableFilterPattern({ ...tableFilterPattern, include: value });
|
||||
setTableFilterPattern({ ...tableFilterPattern, includes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.TOPIC:
|
||||
setTopicFilterPattern({ ...topicFilterPattern, include: value });
|
||||
setTopicFilterPattern({ ...topicFilterPattern, includes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.CHART:
|
||||
setChartFilterPattern({ ...topicFilterPattern, include: value });
|
||||
setChartFilterPattern({ ...topicFilterPattern, includes: value });
|
||||
|
||||
break;
|
||||
}
|
||||
@ -101,24 +151,24 @@ const AddIngestion = ({
|
||||
case FilterPatternEnum.DASHBOARD:
|
||||
setDashboardFilterPattern({
|
||||
...dashboardFilterPattern,
|
||||
exclude: value,
|
||||
excludes: value,
|
||||
});
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.SCHEMA:
|
||||
setSchemaFilterPattern({ ...schemaFilterPattern, exclude: value });
|
||||
setSchemaFilterPattern({ ...schemaFilterPattern, excludes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.TABLE:
|
||||
setTableFilterPattern({ ...tableFilterPattern, exclude: value });
|
||||
setTableFilterPattern({ ...tableFilterPattern, excludes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.TOPIC:
|
||||
setTopicFilterPattern({ ...topicFilterPattern, exclude: value });
|
||||
setTopicFilterPattern({ ...topicFilterPattern, excludes: value });
|
||||
|
||||
break;
|
||||
case FilterPatternEnum.CHART:
|
||||
setChartFilterPattern({ ...topicFilterPattern, exclude: value });
|
||||
setChartFilterPattern({ ...topicFilterPattern, excludes: value });
|
||||
|
||||
break;
|
||||
}
|
||||
@ -150,7 +200,7 @@ const AddIngestion = ({
|
||||
};
|
||||
|
||||
const handleConfigureIngestionCancelClick = () => {
|
||||
handleAddIngestion(false);
|
||||
handleCancelClick();
|
||||
};
|
||||
|
||||
const handleConfigureIngestionNextClick = () => {
|
||||
@ -161,22 +211,22 @@ const AddIngestion = ({
|
||||
setActiveStepperStep(1);
|
||||
};
|
||||
|
||||
const getFilterPatternData = (data: PatternType) => {
|
||||
const { include, exclude } = data;
|
||||
const getFilterPatternData = (data: FilterPattern) => {
|
||||
const { includes, excludes } = data;
|
||||
|
||||
return include.length === 0 && exclude.length === 0
|
||||
return isUndefined(includes) && isUndefined(excludes)
|
||||
? undefined
|
||||
: {
|
||||
includes: include.length > 0 ? include : undefined,
|
||||
excludes: exclude.length > 0 ? exclude : undefined,
|
||||
includes: includes && includes.length > 0 ? includes : undefined,
|
||||
excludes: excludes && excludes.length > 0 ? excludes : undefined,
|
||||
};
|
||||
};
|
||||
|
||||
const handleScheduleIntervalDeployClick = () => {
|
||||
const createNewIngestion = () => {
|
||||
const ingestionDetails: CreateIngestionPipeline = {
|
||||
airflowConfig: {
|
||||
startDate: startDate as unknown as Date,
|
||||
endDate: startDate as unknown as Date,
|
||||
endDate: endDate as unknown as Date,
|
||||
scheduleInterval: repeatFrequency,
|
||||
forceDeploy: true,
|
||||
},
|
||||
@ -186,7 +236,7 @@ const AddIngestion = ({
|
||||
id: getCurrentUserId(),
|
||||
type: 'user',
|
||||
},
|
||||
pipelineType: PipelineType.Metadata,
|
||||
pipelineType: pipelineType,
|
||||
service: {
|
||||
id: serviceData.id as string,
|
||||
type: serviceCategory.slice(0, -1),
|
||||
@ -206,13 +256,64 @@ const AddIngestion = ({
|
||||
};
|
||||
|
||||
onAddIngestionSave(ingestionDetails).then(() => {
|
||||
setActiveStepperStep(3);
|
||||
if (showSuccessScreen) {
|
||||
setActiveStepperStep(3);
|
||||
} else {
|
||||
onSuccessSave?.();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const updateIngestion = () => {
|
||||
if (data) {
|
||||
const updatedData: IngestionPipeline = {
|
||||
...data,
|
||||
airflowConfig: {
|
||||
...data.airflowConfig,
|
||||
startDate: startDate as unknown as Date,
|
||||
endDate: endDate as unknown as Date,
|
||||
scheduleInterval: repeatFrequency,
|
||||
},
|
||||
source: {
|
||||
...data.source,
|
||||
sourceConfig: {
|
||||
config: {
|
||||
...(data.source.sourceConfig.config as ConfigClass),
|
||||
enableDataProfiler: enableDataProfiler,
|
||||
generateSampleData: ingestSampleData,
|
||||
includeViews: includeView,
|
||||
schemaFilterPattern: getFilterPatternData(schemaFilterPattern),
|
||||
tableFilterPattern: getFilterPatternData(tableFilterPattern),
|
||||
chartFilterPattern: getFilterPatternData(chartFilterPattern),
|
||||
dashboardFilterPattern: getFilterPatternData(
|
||||
dashboardFilterPattern
|
||||
),
|
||||
topicFilterPattern: getFilterPatternData(topicFilterPattern),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
onUpdateIngestion &&
|
||||
onUpdateIngestion(updatedData, data, data.id as string, data.name).then(
|
||||
() => {
|
||||
onSuccessSave?.();
|
||||
}
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const handleScheduleIntervalDeployClick = () => {
|
||||
if (status === FormSubmitType.ADD) {
|
||||
createNewIngestion();
|
||||
} else {
|
||||
updateIngestion();
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div data-testid="add-ingestion-container">
|
||||
<h6 className="tw-heading tw-text-base">Add New Ingestion</h6>
|
||||
<h6 className="tw-heading tw-text-base">{heading}</h6>
|
||||
|
||||
<IngestionStepper
|
||||
activeStep={activeStepperStep}
|
||||
@ -254,20 +355,20 @@ const AddIngestion = ({
|
||||
|
||||
{activeStepperStep === 2 && (
|
||||
<ScheduleInterval
|
||||
endDate={endDate}
|
||||
endDate={endDate as string}
|
||||
handleEndDateChange={(value: string) => setEndDate(value)}
|
||||
handleRepeatFrequencyChange={(value: string) =>
|
||||
setRepeatFrequency(value)
|
||||
}
|
||||
handleStartDateChange={(value: string) => setStartDate(value)}
|
||||
repeatFrequency={repeatFrequency}
|
||||
startDate={startDate}
|
||||
startDate={startDate as string}
|
||||
onBack={handleScheduleIntervalBackClick}
|
||||
onDeloy={handleScheduleIntervalDeployClick}
|
||||
/>
|
||||
)}
|
||||
|
||||
{activeStepperStep > 2 && (
|
||||
{activeStepperStep > 2 && handleViewServiceClick && (
|
||||
<SuccessScreen
|
||||
handleViewServiceClick={handleViewServiceClick}
|
||||
name={ingestionName}
|
||||
|
||||
@ -13,7 +13,9 @@
|
||||
|
||||
import { findByTestId, findByText, render } from '@testing-library/react';
|
||||
import React from 'react';
|
||||
import { FormSubmitType } from '../../enums/form.enum';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import { DataObj } from '../../interface/service.interface';
|
||||
import AddIngestion from './AddIngestion.component';
|
||||
import { AddIngestionProps } from './addIngestion.interface';
|
||||
@ -22,10 +24,13 @@ const mockAddIngestionProps: AddIngestionProps = {
|
||||
serviceData: {
|
||||
name: 'serviceName',
|
||||
} as DataObj,
|
||||
handleAddIngestion: jest.fn(),
|
||||
handleCancelClick: jest.fn(),
|
||||
serviceCategory: ServiceCategory.DASHBOARD_SERVICES,
|
||||
onAddIngestionSave: jest.fn(),
|
||||
handleViewServiceClick: jest.fn(),
|
||||
pipelineType: PipelineType.Metadata,
|
||||
heading: 'add ingestion',
|
||||
status: FormSubmitType.ADD,
|
||||
};
|
||||
|
||||
jest.mock('./Steps/ConfigureIngestion', () => {
|
||||
|
||||
@ -30,24 +30,24 @@ jest.mock('../../common/toggle-switch/ToggleSwitchV1', () => {
|
||||
const mockConfigureIngestion: ConfigureIngestionProps = {
|
||||
ingestionName: '',
|
||||
dashboardFilterPattern: {
|
||||
include: [],
|
||||
exclude: [],
|
||||
includes: [],
|
||||
excludes: [],
|
||||
},
|
||||
chartFilterPattern: {
|
||||
include: [],
|
||||
exclude: [],
|
||||
includes: [],
|
||||
excludes: [],
|
||||
},
|
||||
schemaFilterPattern: {
|
||||
include: [],
|
||||
exclude: [],
|
||||
includes: [],
|
||||
excludes: [],
|
||||
},
|
||||
tableFilterPattern: {
|
||||
include: [],
|
||||
exclude: [],
|
||||
includes: [],
|
||||
excludes: [],
|
||||
},
|
||||
topicFilterPattern: {
|
||||
include: [],
|
||||
exclude: [],
|
||||
includes: [],
|
||||
excludes: [],
|
||||
},
|
||||
includeView: false,
|
||||
enableDataProfiler: false,
|
||||
|
||||
@ -52,24 +52,24 @@ const ConfigureIngestion = ({
|
||||
<Fragment>
|
||||
<FilterPattern
|
||||
checked={showSchemaFilter}
|
||||
excludePattern={schemaFilterPattern.exclude}
|
||||
excludePattern={schemaFilterPattern?.excludes ?? []}
|
||||
getExcludeValue={getExcludeValue}
|
||||
getIncludeValue={getIncludeValue}
|
||||
handleChecked={(value) =>
|
||||
handleShowFilter(value, FilterPatternEnum.SCHEMA)
|
||||
}
|
||||
includePattern={schemaFilterPattern.include}
|
||||
includePattern={schemaFilterPattern?.includes ?? []}
|
||||
type={FilterPatternEnum.SCHEMA}
|
||||
/>
|
||||
<FilterPattern
|
||||
checked={showTableFilter}
|
||||
excludePattern={tableFilterPattern.exclude}
|
||||
excludePattern={tableFilterPattern?.excludes ?? []}
|
||||
getExcludeValue={getExcludeValue}
|
||||
getIncludeValue={getIncludeValue}
|
||||
handleChecked={(value) =>
|
||||
handleShowFilter(value, FilterPatternEnum.TABLE)
|
||||
}
|
||||
includePattern={tableFilterPattern.include}
|
||||
includePattern={tableFilterPattern?.includes ?? []}
|
||||
showSeparator={false}
|
||||
type={FilterPatternEnum.TABLE}
|
||||
/>
|
||||
@ -80,24 +80,24 @@ const ConfigureIngestion = ({
|
||||
<Fragment>
|
||||
<FilterPattern
|
||||
checked={showDashboardFilter}
|
||||
excludePattern={dashboardFilterPattern.exclude}
|
||||
excludePattern={dashboardFilterPattern.excludes ?? []}
|
||||
getExcludeValue={getExcludeValue}
|
||||
getIncludeValue={getIncludeValue}
|
||||
handleChecked={(value) =>
|
||||
handleShowFilter(value, FilterPatternEnum.DASHBOARD)
|
||||
}
|
||||
includePattern={dashboardFilterPattern.include}
|
||||
includePattern={dashboardFilterPattern.includes ?? []}
|
||||
type={FilterPatternEnum.DASHBOARD}
|
||||
/>
|
||||
<FilterPattern
|
||||
checked={showChartFilter}
|
||||
excludePattern={chartFilterPattern.exclude}
|
||||
excludePattern={chartFilterPattern.excludes ?? []}
|
||||
getExcludeValue={getExcludeValue}
|
||||
getIncludeValue={getIncludeValue}
|
||||
handleChecked={(value) =>
|
||||
handleShowFilter(value, FilterPatternEnum.CHART)
|
||||
}
|
||||
includePattern={chartFilterPattern.include}
|
||||
includePattern={chartFilterPattern.includes ?? []}
|
||||
showSeparator={false}
|
||||
type={FilterPatternEnum.CHART}
|
||||
/>
|
||||
@ -108,13 +108,13 @@ const ConfigureIngestion = ({
|
||||
return (
|
||||
<FilterPattern
|
||||
checked={showTopicFilter}
|
||||
excludePattern={topicFilterPattern.exclude}
|
||||
excludePattern={topicFilterPattern.excludes ?? []}
|
||||
getExcludeValue={getExcludeValue}
|
||||
getIncludeValue={getIncludeValue}
|
||||
handleChecked={(value) =>
|
||||
handleShowFilter(value, FilterPatternEnum.TOPIC)
|
||||
}
|
||||
includePattern={topicFilterPattern.include}
|
||||
includePattern={topicFilterPattern.includes ?? []}
|
||||
showSeparator={false}
|
||||
type={FilterPatternEnum.TOPIC}
|
||||
/>
|
||||
|
||||
@ -12,31 +12,45 @@
|
||||
*/
|
||||
|
||||
import { FilterPatternEnum } from '../../enums/filterPattern.enum';
|
||||
import { FormSubmitType } from '../../enums/form.enum';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
|
||||
import {
|
||||
FilterPattern,
|
||||
IngestionPipeline,
|
||||
PipelineType,
|
||||
} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import { DataObj } from '../../interface/service.interface';
|
||||
|
||||
export interface AddIngestionProps {
|
||||
pipelineType: PipelineType;
|
||||
heading: string;
|
||||
status: FormSubmitType;
|
||||
data?: IngestionPipeline;
|
||||
serviceCategory: ServiceCategory;
|
||||
serviceData: DataObj;
|
||||
handleAddIngestion: (value: boolean) => void;
|
||||
showSuccessScreen?: boolean;
|
||||
handleCancelClick: () => void;
|
||||
onAddIngestionSave: (ingestion: CreateIngestionPipeline) => Promise<void>;
|
||||
handleViewServiceClick: () => void;
|
||||
onUpdateIngestion?: (
|
||||
data: IngestionPipeline,
|
||||
oldData: IngestionPipeline,
|
||||
id: string,
|
||||
displayName: string,
|
||||
triggerIngestion?: boolean
|
||||
) => Promise<void>;
|
||||
onSuccessSave?: () => void;
|
||||
handleViewServiceClick?: () => void;
|
||||
}
|
||||
|
||||
export type PatternType = {
|
||||
include: Array<string>;
|
||||
exclude: Array<string>;
|
||||
};
|
||||
|
||||
export interface ConfigureIngestionProps {
|
||||
ingestionName: string;
|
||||
serviceCategory: ServiceCategory;
|
||||
dashboardFilterPattern: PatternType;
|
||||
schemaFilterPattern: PatternType;
|
||||
tableFilterPattern: PatternType;
|
||||
topicFilterPattern: PatternType;
|
||||
chartFilterPattern: PatternType;
|
||||
dashboardFilterPattern: FilterPattern;
|
||||
schemaFilterPattern: FilterPattern;
|
||||
tableFilterPattern: FilterPattern;
|
||||
topicFilterPattern: FilterPattern;
|
||||
chartFilterPattern: FilterPattern;
|
||||
includeView: boolean;
|
||||
enableDataProfiler: boolean;
|
||||
ingestSampleData: boolean;
|
||||
|
||||
@ -11,14 +11,16 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { isUndefined } from 'lodash';
|
||||
import { capitalize, isUndefined } from 'lodash';
|
||||
import { LoadingState } from 'Models';
|
||||
import React, { useState } from 'react';
|
||||
import { useHistory } from 'react-router-dom';
|
||||
import { getServiceDetailsPath, ROUTES } from '../../constants/constants';
|
||||
import { STEPS_FOR_ADD_SERVICE } from '../../constants/services.const';
|
||||
import { FormSubmitType } from '../../enums/form.enum';
|
||||
import { PageLayoutType } from '../../enums/layout.enum';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import {
|
||||
ConfigData,
|
||||
DataObj,
|
||||
@ -248,10 +250,13 @@ const AddService = ({
|
||||
<div className="tw-form-container">
|
||||
{addIngestion ? (
|
||||
<AddIngestion
|
||||
handleAddIngestion={handleAddIngestion}
|
||||
handleCancelClick={() => handleAddIngestion(false)}
|
||||
handleViewServiceClick={handleViewServiceClick}
|
||||
heading={`Add ${capitalize(PipelineType.Metadata)} Ingestion`}
|
||||
pipelineType={PipelineType.Metadata}
|
||||
serviceCategory={serviceCategory}
|
||||
serviceData={newServiceData as DataObj}
|
||||
status={FormSubmitType.ADD}
|
||||
onAddIngestionSave={onAddIngestionSave}
|
||||
/>
|
||||
) : (
|
||||
|
||||
@ -15,37 +15,36 @@ import { faExclamationCircle } from '@fortawesome/free-solid-svg-icons';
|
||||
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
|
||||
import classNames from 'classnames';
|
||||
import cronstrue from 'cronstrue';
|
||||
import { capitalize, isNil, lowerCase } from 'lodash';
|
||||
import React, { Fragment, useCallback, useState } from 'react';
|
||||
import { capitalize, isNil, isUndefined, lowerCase } from 'lodash';
|
||||
import React, { useCallback, useState } from 'react';
|
||||
import { useAuthContext } from '../../authentication/auth-provider/AuthProvider';
|
||||
import {
|
||||
PAGE_SIZE,
|
||||
TITLE_FOR_NON_ADMIN_ACTION,
|
||||
} from '../../constants/constants';
|
||||
import { FormSubmitType } from '../../enums/form.enum';
|
||||
import {
|
||||
AirflowPipeline,
|
||||
ConfigClass,
|
||||
IngestionPipeline,
|
||||
PipelineType,
|
||||
} from '../../generated/operations/pipelines/airflowPipeline';
|
||||
} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import { useAuth } from '../../hooks/authHooks';
|
||||
import { isEven } from '../../utils/CommonUtils';
|
||||
import { getAirflowPipelineTypes } from '../../utils/ServiceUtils';
|
||||
import { showInfoToast } from '../../utils/ToastUtils';
|
||||
import AddIngestion from '../AddIngestion/AddIngestion.component';
|
||||
import { Button } from '../buttons/Button/Button';
|
||||
import NextPrevious from '../common/next-previous/NextPrevious';
|
||||
import NonAdminAction from '../common/non-admin-action/NonAdminAction';
|
||||
import PopOver from '../common/popover/PopOver';
|
||||
import Searchbar from '../common/searchbar/Searchbar';
|
||||
import IngestionModal from '../IngestionModal/IngestionModal.component';
|
||||
import Loader from '../Loader/Loader';
|
||||
import ConfirmationModal from '../Modals/ConfirmationModal/ConfirmationModal';
|
||||
import { Props } from './ingestion.interface';
|
||||
import EntityDeleteModal from '../Modals/EntityDeleteModal/EntityDeleteModal';
|
||||
import { IngestionProps, ModifiedConfig } from './ingestion.interface';
|
||||
|
||||
const Ingestion: React.FC<Props> = ({
|
||||
serviceType = '',
|
||||
const Ingestion: React.FC<IngestionProps> = ({
|
||||
serviceDetails,
|
||||
serviceName,
|
||||
serviceCategory,
|
||||
ingestionList,
|
||||
serviceList,
|
||||
isRequiredDetailsAvailable,
|
||||
deleteIngestion,
|
||||
triggerIngestion,
|
||||
@ -54,42 +53,45 @@ const Ingestion: React.FC<Props> = ({
|
||||
paging,
|
||||
pagingHandler,
|
||||
currrentPage,
|
||||
}: Props) => {
|
||||
}: IngestionProps) => {
|
||||
const { isAdminUser } = useAuth();
|
||||
const { isAuthDisabled } = useAuthContext();
|
||||
const [searchText, setSearchText] = useState('');
|
||||
const [currTriggerId, setCurrTriggerId] = useState({ id: '', state: '' });
|
||||
const [isAdding, setIsAdding] = useState<boolean>(false);
|
||||
const [isUpdating, setIsUpdating] = useState<boolean>(false);
|
||||
const [showIngestionForm, setShowIngestionForm] = useState(false);
|
||||
const [isConfirmationModalOpen, setIsConfirmationModalOpen] = useState(false);
|
||||
const [deleteSelection, setDeleteSelection] = useState({
|
||||
id: '',
|
||||
name: '',
|
||||
state: '',
|
||||
});
|
||||
const [updateSelection, setUpdateSelection] = useState({
|
||||
id: '',
|
||||
name: '',
|
||||
state: '',
|
||||
ingestion: {} as AirflowPipeline,
|
||||
});
|
||||
const [updateSelection, setUpdateSelection] = useState<IngestionPipeline>();
|
||||
const noConnectionMsg = `${serviceName} doesn't have connection details filled in. Please add the details before scheduling an ingestion job.`;
|
||||
|
||||
const handleSearchAction = (searchValue: string) => {
|
||||
setSearchText(searchValue);
|
||||
};
|
||||
|
||||
const getAirflowPipelineTypeOption = (): PipelineType[] => {
|
||||
const types = getAirflowPipelineTypes(serviceType) || [];
|
||||
const getIngestionPipelineTypeOption = (): PipelineType[] => {
|
||||
if (ingestionList.length > 0) {
|
||||
const ingestion = ingestionList[0]?.source?.serviceConnection
|
||||
?.config as ModifiedConfig;
|
||||
const pipelineType = [];
|
||||
ingestion?.supportsMetadataExtraction &&
|
||||
pipelineType.push(PipelineType.Metadata);
|
||||
ingestion?.supportsUsageExtraction &&
|
||||
pipelineType.push(PipelineType.Usage);
|
||||
|
||||
return ingestionList.reduce((prev, curr) => {
|
||||
const index = prev.indexOf(curr.pipelineType);
|
||||
if (index > -1) {
|
||||
prev.splice(index, 1);
|
||||
}
|
||||
return pipelineType.reduce((prev, curr) => {
|
||||
if (ingestionList.find((d) => d.pipelineType === curr)) {
|
||||
return prev;
|
||||
} else {
|
||||
return [...prev, curr];
|
||||
}
|
||||
}, [] as PipelineType[]);
|
||||
}
|
||||
|
||||
return prev;
|
||||
}, types);
|
||||
return [PipelineType.Metadata, PipelineType.Usage];
|
||||
};
|
||||
|
||||
const handleTriggerIngestion = (id: string, displayName: string) => {
|
||||
@ -111,59 +113,14 @@ const Ingestion: React.FC<Props> = ({
|
||||
});
|
||||
};
|
||||
|
||||
const handleUpdate = (ingestion: AirflowPipeline) => {
|
||||
setUpdateSelection({
|
||||
id: ingestion.id as string,
|
||||
name: ingestion.name,
|
||||
state: '',
|
||||
ingestion: ingestion,
|
||||
});
|
||||
setIsUpdating(true);
|
||||
const handleUpdate = (ingestion: IngestionPipeline) => {
|
||||
setUpdateSelection(ingestion);
|
||||
setShowIngestionForm(true);
|
||||
};
|
||||
|
||||
const handleCancelUpdate = () => {
|
||||
setUpdateSelection({
|
||||
id: '',
|
||||
name: '',
|
||||
state: '',
|
||||
ingestion: {} as AirflowPipeline,
|
||||
});
|
||||
setIsUpdating(false);
|
||||
};
|
||||
const handleUpdateIngestion = (
|
||||
data: AirflowPipeline,
|
||||
triggerIngestion?: boolean
|
||||
) => {
|
||||
const { pipelineConfig } = updateSelection.ingestion;
|
||||
const updatedData: AirflowPipeline = {
|
||||
...updateSelection.ingestion,
|
||||
pipelineConfig: {
|
||||
...pipelineConfig,
|
||||
config: {
|
||||
...(pipelineConfig.config as ConfigClass),
|
||||
|
||||
...(data.pipelineConfig.config as ConfigClass),
|
||||
},
|
||||
},
|
||||
scheduleInterval: data.scheduleInterval,
|
||||
};
|
||||
setUpdateSelection((prev) => ({ ...prev, state: 'waiting' }));
|
||||
updateIngestion(
|
||||
updatedData as AirflowPipeline,
|
||||
updateSelection.ingestion,
|
||||
updateSelection.id,
|
||||
updateSelection.name,
|
||||
triggerIngestion
|
||||
)
|
||||
.then(() => {
|
||||
setTimeout(() => {
|
||||
setUpdateSelection((prev) => ({ ...prev, state: 'success' }));
|
||||
handleCancelUpdate();
|
||||
}, 500);
|
||||
})
|
||||
.catch(() => {
|
||||
handleCancelUpdate();
|
||||
});
|
||||
setUpdateSelection(undefined);
|
||||
setShowIngestionForm(false);
|
||||
};
|
||||
|
||||
const handleDelete = (id: string, displayName: string) => {
|
||||
@ -190,12 +147,12 @@ const Ingestion: React.FC<Props> = ({
|
||||
};
|
||||
|
||||
const handleAddIngestionClick = () => {
|
||||
if (!getAirflowPipelineTypeOption().length) {
|
||||
if (!getIngestionPipelineTypeOption().length) {
|
||||
showInfoToast(
|
||||
`${serviceName} already has all the supported ingestion jobs added.`
|
||||
);
|
||||
} else {
|
||||
setIsAdding(true);
|
||||
setShowIngestionForm(true);
|
||||
}
|
||||
};
|
||||
|
||||
@ -211,53 +168,59 @@ const Ingestion: React.FC<Props> = ({
|
||||
: ingestionList;
|
||||
}, [searchText, ingestionList]);
|
||||
|
||||
const getStatuses = (ingestion: AirflowPipeline) => {
|
||||
const lastFiveIngestions = ingestion.pipelineStatuses
|
||||
?.sort((a, b) => {
|
||||
// Turn your strings into millis, and then subtract them
|
||||
// to get a value that is either negative, positive, or zero.
|
||||
const date1 = new Date(a.startDate || '');
|
||||
const date2 = new Date(b.startDate || '');
|
||||
/* eslint-disable max-len */
|
||||
// TODO:- once api support status we need below function
|
||||
// const getStatuses = (ingestion: AirflowPipeline) => {
|
||||
// const lastFiveIngestions = ingestion.pipelineStatuses
|
||||
// ?.sort((a, b) => {
|
||||
// // Turn your strings into millis, and then subtract them
|
||||
// // to get a value that is either negative, positive, or zero.
|
||||
// const date1 = new Date(a.startDate || '');
|
||||
// const date2 = new Date(b.startDate || '');
|
||||
|
||||
return date1.getTime() - date2.getTime();
|
||||
})
|
||||
.slice(Math.max(ingestion.pipelineStatuses.length - 5, 0));
|
||||
// return date1.getTime() - date2.getTime();
|
||||
// })
|
||||
// .slice(Math.max(ingestion.pipelineStatuses.length - 5, 0));
|
||||
|
||||
return lastFiveIngestions?.map((r, i) => {
|
||||
return (
|
||||
<PopOver
|
||||
html={
|
||||
<div className="tw-text-left">
|
||||
{r.startDate ? (
|
||||
<p>Start Date: {new Date(r.startDate).toUTCString()}</p>
|
||||
) : null}
|
||||
{r.endDate ? (
|
||||
<p>End Date: {new Date(r.endDate).toUTCString()}</p>
|
||||
) : null}
|
||||
</div>
|
||||
}
|
||||
key={i}
|
||||
position="bottom"
|
||||
theme="light"
|
||||
trigger="mouseenter">
|
||||
{i === lastFiveIngestions.length - 1 ? (
|
||||
<p
|
||||
className={`tw-h-5 tw-w-16 tw-rounded-sm tw-bg-status-${r.state} tw-mr-1 tw-px-1 tw-text-white tw-text-center`}>
|
||||
{capitalize(r.state)}
|
||||
</p>
|
||||
) : (
|
||||
<p
|
||||
className={`tw-w-4 tw-h-5 tw-rounded-sm tw-bg-status-${r.state} tw-mr-1`}
|
||||
/>
|
||||
)}
|
||||
</PopOver>
|
||||
);
|
||||
});
|
||||
};
|
||||
// return lastFiveIngestions?.map((r, i) => {
|
||||
// return (
|
||||
// <PopOver
|
||||
// html={
|
||||
// <div className="tw-text-left">
|
||||
// {r.startDate ? (
|
||||
// <p>Start Date: {new Date(r.startDate).toUTCString()}</p>
|
||||
// ) : null}
|
||||
// {r.endDate ? (
|
||||
// <p>End Date: {new Date(r.endDate).toUTCString()}</p>
|
||||
// ) : null}
|
||||
// </div>
|
||||
// }
|
||||
// key={i}
|
||||
// position="bottom"
|
||||
// theme="light"
|
||||
// trigger="mouseenter">
|
||||
// {i === lastFiveIngestions.length - 1 ? (
|
||||
// <p
|
||||
// className={`tw-h-5 tw-w-16 tw-rounded-sm tw-bg-status-${r.state} tw-mr-1 tw-px-1 tw-text-white tw-text-center`}>
|
||||
// {capitalize(r.state)}
|
||||
// </p>
|
||||
// ) : (
|
||||
// <p
|
||||
// className={`tw-w-4 tw-h-5 tw-rounded-sm tw-bg-status-${r.state} tw-mr-1`}
|
||||
// />
|
||||
// )}
|
||||
// </PopOver>
|
||||
// );
|
||||
// });
|
||||
// };
|
||||
|
||||
return (
|
||||
<Fragment>
|
||||
<div className="tw-px-4" data-testid="ingestion-container">
|
||||
/* eslint-enable max-len */
|
||||
|
||||
const getIngestionTab = () => {
|
||||
return (
|
||||
<div
|
||||
className="tw-px-4 tw-mt-4"
|
||||
data-testid="ingestion-details-container">
|
||||
<div className="tw-flex">
|
||||
{!isRequiredDetailsAvailable && (
|
||||
<div className="tw-rounded tw-bg-error-lite tw-text-error tw-font-medium tw-px-4 tw-py-1 tw-mb-4 tw-flex tw-items-center tw-gap-1">
|
||||
@ -283,10 +246,12 @@ const Ingestion: React.FC<Props> = ({
|
||||
position="bottom"
|
||||
title={TITLE_FOR_NON_ADMIN_ACTION}>
|
||||
<Button
|
||||
className={classNames('tw-h-8 tw-rounded tw-mb-2', {
|
||||
'tw-opacity-40': !isAdminUser && !isAuthDisabled,
|
||||
})}
|
||||
className={classNames('tw-h-8 tw-rounded tw-mb-2')}
|
||||
data-testid="add-new-ingestion-button"
|
||||
disabled={
|
||||
getIngestionPipelineTypeOption().length === 0 ||
|
||||
(!isAdminUser && !isAuthDisabled)
|
||||
}
|
||||
size="small"
|
||||
theme="primary"
|
||||
variant="contained"
|
||||
@ -308,7 +273,6 @@ const Ingestion: React.FC<Props> = ({
|
||||
<th className="tableHead-cell">Type</th>
|
||||
<th className="tableHead-cell">Schedule</th>
|
||||
<th className="tableHead-cell">Recent Runs</th>
|
||||
{/* <th className="tableHead-cell">Next Run</th> */}
|
||||
<th className="tableHead-cell">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@ -323,30 +287,35 @@ const Ingestion: React.FC<Props> = ({
|
||||
<td className="tableBody-cell">{ingestion.name}</td>
|
||||
<td className="tableBody-cell">{ingestion.pipelineType}</td>
|
||||
<td className="tableBody-cell">
|
||||
<PopOver
|
||||
html={
|
||||
<div>
|
||||
{cronstrue.toString(
|
||||
ingestion.scheduleInterval || '',
|
||||
{
|
||||
use24HourTimeFormat: true,
|
||||
verbose: true,
|
||||
}
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
position="bottom"
|
||||
theme="light"
|
||||
trigger="mouseenter">
|
||||
<span>{ingestion.scheduleInterval}</span>
|
||||
</PopOver>
|
||||
{ingestion.airflowConfig?.scheduleInterval ? (
|
||||
<PopOver
|
||||
html={
|
||||
<div>
|
||||
{cronstrue.toString(
|
||||
ingestion.airflowConfig.scheduleInterval || '',
|
||||
{
|
||||
use24HourTimeFormat: true,
|
||||
verbose: true,
|
||||
}
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
position="bottom"
|
||||
theme="light"
|
||||
trigger="mouseenter">
|
||||
<span>
|
||||
{ingestion.airflowConfig.scheduleInterval ?? '--'}
|
||||
</span>
|
||||
</PopOver>
|
||||
) : (
|
||||
<span>--</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="tableBody-cell">
|
||||
<div className="tw-flex">{getStatuses(ingestion)}</div>
|
||||
{/* TODO:- update this once api support pipeline status */}
|
||||
{/* <div className="tw-flex">{getStatuses(ingestion)}</div> */}
|
||||
</td>
|
||||
{/* <td className="tableBody-cell">
|
||||
{ingestion.nextExecutionDate || '--'}
|
||||
</td> */}
|
||||
|
||||
<td className="tableBody-cell">
|
||||
<NonAdminAction
|
||||
position="bottom"
|
||||
@ -376,15 +345,7 @@ const Ingestion: React.FC<Props> = ({
|
||||
data-testid="edit"
|
||||
disabled={!isRequiredDetailsAvailable}
|
||||
onClick={() => handleUpdate(ingestion)}>
|
||||
{updateSelection.id === ingestion.id ? (
|
||||
updateSelection.state === 'success' ? (
|
||||
<FontAwesomeIcon icon="check" />
|
||||
) : (
|
||||
<Loader size="small" type="default" />
|
||||
)
|
||||
) : (
|
||||
'Edit'
|
||||
)}
|
||||
Edit
|
||||
</button>
|
||||
<button
|
||||
className="link-text tw-mr-2"
|
||||
@ -436,68 +397,60 @@ const Ingestion: React.FC<Props> = ({
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{isAdding ? (
|
||||
<IngestionModal
|
||||
addIngestion={(data, triggerIngestion) => {
|
||||
setIsAdding(false);
|
||||
addIngestion(data, triggerIngestion);
|
||||
}}
|
||||
header="Add Ingestion"
|
||||
ingestionList={ingestionList}
|
||||
ingestionTypes={getAirflowPipelineTypeOption()}
|
||||
name=""
|
||||
service={serviceName}
|
||||
serviceList={serviceList.map((s) => ({
|
||||
name: s.name,
|
||||
serviceType: s.serviceType,
|
||||
}))}
|
||||
serviceType={serviceType}
|
||||
type=""
|
||||
onCancel={() => setIsAdding(false)}
|
||||
/>
|
||||
) : null}
|
||||
{isUpdating ? (
|
||||
<IngestionModal
|
||||
isUpdating
|
||||
header={<span>{`Edit ${updateSelection.name}`}</span>}
|
||||
ingestionList={ingestionList}
|
||||
ingestionTypes={getAirflowPipelineTypes(serviceType) || []}
|
||||
selectedIngestion={updateSelection.ingestion}
|
||||
service={serviceName}
|
||||
serviceList={serviceList.map((s) => ({
|
||||
name: s.name,
|
||||
serviceType: s.serviceType,
|
||||
}))}
|
||||
serviceType={serviceType}
|
||||
updateIngestion={(data, triggerIngestion) => {
|
||||
setIsUpdating(false);
|
||||
handleUpdateIngestion(data, triggerIngestion);
|
||||
}}
|
||||
onCancel={() => handleCancelUpdate()}
|
||||
/>
|
||||
) : null}
|
||||
);
|
||||
};
|
||||
|
||||
const getIngestionForm = () => {
|
||||
const type = getIngestionPipelineTypeOption();
|
||||
let heading = '';
|
||||
|
||||
if (isUndefined(updateSelection)) {
|
||||
heading = `Add ${capitalize(type[0])} Ingestion`;
|
||||
} else {
|
||||
heading = `Edit ${capitalize(updateSelection.pipelineType)} Ingestion`;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="tw-bg-white tw-pt-4 tw-w-full">
|
||||
<div className="tw-max-w-2xl tw-mx-auto tw-pb-6">
|
||||
<AddIngestion
|
||||
data={updateSelection}
|
||||
handleCancelClick={handleCancelUpdate}
|
||||
heading={heading}
|
||||
pipelineType={type[0]}
|
||||
serviceCategory={serviceCategory}
|
||||
serviceData={serviceDetails}
|
||||
showSuccessScreen={false}
|
||||
status={
|
||||
isUndefined(updateSelection)
|
||||
? FormSubmitType.ADD
|
||||
: FormSubmitType.EDIT
|
||||
}
|
||||
onAddIngestionSave={addIngestion}
|
||||
onSuccessSave={handleCancelUpdate}
|
||||
onUpdateIngestion={updateIngestion}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<div data-testid="ingestion-container">
|
||||
{showIngestionForm ? getIngestionForm() : getIngestionTab()}
|
||||
|
||||
{isConfirmationModalOpen && (
|
||||
<ConfirmationModal
|
||||
bodyText={`You want to delete ingestion ${deleteSelection.name} permanently? This action cannot be reverted.`}
|
||||
cancelText="Discard"
|
||||
confirmButtonCss="tw-bg-error hover:tw-bg-error focus:tw-bg-error"
|
||||
confirmText={
|
||||
deleteSelection.state === 'waiting' ? (
|
||||
<Loader size="small" type="white" />
|
||||
) : deleteSelection.state === 'success' ? (
|
||||
<FontAwesomeIcon icon="check" />
|
||||
) : (
|
||||
'Delete'
|
||||
)
|
||||
}
|
||||
header="Are you sure?"
|
||||
<EntityDeleteModal
|
||||
entityName={deleteSelection.name}
|
||||
entityType="ingestion"
|
||||
loadingState={deleteSelection.state}
|
||||
onCancel={handleCancelConfirmationModal}
|
||||
onConfirm={() =>
|
||||
handleDelete(deleteSelection.id, deleteSelection.name)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</Fragment>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@ -15,51 +15,82 @@ export const mockIngestionWorkFlow = {
|
||||
data: {
|
||||
data: [
|
||||
{
|
||||
id: '3dae41fd-0469-483b-9d48-622577f2e075',
|
||||
name: 'test1',
|
||||
displayName: 'Test1',
|
||||
id: 'c804ec51-8fcf-4040-b830-5d967c4cbf49',
|
||||
name: 'test3_metadata',
|
||||
displayName: 'test3_metadata',
|
||||
pipelineType: 'metadata',
|
||||
owner: {
|
||||
id: '360d5fd9-ba6b-4205-a92c-8eb98286c1c5',
|
||||
id: 'fd96fdc7-a159-4802-84be-33c68d8b7e07',
|
||||
type: 'user',
|
||||
name: 'Aaron Johnson',
|
||||
href: 'http://localhost:8585/api/v1/users/360d5fd9-ba6b-4205-a92c-8eb98286c1c5',
|
||||
name: 'anonymous',
|
||||
fullyQualifiedName: 'anonymous',
|
||||
deleted: false,
|
||||
href: 'http://localhost:8585/api/v1/users/fd96fdc7-a159-4802-84be-33c68d8b7e07',
|
||||
},
|
||||
fullyQualifiedName: 'bigquery.test1',
|
||||
ingestionType: 'bigquery',
|
||||
tags: [],
|
||||
forceDeploy: true,
|
||||
pauseWorkflow: false,
|
||||
concurrency: 1,
|
||||
startDate: '2021-11-24',
|
||||
endDate: '2022-11-25',
|
||||
workflowTimezone: 'UTC',
|
||||
retries: 1,
|
||||
retryDelay: 300,
|
||||
workflowCatchup: false,
|
||||
scheduleInterval: '0 12 * * *',
|
||||
workflowTimeout: 60,
|
||||
connectorConfig: {
|
||||
username: 'test',
|
||||
password: 'test',
|
||||
host: 'http://localhost:3000/ingestion',
|
||||
database: 'mysql',
|
||||
includeViews: true,
|
||||
enableDataProfiler: false,
|
||||
includeFilterPattern: [],
|
||||
excludeFilterPattern: [],
|
||||
fullyQualifiedName: 'test3.test3_metadata',
|
||||
source: {
|
||||
type: 'bigquery',
|
||||
serviceName: 'test3',
|
||||
serviceConnection: {
|
||||
config: {
|
||||
type: 'BigQuery',
|
||||
scheme: 'bigquery',
|
||||
hostPort: 'bigquery.googleapis.com',
|
||||
partitionField: '_PARTITIONTIME',
|
||||
partitionQuery: 'select * from {}.{} WHERE {} = "{}" LIMIT 1000',
|
||||
tagCategoryName: 'BigqueryPolicyTags',
|
||||
connectionOptions: {},
|
||||
connectionArguments: {},
|
||||
enablePolicyTagImport: true,
|
||||
partitionQueryDuration: 1,
|
||||
supportsUsageExtraction: true,
|
||||
supportsMetadataExtraction: true,
|
||||
},
|
||||
},
|
||||
sourceConfig: {
|
||||
config: {
|
||||
includeViews: false,
|
||||
enableDataProfiler: true,
|
||||
generateSampleData: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
openMetadataServerConnection: {
|
||||
hostPort: 'http://localhost:8585/api',
|
||||
authProvider: 'no-auth',
|
||||
apiVersion: 'v1',
|
||||
},
|
||||
airflowConfig: {
|
||||
forceDeploy: true,
|
||||
pausePipeline: false,
|
||||
concurrency: 1,
|
||||
startDate: '2022-04-14',
|
||||
endDate: '2022-04-14',
|
||||
pipelineTimezone: 'UTC',
|
||||
retries: 3,
|
||||
retryDelay: 300,
|
||||
pipelineCatchup: false,
|
||||
scheduleInterval: '5 * * * *',
|
||||
pipelineTimeout: 60,
|
||||
maxActiveRuns: 1,
|
||||
workflowTimeout: 60,
|
||||
workflowDefaultView: 'tree',
|
||||
workflowDefaultViewOrientation: 'LR',
|
||||
},
|
||||
ingestionStatuses: [],
|
||||
service: {
|
||||
id: 'e7e34bc7-fc12-40d6-9478-a6297cdefe7a',
|
||||
id: 'c68e904a-4262-4b58-84c1-8a986b4aa47d',
|
||||
type: 'databaseService',
|
||||
name: 'bigquery',
|
||||
description: 'BigQuery service used for shopify data',
|
||||
href: 'http://localhost:8585/api/v1/services/databaseServices/e7e34bc7-fc12-40d6-9478-a6297cdefe7a',
|
||||
name: 'test3',
|
||||
fullyQualifiedName: 'test3',
|
||||
description: '',
|
||||
deleted: false,
|
||||
href: 'http://localhost:8585/api/v1/services/databaseServices/c68e904a-4262-4b58-84c1-8a986b4aa47d',
|
||||
},
|
||||
href: 'http://localhost:8585/api/ingestion/3dae41fd-0469-483b-9d48-622577f2e075',
|
||||
href: 'http://localhost:8585/api/v1/services/ingestionPipelines/c804ec51-8fcf-4040-b830-5d967c4cbf49',
|
||||
version: 0.1,
|
||||
updatedAt: 1637736180218,
|
||||
updatedAt: 1649941364738,
|
||||
updatedBy: 'anonymous',
|
||||
deleted: false,
|
||||
},
|
||||
],
|
||||
paging: {
|
||||
@ -69,25 +100,37 @@ export const mockIngestionWorkFlow = {
|
||||
};
|
||||
|
||||
export const mockService = {
|
||||
data: {
|
||||
data: [
|
||||
{
|
||||
id: 'e7e34bc7-fc12-40d6-9478-a6297cdefe7a',
|
||||
name: 'bigquery',
|
||||
serviceType: 'BigQuery',
|
||||
description: 'BigQuery service used for shopify data',
|
||||
version: 0.1,
|
||||
updatedAt: 1637734235276,
|
||||
updatedBy: 'anonymous',
|
||||
href: 'http://localhost:8585/api/v1/services/databaseServices/e7e34bc7-fc12-40d6-9478-a6297cdefe7a',
|
||||
jdbc: {
|
||||
driverClass: 'jdbc',
|
||||
connectionUrl: 'jdbc://localhost',
|
||||
},
|
||||
},
|
||||
],
|
||||
paging: {
|
||||
total: 1,
|
||||
id: 'c68e904a-4262-4b58-84c1-8a986b4aa47d',
|
||||
name: 'test3',
|
||||
serviceType: 'BigQuery',
|
||||
description: '',
|
||||
connection: {
|
||||
config: {
|
||||
type: 'BigQuery',
|
||||
scheme: 'bigquery',
|
||||
hostPort: 'bigquery.googleapis.com',
|
||||
partitionField: '_PARTITIONTIME',
|
||||
partitionQuery: 'select * from {}.{} WHERE {} = "{}" LIMIT 1000',
|
||||
tagCategoryName: 'BigqueryPolicyTags',
|
||||
connectionOptions: {},
|
||||
connectionArguments: {},
|
||||
enablePolicyTagImport: true,
|
||||
partitionQueryDuration: 1,
|
||||
supportsUsageExtraction: true,
|
||||
supportsMetadataExtraction: true,
|
||||
},
|
||||
},
|
||||
version: 0.1,
|
||||
updatedAt: 1649941355557,
|
||||
updatedBy: 'anonymous',
|
||||
owner: {
|
||||
id: 'fd96fdc7-a159-4802-84be-33c68d8b7e07',
|
||||
type: 'user',
|
||||
name: 'anonymous',
|
||||
fullyQualifiedName: 'anonymous',
|
||||
deleted: false,
|
||||
href: 'http://localhost:8585/api/v1/users/fd96fdc7-a159-4802-84be-33c68d8b7e07',
|
||||
},
|
||||
href: 'http://localhost:8585/api/v1/services/databaseServices/c68e904a-4262-4b58-84c1-8a986b4aa47d',
|
||||
deleted: false,
|
||||
};
|
||||
|
||||
@ -20,14 +20,15 @@ import {
|
||||
} from '@testing-library/react';
|
||||
import React from 'react';
|
||||
import { MemoryRouter } from 'react-router';
|
||||
import { AirflowPipeline } from '../../generated/operations/pipelines/airflowPipeline';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import Ingestion from './Ingestion.component';
|
||||
import { mockIngestionWorkFlow } from './Ingestion.mock';
|
||||
import { mockIngestionWorkFlow, mockService } from './Ingestion.mock';
|
||||
|
||||
jest.mock('../../authentication/auth-provider/AuthProvider', () => {
|
||||
return {
|
||||
useAuthContext: jest.fn(() => ({
|
||||
isAuthDisabled: false,
|
||||
isAuthDisabled: true,
|
||||
isAuthenticated: true,
|
||||
isProtectedRoute: jest.fn().mockReturnValue(true),
|
||||
isTourRoute: jest.fn().mockReturnValue(false),
|
||||
@ -73,16 +74,16 @@ jest.mock('../common/next-previous/NextPrevious', () => {
|
||||
return jest.fn().mockImplementation(() => <div>NextPrevious</div>);
|
||||
});
|
||||
|
||||
jest.mock('../IngestionModal/IngestionModal.component', () => {
|
||||
jest.mock('../AddIngestion/AddIngestion.component', () => {
|
||||
return jest
|
||||
.fn()
|
||||
.mockImplementation(() => (
|
||||
<div data-testid="ingestion-modal">IngestionModal</div>
|
||||
<div data-testid="ingestion-form">AddIngestion</div>
|
||||
));
|
||||
});
|
||||
|
||||
jest.mock('../Modals/ConfirmationModal/ConfirmationModal', () => {
|
||||
return jest.fn().mockImplementation(() => <div>ConfirmationModal</div>);
|
||||
jest.mock('../Modals/EntityDeleteModal/EntityDeleteModal', () => {
|
||||
return jest.fn().mockImplementation(() => <div>EntityDeleteModal</div>);
|
||||
});
|
||||
|
||||
describe('Test Ingestion page', () => {
|
||||
@ -94,10 +95,12 @@ describe('Test Ingestion page', () => {
|
||||
currrentPage={1}
|
||||
deleteIngestion={mockDeleteIngestion}
|
||||
ingestionList={
|
||||
mockIngestionWorkFlow.data.data as unknown as AirflowPipeline[]
|
||||
mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
|
||||
}
|
||||
paging={mockPaging}
|
||||
pagingHandler={mockPaginghandler}
|
||||
serviceCategory={ServiceCategory.DASHBOARD_SERVICES}
|
||||
serviceDetails={mockService}
|
||||
serviceList={[]}
|
||||
triggerIngestion={mockTriggerIngestion}
|
||||
updateIngestion={mockFunction}
|
||||
@ -127,15 +130,17 @@ describe('Test Ingestion page', () => {
|
||||
it('Table should render necessary fields', async () => {
|
||||
const { container } = render(
|
||||
<Ingestion
|
||||
isRequiredDetailsAvailable
|
||||
addIngestion={mockFunction}
|
||||
currrentPage={1}
|
||||
deleteIngestion={mockDeleteIngestion}
|
||||
ingestionList={
|
||||
mockIngestionWorkFlow.data.data as unknown as AirflowPipeline[]
|
||||
mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
|
||||
}
|
||||
isRequiredDetailsAvailable={false}
|
||||
paging={mockPaging}
|
||||
pagingHandler={mockPaginghandler}
|
||||
serviceCategory={ServiceCategory.DASHBOARD_SERVICES}
|
||||
serviceDetails={mockService}
|
||||
serviceList={[]}
|
||||
triggerIngestion={mockTriggerIngestion}
|
||||
updateIngestion={mockFunction}
|
||||
@ -179,15 +184,17 @@ describe('Test Ingestion page', () => {
|
||||
};
|
||||
const { container } = render(
|
||||
<Ingestion
|
||||
isRequiredDetailsAvailable
|
||||
addIngestion={mockFunction}
|
||||
currrentPage={1}
|
||||
deleteIngestion={mockDeleteIngestion}
|
||||
ingestionList={
|
||||
mockIngestionWorkFlow.data.data as unknown as AirflowPipeline[]
|
||||
mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
|
||||
}
|
||||
isRequiredDetailsAvailable={false}
|
||||
paging={mockPagingAfter}
|
||||
pagingHandler={mockPaginghandler}
|
||||
serviceCategory={ServiceCategory.DASHBOARD_SERVICES}
|
||||
serviceDetails={mockService}
|
||||
serviceList={[]}
|
||||
triggerIngestion={mockTriggerIngestion}
|
||||
updateIngestion={mockFunction}
|
||||
@ -216,12 +223,13 @@ describe('Test Ingestion page', () => {
|
||||
currrentPage={1}
|
||||
deleteIngestion={mockDeleteIngestion}
|
||||
ingestionList={
|
||||
mockIngestionWorkFlow.data.data as unknown as AirflowPipeline[]
|
||||
mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
|
||||
}
|
||||
paging={mockPagingAfter}
|
||||
pagingHandler={mockPaginghandler}
|
||||
serviceCategory={ServiceCategory.DASHBOARD_SERVICES}
|
||||
serviceDetails={mockService}
|
||||
serviceList={[]}
|
||||
serviceType="BigQuery"
|
||||
triggerIngestion={mockTriggerIngestion}
|
||||
updateIngestion={mockFunction}
|
||||
/>,
|
||||
@ -234,7 +242,7 @@ describe('Test Ingestion page', () => {
|
||||
const editButton = await findByTestId(container, 'edit');
|
||||
fireEvent.click(editButton);
|
||||
|
||||
const ingestionModal = await findByTestId(container, 'ingestion-modal');
|
||||
const ingestionModal = await findByTestId(container, 'ingestion-form');
|
||||
|
||||
expect(ingestionModal).toBeInTheDocument();
|
||||
});
|
||||
@ -253,12 +261,13 @@ describe('Test Ingestion page', () => {
|
||||
currrentPage={1}
|
||||
deleteIngestion={mockDeleteIngestion}
|
||||
ingestionList={
|
||||
mockIngestionWorkFlow.data.data as unknown as AirflowPipeline[]
|
||||
mockIngestionWorkFlow.data.data as unknown as IngestionPipeline[]
|
||||
}
|
||||
paging={mockPagingAfter}
|
||||
pagingHandler={mockPaginghandler}
|
||||
serviceCategory={ServiceCategory.DASHBOARD_SERVICES}
|
||||
serviceDetails={mockService}
|
||||
serviceList={[]}
|
||||
serviceType="BigQuery"
|
||||
triggerIngestion={mockTriggerIngestion}
|
||||
updateIngestion={mockFunction}
|
||||
/>,
|
||||
@ -267,17 +276,6 @@ describe('Test Ingestion page', () => {
|
||||
}
|
||||
);
|
||||
|
||||
// on click of add ingestion
|
||||
const addIngestionButton = await findByTestId(
|
||||
container,
|
||||
'add-new-ingestion-button'
|
||||
);
|
||||
fireEvent.click(addIngestionButton);
|
||||
|
||||
const ingestionModal = await findByTestId(container, 'ingestion-modal');
|
||||
|
||||
expect(ingestionModal).toBeInTheDocument();
|
||||
|
||||
// on click of run button
|
||||
|
||||
await act(async () => {
|
||||
@ -293,7 +291,18 @@ describe('Test Ingestion page', () => {
|
||||
fireEvent.click(deleteButton);
|
||||
|
||||
expect(
|
||||
await findByText(container, /ConfirmationModal/i)
|
||||
await findByText(container, /EntityDeleteModal/i)
|
||||
).toBeInTheDocument();
|
||||
|
||||
// on click of add ingestion
|
||||
const addIngestionButton = await findByTestId(
|
||||
container,
|
||||
'add-new-ingestion-button'
|
||||
);
|
||||
fireEvent.click(addIngestionButton);
|
||||
|
||||
const ingestionModal = await findByTestId(container, 'ingestion-form');
|
||||
|
||||
expect(ingestionModal).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
@ -11,11 +11,16 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { IngestionType } from '../../enums/service.enum';
|
||||
import { IngestionType, ServiceCategory } from '../../enums/service.enum';
|
||||
import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
|
||||
import { DatabaseService } from '../../generated/entity/services/databaseService';
|
||||
import { AirflowPipeline } from '../../generated/operations/pipelines/airflowPipeline';
|
||||
import {
|
||||
Connection,
|
||||
IngestionPipeline,
|
||||
} from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import { EntityReference } from '../../generated/type/entityReference';
|
||||
import { Paging } from '../../generated/type/paging';
|
||||
import { DataObj } from '../../interface/service.interface';
|
||||
|
||||
export interface ConnectorConfig {
|
||||
username: string;
|
||||
@ -48,23 +53,29 @@ export interface IngestionData {
|
||||
endDate?: string;
|
||||
}
|
||||
|
||||
export interface Props {
|
||||
serviceType?: string;
|
||||
export interface IngestionProps {
|
||||
serviceDetails: DataObj;
|
||||
serviceName?: string;
|
||||
serviceCategory: ServiceCategory;
|
||||
isRequiredDetailsAvailable: boolean;
|
||||
paging: Paging;
|
||||
ingestionList: Array<AirflowPipeline>;
|
||||
ingestionList: Array<IngestionPipeline>;
|
||||
serviceList: Array<DatabaseService>;
|
||||
currrentPage: number;
|
||||
pagingHandler: (value: string | number, activePage?: number) => void;
|
||||
deleteIngestion: (id: string, displayName: string) => Promise<void>;
|
||||
triggerIngestion: (id: string, displayName: string) => Promise<void>;
|
||||
addIngestion: (data: AirflowPipeline, triggerIngestion?: boolean) => void;
|
||||
addIngestion: (data: CreateIngestionPipeline) => Promise<void>;
|
||||
updateIngestion: (
|
||||
data: AirflowPipeline,
|
||||
oldData: AirflowPipeline,
|
||||
data: IngestionPipeline,
|
||||
oldData: IngestionPipeline,
|
||||
id: string,
|
||||
displayName: string,
|
||||
triggerIngestion?: boolean
|
||||
) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface ModifiedConfig extends Connection {
|
||||
supportsMetadataExtraction: boolean;
|
||||
supportsUsageExtraction: boolean;
|
||||
}
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
*/
|
||||
|
||||
import { StepperStepType } from 'Models';
|
||||
import { PatternType } from '../components/AddIngestion/addIngestion.interface';
|
||||
import { FilterPattern } from '../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
|
||||
export const STEPS_FOR_ADD_INGESTION: Array<StepperStepType> = [
|
||||
{ name: 'Configure Ingestion', step: 1 },
|
||||
@ -21,7 +21,7 @@ export const STEPS_FOR_ADD_INGESTION: Array<StepperStepType> = [
|
||||
|
||||
export const INGESTION_SCHEDULER_INITIAL_VALUE = '5 * * * *';
|
||||
|
||||
export const INITIAL_FILTER_PATTERN: PatternType = {
|
||||
include: [],
|
||||
exclude: [],
|
||||
export const INITIAL_FILTER_PATTERN: FilterPattern = {
|
||||
includes: [],
|
||||
excludes: [],
|
||||
};
|
||||
|
||||
@ -26,7 +26,7 @@ import ServicePage from './index';
|
||||
jest.mock('../../authentication/auth-provider/AuthProvider', () => {
|
||||
return {
|
||||
useAuthContext: jest.fn(() => ({
|
||||
isAuthDisabled: false,
|
||||
isAuthDisabled: true,
|
||||
isAuthenticated: true,
|
||||
isProtectedRoute: jest.fn().mockReturnValue(true),
|
||||
isTourRoute: jest.fn().mockReturnValue(false),
|
||||
@ -77,8 +77,8 @@ const mockDatabase = {
|
||||
},
|
||||
};
|
||||
|
||||
jest.mock('../../axiosAPIs/airflowPipelineAPI', () => ({
|
||||
getAirflowPipelines: jest.fn().mockImplementation(() =>
|
||||
jest.mock('../../axiosAPIs/ingestionPipelineAPI', () => ({
|
||||
getIngestionPipelines: jest.fn().mockImplementation(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
data: [],
|
||||
@ -86,10 +86,10 @@ jest.mock('../../axiosAPIs/airflowPipelineAPI', () => ({
|
||||
},
|
||||
})
|
||||
),
|
||||
deleteAirflowPipelineById: jest.fn(),
|
||||
addAirflowPipeline: jest.fn(),
|
||||
triggerAirflowPipelineById: jest.fn(),
|
||||
updateAirflowPipeline: jest.fn(),
|
||||
deleteIngestionPipelineById: jest.fn(),
|
||||
addIngestionPipeline: jest.fn(),
|
||||
triggerIngestionPipelineById: jest.fn(),
|
||||
updateIngestionPipeline: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('../../axiosAPIs/serviceAPI', () => ({
|
||||
|
||||
@ -25,13 +25,6 @@ import React, { Fragment, FunctionComponent, useEffect, useState } from 'react';
|
||||
import { Link, useHistory, useParams } from 'react-router-dom';
|
||||
import AppState from '../../AppState';
|
||||
import { useAuthContext } from '../../authentication/auth-provider/AuthProvider';
|
||||
import {
|
||||
addAirflowPipeline,
|
||||
deleteAirflowPipelineById,
|
||||
getAirflowPipelines,
|
||||
triggerAirflowPipelineById,
|
||||
updateAirflowPipeline,
|
||||
} from '../../axiosAPIs/airflowPipelineAPI';
|
||||
import { getDashboards } from '../../axiosAPIs/dashboardAPI';
|
||||
import { getDatabases } from '../../axiosAPIs/databaseAPI';
|
||||
import {
|
||||
@ -40,6 +33,13 @@ import {
|
||||
postFeedById,
|
||||
postThread,
|
||||
} from '../../axiosAPIs/feedsAPI';
|
||||
import {
|
||||
addIngestionPipeline,
|
||||
deleteIngestionPipelineById,
|
||||
getIngestionPipelines,
|
||||
triggerIngestionPipelineById,
|
||||
updateIngestionPipeline,
|
||||
} from '../../axiosAPIs/ingestionPipelineAPI';
|
||||
import { getPipelines } from '../../axiosAPIs/pipelineAPI';
|
||||
import { getServiceByFQN, updateService } from '../../axiosAPIs/serviceAPI';
|
||||
import { getTopics } from '../../axiosAPIs/topicsAPI';
|
||||
@ -70,20 +70,17 @@ import { TabSpecificField } from '../../enums/entity.enum';
|
||||
import { SearchIndex } from '../../enums/search.enum';
|
||||
import { ServiceCategory } from '../../enums/service.enum';
|
||||
import { CreateThread } from '../../generated/api/feed/createThread';
|
||||
import { CreateIngestionPipeline } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
|
||||
import { Dashboard } from '../../generated/entity/data/dashboard';
|
||||
import { Database } from '../../generated/entity/data/database';
|
||||
import { Pipeline } from '../../generated/entity/data/pipeline';
|
||||
import { Topic } from '../../generated/entity/data/topic';
|
||||
import { DatabaseService } from '../../generated/entity/services/databaseService';
|
||||
import {
|
||||
AirflowPipeline,
|
||||
PipelineType,
|
||||
Schema,
|
||||
} from '../../generated/operations/pipelines/airflowPipeline';
|
||||
import { IngestionPipeline } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
|
||||
import { EntityReference } from '../../generated/type/entityReference';
|
||||
import { Paging } from '../../generated/type/paging';
|
||||
import { useAuth } from '../../hooks/authHooks';
|
||||
import { ServiceDataObj } from '../../interface/service.interface';
|
||||
import { DataObj, ServiceDataObj } from '../../interface/service.interface';
|
||||
import jsonData from '../../jsons/en';
|
||||
import {
|
||||
getEntityMissingError,
|
||||
@ -101,7 +98,6 @@ import {
|
||||
getCurrentServiceTab,
|
||||
getIsIngestionEnable,
|
||||
getServiceCategoryFromType,
|
||||
isRequiredDetailsAvailableForIngestion,
|
||||
servicePageTabs,
|
||||
serviceTypeLogo,
|
||||
} from '../../utils/ServiceUtils';
|
||||
@ -137,7 +133,7 @@ const ServicePage: FunctionComponent = () => {
|
||||
const [isConnectionAvailable, setConnectionAvailable] =
|
||||
useState<boolean>(true);
|
||||
const [isError, setIsError] = useState(false);
|
||||
const [ingestions, setIngestions] = useState<AirflowPipeline[]>([]);
|
||||
const [ingestions, setIngestions] = useState<IngestionPipeline[]>([]);
|
||||
const [serviceList] = useState<Array<DatabaseService>>([]);
|
||||
const [ingestionPaging, setIngestionPaging] = useState<Paging>({} as Paging);
|
||||
const [entityThread, setEntityThread] = useState<EntityThread[]>([]);
|
||||
@ -298,22 +294,9 @@ const ServicePage: FunctionComponent = () => {
|
||||
});
|
||||
};
|
||||
|
||||
const getSchemaFromType = (type: AirflowPipeline['pipelineType']) => {
|
||||
switch (type) {
|
||||
case PipelineType.Metadata:
|
||||
return Schema.DatabaseServiceMetadataPipeline;
|
||||
|
||||
case PipelineType.QueryUsage:
|
||||
return Schema.DatabaseServiceQueryUsagePipeline;
|
||||
|
||||
default:
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const getAllIngestionWorkflows = (paging?: string) => {
|
||||
setIsloading(true);
|
||||
getAirflowPipelines(['owner', 'pipelineStatuses'], serviceFQN, '', paging)
|
||||
getIngestionPipelines(['owner'], serviceFQN, paging)
|
||||
.then((res) => {
|
||||
if (res.data.data) {
|
||||
setIngestions(res.data.data);
|
||||
@ -339,7 +322,7 @@ const ServicePage: FunctionComponent = () => {
|
||||
displayName: string
|
||||
): Promise<void> => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
triggerAirflowPipelineById(id)
|
||||
triggerIngestionPipelineById(id)
|
||||
.then((res) => {
|
||||
if (res.data) {
|
||||
resolve();
|
||||
@ -367,7 +350,7 @@ const ServicePage: FunctionComponent = () => {
|
||||
displayName: string
|
||||
): Promise<void> => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
deleteAirflowPipelineById(id)
|
||||
deleteIngestionPipelineById(id)
|
||||
.then(() => {
|
||||
resolve();
|
||||
getAllIngestionWorkflows();
|
||||
@ -383,8 +366,8 @@ const ServicePage: FunctionComponent = () => {
|
||||
};
|
||||
|
||||
const updateIngestion = (
|
||||
data: AirflowPipeline,
|
||||
oldData: AirflowPipeline,
|
||||
data: IngestionPipeline,
|
||||
oldData: IngestionPipeline,
|
||||
id: string,
|
||||
displayName: string,
|
||||
triggerIngestion?: boolean
|
||||
@ -392,7 +375,7 @@ const ServicePage: FunctionComponent = () => {
|
||||
const jsonPatch = compare(oldData, data);
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
updateAirflowPipeline(id, jsonPatch)
|
||||
updateIngestionPipeline(id, jsonPatch)
|
||||
.then(() => {
|
||||
resolve();
|
||||
getAllIngestionWorkflows();
|
||||
@ -415,55 +398,33 @@ const ServicePage: FunctionComponent = () => {
|
||||
});
|
||||
};
|
||||
|
||||
const addIngestionWorkflowHandler = (
|
||||
data: AirflowPipeline,
|
||||
triggerIngestion?: boolean
|
||||
) => {
|
||||
setIsloading(true);
|
||||
|
||||
const ingestionData: AirflowPipeline = {
|
||||
...data,
|
||||
pipelineConfig: {
|
||||
...data.pipelineConfig,
|
||||
schema: getSchemaFromType(data.pipelineType),
|
||||
},
|
||||
service: {
|
||||
id: serviceDetails?.id,
|
||||
type: 'databaseService',
|
||||
name: data.service.name,
|
||||
} as EntityReference,
|
||||
};
|
||||
|
||||
addAirflowPipeline(ingestionData)
|
||||
.then((res: AxiosResponse) => {
|
||||
if (res.data) {
|
||||
const { id, displayName } = res.data;
|
||||
setIsloading(false);
|
||||
getAllIngestionWorkflows();
|
||||
if (triggerIngestion) {
|
||||
triggerIngestionById(id, displayName).catch((error: AxiosError) => {
|
||||
showErrorToast(
|
||||
error,
|
||||
`${jsonData['api-error-messages']['triggering-ingestion-error']} ${displayName}`
|
||||
);
|
||||
});
|
||||
const onAddIngestionSave = (data: CreateIngestionPipeline) => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
return addIngestionPipeline(data)
|
||||
.then((res: AxiosResponse) => {
|
||||
if (res.data) {
|
||||
getAllIngestionWorkflows();
|
||||
resolve();
|
||||
} else {
|
||||
showErrorToast(
|
||||
jsonData['api-error-messages']['create-ingestion-error']
|
||||
);
|
||||
reject();
|
||||
}
|
||||
} else {
|
||||
showErrorToast(jsonData['api-error-messages']['add-ingestion-error']);
|
||||
}
|
||||
})
|
||||
.catch((error: AxiosError) => {
|
||||
const message = getErrorText(
|
||||
error,
|
||||
jsonData['api-error-messages']['add-ingestion-error']
|
||||
);
|
||||
if (message.includes('Connection refused')) {
|
||||
setConnectionAvailable(false);
|
||||
} else {
|
||||
showErrorToast(message);
|
||||
}
|
||||
setIsloading(false);
|
||||
});
|
||||
})
|
||||
.catch((error: AxiosError) => {
|
||||
const message = getErrorText(
|
||||
error,
|
||||
jsonData['api-error-messages']['create-ingestion-error']
|
||||
);
|
||||
if (message.includes('Connection refused')) {
|
||||
setConnectionAvailable(false);
|
||||
} else {
|
||||
showErrorToast(message);
|
||||
}
|
||||
reject();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const handleConfigUpdate = (
|
||||
@ -1208,24 +1169,20 @@ const ServicePage: FunctionComponent = () => {
|
||||
)}
|
||||
|
||||
{activeTab === 3 && (
|
||||
<div
|
||||
className="tw-mt-4 tw-px-1"
|
||||
data-testid="ingestion-container">
|
||||
<div data-testid="ingestion-container">
|
||||
{isConnectionAvailable ? (
|
||||
<Ingestion
|
||||
addIngestion={addIngestionWorkflowHandler}
|
||||
isRequiredDetailsAvailable
|
||||
addIngestion={onAddIngestionSave}
|
||||
currrentPage={ingestionCurrentPage}
|
||||
deleteIngestion={deleteIngestionById}
|
||||
ingestionList={ingestions}
|
||||
isRequiredDetailsAvailable={isRequiredDetailsAvailableForIngestion(
|
||||
serviceName as ServiceCategory,
|
||||
serviceDetails as ServicesData
|
||||
)}
|
||||
paging={ingestionPaging}
|
||||
pagingHandler={ingestionPagingHandler}
|
||||
serviceCategory={serviceName as ServiceCategory}
|
||||
serviceDetails={serviceDetails as DataObj}
|
||||
serviceList={serviceList}
|
||||
serviceName={serviceFQN}
|
||||
serviceType={serviceDetails?.serviceType}
|
||||
triggerIngestion={triggerIngestionById}
|
||||
updateIngestion={updateIngestion}
|
||||
/>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user