From af0fb11c2321412f9ed305217e8a51acacadad3b Mon Sep 17 00:00:00 2001 From: darth-coder00 <86726556+darth-coder00@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:14:36 +0530 Subject: [PATCH 1/3] Fix #2389: Revamp Service details with new Ingestion tab --- .../ui/src/axiosAPIs/airflowPipelineAPI.ts | 35 +- .../Ingestion/Ingestion.component.tsx | 126 ++-- .../components/Ingestion/Ingestion.test.tsx | 18 +- .../Ingestion/ingestion.interface.ts | 9 +- .../IngestionModal.component.tsx | 572 +++++++++++------- .../IngestionModal.interface.ts | 16 +- .../IngestionModal/IngestionModal.test.tsx | 2 + .../IngestionStepper/IngestionStepper.css | 5 + .../AddServiceModal/AddServiceModal.tsx | 452 +++++++++----- .../resources/ui/src/constants/constants.ts | 2 +- .../IngestionPage/IngestionPage.component.tsx | 259 ++++---- .../resources/ui/src/pages/service/index.tsx | 91 +-- .../resources/ui/src/pages/services/index.tsx | 34 -- .../ui/src/router/AuthenticatedAppRouter.tsx | 2 - .../resources/ui/src/utils/CommonUtils.tsx | 10 + .../resources/ui/src/utils/ServiceUtils.ts | 45 ++ 16 files changed, 1035 insertions(+), 643 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/airflowPipelineAPI.ts b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/airflowPipelineAPI.ts index 2bc6e47197c..4cb5ec1622d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/airflowPipelineAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/airflowPipelineAPI.ts @@ -1,4 +1,5 @@ import { AxiosResponse } from 'axios'; +import { Operation } from 'fast-json-patch'; import { CreateAirflowPipeline } from '../generated/api/operations/pipelines/createAirflowPipeline'; import { getURLWithQueryFields } from '../utils/APIUtils'; import APIClient from './index'; @@ -20,11 +21,15 @@ export const addAirflowPipeline = ( export const getAirflowPipelines = ( arrQueryFields: Array, + serviceFilter?: string, paging?: string ): Promise => { - const url = `${getURLWithQueryFields('/airflowPipeline', arrQueryFields)}${ - paging ? paging : '' - }`; + const service = `"service="${serviceFilter}`; + const url = `${getURLWithQueryFields( + '/airflowPipeline', + arrQueryFields, + service + )}${paging ? paging : ''}`; return APIClient({ method: 'get', url, baseURL: operationsBaseUrl }); }; @@ -40,3 +45,27 @@ export const triggerAirflowPipelineById = ( return APIClient({ method: 'post', url, baseURL: operationsBaseUrl }); }; + +export const deleteAirflowPipelineById = ( + id: string, + arrQueryFields = '' +): Promise => { + const url = getURLWithQueryFields(`/airflowPipeline/${id}`, arrQueryFields); + + return APIClient({ method: 'delete', url, baseURL: operationsBaseUrl }); +}; + +export const updateAirflowPipeline = ( + id: string, + data: Operation[] +): Promise => { + const url = `/airflowPipeline/${id}`; + + return APIClient({ + method: 'patch', + url, + baseURL: operationsBaseUrl, + data: data, + headers: { 'Content-type': 'application/json-patch+json' }, + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx index f79f378a302..c6a0163389d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx @@ -22,9 +22,15 @@ import { } from '../../constants/constants'; import { NoDataFoundPlaceHolder } from '../../constants/services.const'; import { ServiceCategory } from '../../enums/service.enum'; -// import { useAuth } from '../../hooks/authHooks'; +import { + AirflowPipeline, + ConfigObject, + PipelineType, +} from '../../generated/operations/pipelines/airflowPipeline'; +import { useAuth } from '../../hooks/authHooks'; import { isEven } from '../../utils/CommonUtils'; -// import { Button } from '../buttons/Button/Button'; +import { getAirflowPipelineTypes } from '../../utils/ServiceUtils'; +import { Button } from '../buttons/Button/Button'; import NextPrevious from '../common/next-previous/NextPrevious'; import NonAdminAction from '../common/non-admin-action/NonAdminAction'; import PopOver from '../common/popover/PopOver'; @@ -32,9 +38,10 @@ import Searchbar from '../common/searchbar/Searchbar'; import IngestionModal from '../IngestionModal/IngestionModal.component'; import Loader from '../Loader/Loader'; import ConfirmationModal from '../Modals/ConfirmationModal/ConfirmationModal'; -import { IngestionData, Props } from './ingestion.interface'; +import { Props } from './ingestion.interface'; const Ingestion: React.FC = ({ + serviceType = '', ingestionList, serviceList, deleteIngestion, @@ -44,7 +51,7 @@ const Ingestion: React.FC = ({ paging, pagingHandler, }: Props) => { - // const { isAdminUser, isAuthDisabled } = useAuth(); + const { isAdminUser, isAuthDisabled } = useAuth(); const [searchText, setSearchText] = useState(''); const [currTriggerId, setCurrTriggerId] = useState({ id: '', state: '' }); const [isAdding, setIsAdding] = useState(false); @@ -59,13 +66,26 @@ const Ingestion: React.FC = ({ id: '', name: '', state: '', - ingestion: {} as IngestionData, + ingestion: {} as AirflowPipeline, }); const handleSearchAction = (searchValue: string) => { setSearchText(searchValue); }; + const getAirflowPipelineTypeOption = (): PipelineType[] => { + const types = getAirflowPipelineTypes(serviceType) || []; + + return ingestionList.reduce((prev, curr) => { + const index = prev.indexOf(curr.pipelineType); + if (index > -1) { + prev.splice(index, 1); + } + + return prev; + }, types); + }; + const handleTriggerIngestion = (id: string, displayName: string) => { setCurrTriggerId({ id, state: 'waiting' }); triggerIngestion(id, displayName) @@ -85,39 +105,46 @@ const Ingestion: React.FC = ({ }); }; - // const handleUpdate = (ingestion: IngestionData) => { - // setUpdateSelection({ - // id: ingestion.id as string, - // name: ingestion.displayName, - // state: '', - // ingestion: ingestion, - // }); - // setIsUpdating(true); - // }; + const handleUpdate = (ingestion: AirflowPipeline) => { + setUpdateSelection({ + id: ingestion.id as string, + name: ingestion.name, + state: '', + ingestion: ingestion, + }); + setIsUpdating(true); + }; const handleCancelUpdate = () => { setUpdateSelection({ id: '', name: '', state: '', - ingestion: {} as IngestionData, + ingestion: {} as AirflowPipeline, }); setIsUpdating(false); }; const handleUpdateIngestion = ( - data: IngestionData, + data: AirflowPipeline, triggerIngestion?: boolean ) => { - const { service, owner } = updateSelection.ingestion; - const updatedData = { + const { pipelineConfig } = updateSelection.ingestion; + const updatedData: AirflowPipeline = { ...updateSelection.ingestion, - ...data, - service, - owner, + pipelineConfig: { + ...pipelineConfig, + config: { + ...(pipelineConfig.config as ConfigObject), + + ...(data.pipelineConfig.config as ConfigObject), + }, + }, + scheduleInterval: data.scheduleInterval, }; setUpdateSelection((prev) => ({ ...prev, state: 'waiting' })); updateIngestion( - updatedData, + updatedData as AirflowPipeline, + updateSelection.ingestion, updateSelection.id, updateSelection.name, triggerIngestion @@ -147,14 +174,14 @@ const Ingestion: React.FC = ({ }); }; - // const ConfirmDelete = (id: string, name: string) => { - // setDeleteSelection({ - // id, - // name, - // state: '', - // }); - // setIsConfirmationModalOpen(true); - // }; + const ConfirmDelete = (id: string, name: string) => { + setDeleteSelection({ + id, + name, + state: '', + }); + setIsConfirmationModalOpen(true); + }; const getServiceTypeFromName = (serviceName = ''): string => { return ( @@ -175,17 +202,17 @@ const Ingestion: React.FC = ({ : ingestionList; }, [searchText, ingestionList]); - const getStatuses = (ingestion: IngestionData) => { - const lastFiveIngestions = ingestion.ingestionStatuses + const getStatuses = (ingestion: AirflowPipeline) => { + const lastFiveIngestions = ingestion.pipelineStatuses ?.sort((a, b) => { // Turn your strings into millis, and then subtract them // to get a value that is either negative, positive, or zero. - const date1 = new Date(a.startDate); - const date2 = new Date(b.startDate); + const date1 = new Date(a.startDate || ''); + const date2 = new Date(b.startDate || ''); return date1.getTime() - date2.getTime(); }) - .slice(Math.max(ingestion.ingestionStatuses.length - 5, 0)); + .slice(Math.max(ingestion.pipelineStatuses.length - 5, 0)); return lastFiveIngestions?.map((r, i) => { return ( @@ -234,21 +261,24 @@ const Ingestion: React.FC = ({ ) : null}
- {/* - */} +
{getSearchedIngestions().length ? ( @@ -275,10 +305,8 @@ const Ingestion: React.FC = ({ !isEven(index + 1) ? 'odd-row' : null )} key={index}> - {ingestion.displayName} - - {ingestion.ingestionType} - + {ingestion.name} + {ingestion.pipelineType} = ({ onClick={() => handleTriggerIngestion( ingestion.id as string, - ingestion.displayName + ingestion.name ) }> {currTriggerId.id === ingestion.id ? ( @@ -339,7 +367,7 @@ const Ingestion: React.FC = ({ 'Run' )} - {/*
handleUpdate(ingestion)}> @@ -352,14 +380,14 @@ const Ingestion: React.FC = ({ ) : ( 'Edit' )} -
*/} - {/*
+
ConfirmDelete( ingestion.id as string, - ingestion.displayName + ingestion.name ) }> {deleteSelection.id === ingestion.id ? ( @@ -371,7 +399,7 @@ const Ingestion: React.FC = ({ ) : ( 'Delete' )} -
*/} +
@@ -406,6 +434,7 @@ const Ingestion: React.FC = ({ }} header="Add Ingestion" ingestionList={ingestionList} + ingestionTypes={getAirflowPipelineTypeOption()} name="" service="" serviceList={serviceList.map((s) => ({ @@ -421,6 +450,7 @@ const Ingestion: React.FC = ({ isUpdating header={{`Edit ${updateSelection.name}`}} ingestionList={ingestionList} + ingestionTypes={getAirflowPipelineTypes(serviceType) || []} selectedIngestion={updateSelection.ingestion} serviceList={serviceList.map((s) => ({ name: s.name, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx index d587a5fa060..0ffb5e8ad3d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.test.tsx @@ -19,9 +19,9 @@ import { } from '@testing-library/react'; import React from 'react'; import { MemoryRouter } from 'react-router'; +import { AirflowPipeline } from '../../generated/operations/pipelines/airflowPipeline'; import { mockIngestionWorkFlow } from '../../pages/IngestionPage/IngestionPage.mock'; import Ingestion from './Ingestion.component'; -import { IngestionData } from './ingestion.interface'; const mockPaging = { after: 'after', @@ -74,7 +74,9 @@ describe('Test Ingestion page', () => { { { { ; + ingestionList: Array; serviceList: Array; pagingHandler: (value: string) => void; deleteIngestion: (id: string, displayName: string) => Promise; triggerIngestion: (id: string, displayName: string) => Promise; - addIngestion: (data: IngestionData, triggerIngestion?: boolean) => void; + addIngestion: (data: AirflowPipeline, triggerIngestion?: boolean) => void; updateIngestion: ( - data: IngestionData, + data: AirflowPipeline, + oldData: AirflowPipeline, id: string, displayName: string, triggerIngestion?: boolean diff --git a/openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx index 6a449870e11..9fcd53ea842 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx @@ -13,11 +13,20 @@ import classNames from 'classnames'; import cronstrue from 'cronstrue'; +import { isEmpty } from 'lodash'; import { StepperStepType } from 'Models'; import { utc } from 'moment'; import React, { Fragment, ReactNode, useEffect, useState } from 'react'; -import { IngestionType } from '../../enums/service.enum'; -import { getCurrentDate } from '../../utils/CommonUtils'; +import { + AirflowPipeline, + ConfigObject, +} from '../../generated/operations/pipelines/airflowPipeline'; +import { useAuth } from '../../hooks/authHooks'; +import { + getCurrentDate, + getCurrentUserId, + getSeparator, +} from '../../utils/CommonUtils'; import { getIngestionTypeList } from '../../utils/ServiceUtils'; import SVGIcons from '../../utils/SvgUtils'; import { Button } from '../buttons/Button/Button'; @@ -25,7 +34,6 @@ import CronEditor from '../common/CronEditor/CronEditor'; import IngestionStepper from '../IngestionStepper/IngestionStepper.component'; import { IngestionModalProps, - ServiceData, ValidationErrorMsg, } from './IngestionModal.interface'; @@ -38,10 +46,9 @@ const errorMsg = (value: string) => { }; const STEPS: Array = [ - { name: 'Ingestion details', step: 1 }, - { name: 'Connector config', step: 2 }, - { name: 'Scheduling', step: 3 }, - { name: 'Review and Deploy', step: 4 }, + { name: 'Ingestion config', step: 1 }, + { name: 'Scheduling', step: 2 }, + { name: 'Review and Deploy', step: 3 }, ]; const requiredField = (label: string) => ( @@ -66,7 +73,7 @@ const PreviewSection = ({ return (

{header}

-
+
{data.map((d, i) => (
@@ -90,26 +97,19 @@ const getIngestionName = (name: string) => { return nameString.toLowerCase(); }; -const setService = ( - serviceList: Array, - currentservice: string -) => { - const service = serviceList.find((s) => s.name === currentservice); - - return service ? `${service?.serviceType}$$${service?.name}` : ''; -}; - const IngestionModal: React.FC = ({ isUpdating, header, serviceList = [], // TODO: remove default assignment after resolving prop validation warning + ingestionTypes, ingestionList, onCancel, addIngestion, updateIngestion, selectedIngestion, }: IngestionModalProps) => { - const [activeStep, setActiveStep] = useState(isUpdating ? 2 : 1); + const { isAdminUser } = useAuth(); + const [activeStep, setActiveStep] = useState(1); const [startDate, setStartDate] = useState( utc(selectedIngestion?.startDate).format('YYYY-MM-DD') || getCurrentDate() @@ -121,44 +121,43 @@ const IngestionModal: React.FC = ({ ); const [ingestionName, setIngestionName] = useState( - selectedIngestion?.displayName || '' + selectedIngestion?.name || '' ); const [ingestionType, setIngestionType] = useState( - selectedIngestion?.ingestionType || '' + selectedIngestion?.pipelineType || '' ); const [ingestionService, setIngestionService] = useState( - setService(serviceList, selectedIngestion?.service?.name as string) || '' + selectedIngestion?.service.name || '' ); - - const [username, setUsername] = useState( - selectedIngestion?.connectorConfig?.username || '' + const [pipelineConfig] = useState( + (selectedIngestion?.pipelineConfig.config || {}) as ConfigObject ); - const [password, setPassword] = useState( - selectedIngestion?.connectorConfig?.password || '' + const [tableIncludeFilter, setTableIncludeFilter] = useState( + pipelineConfig.tableFilterPattern?.includes?.join(',') || '' ); - const [host, setHost] = useState( - selectedIngestion?.connectorConfig?.host || '' + const [tableExcludesFilter, setTableExcludesFilter] = useState( + pipelineConfig.tableFilterPattern?.excludes?.join(',') || '' ); - const [database, setDatabase] = useState( - selectedIngestion?.connectorConfig?.database || '' + const [schemaIncludeFilter, setSchemaIncludeFilter] = useState( + pipelineConfig.schemaFilterPattern?.includes?.join(',') || '' + ); + const [schemaExcludesFilter, setSchemaExcludesFilter] = useState( + pipelineConfig.schemaFilterPattern?.excludes?.join(',') || '' ); - const [includeFilterPattern, setIncludeFilterPattern] = useState< - Array - >(selectedIngestion?.connectorConfig?.includeFilterPattern || []); - const [excludeFilterPattern, setExcludeFilterPattern] = useState< - Array - >(selectedIngestion?.connectorConfig?.excludeFilterPattern || []); const [includeViews, setIncludeViews] = useState( - selectedIngestion?.connectorConfig?.includeViews || true + pipelineConfig.includeViews || true + ); + const [ingestSampleData, setIngestSampleData] = useState( + pipelineConfig.generateSampleData || true ); const [excludeDataProfiler, setExcludeDataProfiler] = useState( - selectedIngestion?.connectorConfig?.enableDataProfiler || false + pipelineConfig.enableDataProfiler || false ); const [ingestionSchedule, setIngestionSchedule] = useState( selectedIngestion?.scheduleInterval || '5 * * * *' ); - const [isPasswordVisible, setIspasswordVisible] = useState(false); + const [showErrorMsg, setShowErrorMsg] = useState({ selectService: false, name: false, @@ -176,9 +175,9 @@ const IngestionModal: React.FC = ({ return ingestionList.some( (i) => i.service.name === getServiceName(ingestionService) && - i.ingestionType === ingestionType && + i.pipelineType === ingestionType && i.service.name !== selectedIngestion?.name && - i.service.displayName === selectedIngestion?.ingestionType + i.service.displayName === selectedIngestion?.pipelineType ); }; @@ -210,22 +209,7 @@ const IngestionModal: React.FC = ({ setIngestionType(value); break; - case 'username': - setUsername(value); - break; - case 'password': - setPassword(value); - - break; - case 'host': - setHost(value); - - break; - case 'database': - setDatabase(value); - - break; case 'ingestionSchedule': setIngestionSchedule(value); @@ -245,10 +229,7 @@ const IngestionModal: React.FC = ({ switch (activeStep) { case 1: isValid = Boolean( - ingestionName && - ingestionService && - ingestionType && - !isPipelineExists() + ingestionName && ingestionType && !isPipelineExists() ); setShowErrorMsg({ ...showErrorMsg, @@ -258,18 +239,18 @@ const IngestionModal: React.FC = ({ }); break; - case 2: - isValid = Boolean(username && password && host && database); - setShowErrorMsg({ - ...showErrorMsg, - username: !username, - password: !password, - host: !host, - database: !database, - }); + // case 2: + // isValid = Boolean(username && password && host && database); + // setShowErrorMsg({ + // ...showErrorMsg, + // username: !username, + // password: !password, + // host: !host, + // database: !database, + // }); - break; - case 3: + // break; + case 2: isValid = Boolean(ingestionSchedule); setShowErrorMsg({ ...showErrorMsg, @@ -286,7 +267,7 @@ const IngestionModal: React.FC = ({ const getActiveStepFields = (activeStep: number) => { switch (activeStep) { - case 1: + case 10: return ( @@ -371,106 +352,154 @@ const IngestionModal: React.FC = ({ ); - case 2: + case 1: return ( - + - + - - - {showErrorMsg.host && errorMsg('Host is required')} + {getSeparator('Table Filter Pattern')} +
+
+ + { + setTableIncludeFilter(e.target.value); + }} + /> +
+
+ + { + setTableExcludesFilter(e.target.value); + }} + /> +
+
+ - - - {showErrorMsg.database && errorMsg('Database is required')} - - - - setIncludeFilterPattern([e.target.value])} - /> - - - - setExcludeFilterPattern([e.target.value])} - /> + {getSeparator('Schema Filter Pattern')} +
+
+ + { + setSchemaIncludeFilter(e.target.value); + }} + /> +
+
+ + { + setSchemaExcludesFilter(e.target.value); + }} + /> +
+
+ +
- +
= ({ onClick={() => setIncludeViews(!includeViews)}>
- - +
+
= ({ }>
- +
+
+ +
{ + setIngestSampleData(!ingestSampleData); + }}> +
+
+
); - case 3: + case 2: return (
@@ -539,7 +582,7 @@ const IngestionModal: React.FC = ({
); - case 4: + case 3: return (
= ({ className="tw-mb-4 tw-mt-4" data={[ { key: 'Name', value: ingestionName }, - { - key: 'Service Type', - value: getServiceName(ingestionService), - }, { key: 'Ingestion Type', value: ingestionType }, ]} header="Ingestion Details" /> - - - {isPasswordVisible - ? password - : ''.padStart(password.length, '*')} - - setIspasswordVisible((pre) => !pre)} - /> -
- ), - }, - { key: 'Host', value: host }, - { key: 'Database', value: database }, - { key: 'Include views', value: includeViews ? 'Yes' : 'No' }, - { - key: 'Enable Data Profiler', - value: excludeDataProfiler ? 'Yes' : 'No', - }, - ]} - header="Connector Config" - /> + {(!isEmpty(tableIncludeFilter) || + !isEmpty(tableExcludesFilter)) && ( + + )} + + {(!isEmpty(schemaIncludeFilter) || + !isEmpty(schemaExcludesFilter)) && ( + + )} + -

+

{cronstrue.toString(ingestionSchedule || '', { use24HourTimeFormat: true, verbose: true, @@ -618,34 +679,87 @@ const IngestionModal: React.FC = ({ }; const onSaveHandler = (triggerIngestion = false) => { - const ingestionData = { - ingestionType: ingestionType as IngestionType, - displayName: ingestionName, - name: getIngestionName(ingestionName), - service: { name: getServiceName(ingestionService), id: '', type: '' }, - startDate: startDate || getCurrentDate(), - endDate: endDate || '', - scheduleInterval: ingestionSchedule, - forceDeploy: true, - connectorConfig: { - database: database, - enableDataProfiler: excludeDataProfiler, - excludeFilterPattern: excludeFilterPattern, - host: host, - includeFilterPattern: includeFilterPattern, - includeViews: includeViews, - password: password, - username: username, + // const ingestionData = { + // ingestionType: ingestionType, + // displayName: ingestionName, + // name: getIngestionName(ingestionName), + // service: { name: getServiceName(ingestionService), id: '', type: '' }, + // startDate: startDate || getCurrentDate(), + // endDate: endDate || '', + // scheduleInterval: ingestionSchedule, + // forceDeploy: true, + // connectorConfig: { + // database: database, + // enableDataProfiler: excludeDataProfiler, + // excludeFilterPattern: excludeFilterPattern, + // host: host, + // includeFilterPattern: includeFilterPattern, + // includeViews: includeViews, + // password: password, + // username: username, + // }, + // }; + + const ingestionObj: AirflowPipeline = { + name: ingestionName, + pipelineConfig: { + schema: selectedIngestion?.pipelineConfig.schema, + config: { + includeViews: includeViews, + generateSampleData: ingestSampleData, + enableDataProfiler: excludeDataProfiler, + schemaFilterPattern: + !isEmpty(schemaIncludeFilter) || !isEmpty(schemaExcludesFilter) + ? { + includes: !isEmpty(schemaIncludeFilter) + ? schemaIncludeFilter.split(',') + : undefined, + excludes: !isEmpty(schemaExcludesFilter) + ? schemaExcludesFilter.split(',') + : undefined, + } + : undefined, + tableFilterPattern: + !isEmpty(tableIncludeFilter) || !isEmpty(tableExcludesFilter) + ? { + includes: !isEmpty(tableIncludeFilter) + ? tableIncludeFilter.split(',') + : undefined, + excludes: !isEmpty(tableExcludesFilter) + ? tableExcludesFilter.split(',') + : undefined, + } + : undefined, + }, }, + service: { + type: selectedIngestion?.service.type || '', + id: selectedIngestion?.service.id || '', + }, + owner: { + id: selectedIngestion?.owner?.id || getCurrentUserId(), + type: selectedIngestion?.owner?.type || isAdminUser ? 'admin' : 'user', + }, + scheduleInterval: ingestionSchedule, + startDate: startDate as unknown as Date, + endDate: endDate as unknown as Date, + forceDeploy: true, + pipelineType: + selectedIngestion?.pipelineType || + (ingestionType as AirflowPipeline['pipelineType']), }; - addIngestion?.(ingestionData, triggerIngestion); - updateIngestion?.(ingestionData, triggerIngestion); + + if (isUpdating) { + updateIngestion?.(ingestionObj, triggerIngestion); + } else { + addIngestion?.(ingestionObj, triggerIngestion); + } }; useEffect(() => { setShowErrorMsg({ ...showErrorMsg, - isPipelineExists: isPipelineExists(), + // isPipelineExists: isPipelineExists(), isPipelineNameExists: isPipeLineNameExists(), }); }, [ingestionType, ingestionService, ingestionName]); @@ -687,7 +801,11 @@ const IngestionModal: React.FC = ({

- +
{getActiveStepFields(activeStep)}
@@ -709,7 +827,7 @@ const IngestionModal: React.FC = ({ Previous - {activeStep === 4 ? ( + {activeStep === 3 ? (
- diff --git a/openmetadata-ui/src/main/resources/ui/src/router/AuthenticatedAppRouter.tsx b/openmetadata-ui/src/main/resources/ui/src/router/AuthenticatedAppRouter.tsx index 44824296e10..8a1740a5743 100644 --- a/openmetadata-ui/src/main/resources/ui/src/router/AuthenticatedAppRouter.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/router/AuthenticatedAppRouter.tsx @@ -22,7 +22,6 @@ import DatabaseDetails from '../pages/database-details/index'; import DatasetDetailsPage from '../pages/DatasetDetailsPage/DatasetDetailsPage.component'; import EntityVersionPage from '../pages/EntityVersionPage/EntityVersionPage.component'; import ExplorePage from '../pages/explore/ExplorePage.component'; -import IngestionPage from '../pages/IngestionPage/IngestionPage.component'; import MyDataPage from '../pages/MyDataPage/MyDataPage.component'; import PipelineDetailsPage from '../pages/PipelineDetails/PipelineDetailsPage.component'; import RolesPage from '../pages/RolesPage/RolesPage.component'; @@ -88,7 +87,6 @@ const AuthenticatedAppRouter: FunctionComponent = () => { path={ROUTES.PIPELINE_DETAILS_WITH_TAB} /> - {isAuthDisabled || isAdminUser ? ( <> diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx index 47a8bdfa60c..30d7d277a3b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx @@ -297,6 +297,16 @@ export const errorMsg = (value: string) => { ); }; +export const getSeparator = (title: string | JSX.Element) => { + return ( + +
+ {title && {title}} +
+ + ); +}; + export const getImages = (imageUri: string) => { const imagesObj: typeof imageTypes = imageTypes; for (const type in imageTypes) { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.ts index dd0df9a040f..ab6f141e40c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.ts @@ -48,7 +48,9 @@ import { IngestionType, MessagingServiceType, PipelineServiceType, + ServiceCategory, } from '../enums/service.enum'; +import { PipelineType } from '../generated/operations/pipelines/airflowPipeline'; import { ApiData } from '../pages/services'; export const serviceTypeLogo = (type: string) => { @@ -347,3 +349,46 @@ export const getIngestionTypeList = ( return ingestionType; }; + +export const getAirflowPipelineTypes = ( + serviceType: string, + onlyMetaData = false +): Array | undefined => { + if (onlyMetaData) { + return [PipelineType.Metadata]; + } + switch (serviceType) { + case DatabaseServiceType.REDSHIFT: + case DatabaseServiceType.BIGQUERY: + case DatabaseServiceType.SNOWFLAKE: + return [PipelineType.Metadata, PipelineType.QueryUsage]; + + case DatabaseServiceType.HIVE: + case DatabaseServiceType.MSSQL: + case DatabaseServiceType.MYSQL: + case DatabaseServiceType.POSTGRES: + case DatabaseServiceType.TRINO: + case DatabaseServiceType.VERTICA: + return [PipelineType.Metadata]; + + default: + return; + } +}; + +export const getIsIngestionEnable = (serviceCategory: ServiceCategory) => { + switch (serviceCategory) { + case ServiceCategory.DATABASE_SERVICES: + return true; + + case ServiceCategory.MESSAGING_SERVICES: + case ServiceCategory.PIPELINE_SERVICES: + case ServiceCategory.DASHBOARD_SERVICES: + return false; + + default: + break; + } + + return false; +}; From f199f2fc39b16083b31863041af856ac7693e7dd Mon Sep 17 00:00:00 2001 From: parthp2107 <83201188+parthp2107@users.noreply.github.com> Date: Tue, 25 Jan 2022 07:47:07 +0530 Subject: [PATCH 2/3] added roles page tests (#2376) * added roles page tests * addressing style-check --- .../Modals/RulesModal/AddRuleModal.tsx | 2 + .../selenium/pages/roles/RolesPageTest.java | 136 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 openmetadata-ui/src/test/java/org/openmetadata/catalog/selenium/pages/roles/RolesPageTest.java diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/RulesModal/AddRuleModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/RulesModal/AddRuleModal.tsx index 652cc84bec8..854ac870d6c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/RulesModal/AddRuleModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/RulesModal/AddRuleModal.tsx @@ -101,6 +101,7 @@ const AddRuleModal: FC = ({ 'focus:tw-outline-none focus:tw-border-focus hover:tw-border-hover tw-h-10 tw-bg-white', { 'tw-cursor-not-allowed tw-opacity-60': isEditing } )} + data-testid="select-operation" disabled={isEditing} name="operation" value={data.operation} @@ -123,6 +124,7 @@ const AddRuleModal: FC = ({ className="tw-text-sm tw-appearance-none tw-border tw-border-main tw-rounded tw-w-full tw-py-2 tw-px-3 tw-text-grey-body tw-leading-tight focus:tw-outline-none focus:tw-border-focus hover:tw-border-hover tw-h-10 tw-bg-white" + data-testid="select-access" name="access" value={access} onChange={(e) => setAccess(e.target.value as RuleAccess)}> diff --git a/openmetadata-ui/src/test/java/org/openmetadata/catalog/selenium/pages/roles/RolesPageTest.java b/openmetadata-ui/src/test/java/org/openmetadata/catalog/selenium/pages/roles/RolesPageTest.java new file mode 100644 index 00000000000..75132f7f013 --- /dev/null +++ b/openmetadata-ui/src/test/java/org/openmetadata/catalog/selenium/pages/roles/RolesPageTest.java @@ -0,0 +1,136 @@ +package org.openmetadata.catalog.selenium.pages.roles; + +import com.github.javafaker.Faker; +import java.time.Duration; +import java.util.ArrayList; +import java.util.logging.Logger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.openmetadata.catalog.selenium.events.Events; +import org.openmetadata.catalog.selenium.properties.Property; +import org.openqa.selenium.By; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; +import org.openqa.selenium.interactions.Actions; +import org.openqa.selenium.support.ui.ExpectedConditions; +import org.openqa.selenium.support.ui.WebDriverWait; +import org.testng.Assert; + +@Order(19) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RolesPageTest { + + private static final Logger LOG = Logger.getLogger(RolesPageTest.class.getName()); + + static WebDriver webDriver; + static String url = Property.getInstance().getURL(); + static Faker faker = new Faker(); + static String roleDisplayName = faker.name().firstName(); + static String enterDescription = "//div[@data-testid='enterDescription']/div/div[2]/div/div/div/div/div/div"; + static Actions actions; + static WebDriverWait wait; + Integer waitTime = Property.getInstance().getSleepTime(); + + @BeforeEach + public void openMetadataWindow() { + System.setProperty("webdriver.chrome.driver", "src/test/resources/drivers/linux/chromedriver"); + ChromeOptions options = new ChromeOptions(); + options.addArguments("--headless"); + options.addArguments("--window-size=1280,800"); + webDriver = new ChromeDriver(options); + actions = new Actions(webDriver); + wait = new WebDriverWait(webDriver, Duration.ofSeconds(30)); + webDriver.manage().window().maximize(); + webDriver.get(url); + } + + @Test + @Order(1) + public void openRolesPage() throws InterruptedException { + Events.click(webDriver, By.cssSelector("[data-testid='closeWhatsNew']")); // Close What's new + Events.click(webDriver, By.cssSelector("[data-testid='menu-button'][id='menu-button-Settings']")); // Setting + Events.click(webDriver, By.cssSelector("[data-testid='menu-item-Roles']")); // Setting/Roles + Thread.sleep(waitTime); + } + + @Test + @Order(2) + public void addRole() throws InterruptedException { + openRolesPage(); + Events.click(webDriver, By.cssSelector("[data-testid='add-role']")); + Events.sendKeys(webDriver, By.name("name"), faker.name().firstName()); // name + Events.sendKeys(webDriver, By.name("displayName"), roleDisplayName); // displayName + Events.sendKeys(webDriver, By.xpath(enterDescription), faker.address().toString()); + Events.click(webDriver, By.cssSelector("[data-testid='boldButton']")); + Events.click(webDriver, By.cssSelector("[data-testid='italicButton']")); + Events.click(webDriver, By.cssSelector("[data-testid='linkButton']")); + Events.click(webDriver, By.cssSelector("[data-testid='saveButton']")); + } + + @Test + @Order(3) + public void editDescription() throws InterruptedException { + openRolesPage(); + Events.click(webDriver, By.xpath("//*[text()[contains(.,'" + roleDisplayName + "')]] ")); + Events.click(webDriver, By.cssSelector("[data-testid='edit-description']")); + wait.until(ExpectedConditions.elementToBeClickable(By.xpath(enterDescription))); + Events.sendKeys(webDriver, By.xpath(enterDescription), faker.address().toString()); + Events.click(webDriver, By.cssSelector("[data-testid='save']")); + } + + @Test + @Order(4) + public void addRules() throws InterruptedException { + openRolesPage(); + Events.click(webDriver, By.xpath("//*[text()[contains(.,'" + roleDisplayName + "')]] ")); + Events.click(webDriver, By.cssSelector("[data-testid='add-new-user-button']")); + Events.click(webDriver, By.cssSelector("[data-testid='select-operation']")); + Events.click(webDriver, By.cssSelector("[value='UpdateDescription']")); + Events.click(webDriver, By.cssSelector("[data-testid='select-access']")); + Events.click(webDriver, By.cssSelector("[value='allow']")); + Events.click(webDriver, By.cssSelector("[data-testid='rule-switch']")); + Events.click(webDriver, By.cssSelector("[data-testid='saveButton']")); + } + + @Test + @Order(5) + public void editRule() throws InterruptedException { + openRolesPage(); + Events.click(webDriver, By.xpath("//*[text()[contains(.,'" + roleDisplayName + "')]] ")); + Events.click(webDriver, By.xpath("//tbody[@data-testid='table-body']/tr/td[4]/div/span")); + Events.click(webDriver, By.cssSelector("[data-testid='select-access']")); + Events.click(webDriver, By.cssSelector("[value='deny']")); + Events.click(webDriver, By.cssSelector("[data-testid='saveButton']")); + Thread.sleep(2000); + String access = + webDriver.findElement(By.xpath("//tbody[@data-testid='table-body']/tr/td[2]/p")).getAttribute("innerHTML"); + Assert.assertEquals(access, "DENY"); + } + + @Test + @Order(6) + public void deleteRule() throws InterruptedException { + openRolesPage(); + Events.click(webDriver, By.xpath("//*[text()[contains(.,'" + roleDisplayName + "')]] ")); + Events.click(webDriver, By.cssSelector("[data-testid='image'][title='Delete']")); + Events.click(webDriver, By.cssSelector("[data-testid='save-button']")); + } + + @AfterEach + public void closeTabs() { + ArrayList tabs = new ArrayList<>(webDriver.getWindowHandles()); + String originalHandle = webDriver.getWindowHandle(); + for (String handle : webDriver.getWindowHandles()) { + if (!handle.equals(originalHandle)) { + webDriver.switchTo().window(handle); + webDriver.close(); + } + } + webDriver.switchTo().window(tabs.get(0)).close(); + } +} From b4d63bd95148f77a1ff924d570e2a0ca1cf7102b Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Tue, 25 Jan 2022 11:58:10 +0530 Subject: [PATCH 3/3] Fix: Typo in setting up pyarrow dependency (#2398) --- ingestion/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/setup.py b/ingestion/setup.py index 625904a3087..28f1a0389ba 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -62,7 +62,7 @@ base_plugins = { plugins: Dict[str, Set[str]] = { "amundsen": {"neo4j~=4.4.0"}, "athena": {"PyAthena[SQLAlchemy]"}, - "bigquery": {"sqlalchemy-bigquery==1.2.2", "pyarrow-6.0.1"}, + "bigquery": {"sqlalchemy-bigquery==1.2.2", "pyarrow~=6.0.1"}, "bigquery-usage": {"google-cloud-logging", "cachetools"}, # "docker": {"docker==5.0.3"}, "docker": {"python_on_whales==0.34.0"},