From e7e5a4dd2ca617059024c8b04370a97c543d2a9f Mon Sep 17 00:00:00 2001 From: darth-coder00 <86726556+darth-coder00@users.noreply.github.com> Date: Tue, 25 Jan 2022 18:41:17 +0530 Subject: [PATCH] Addressing service page changes (#2421) * Addressing service page changes * Reverting unnecessary change * Minor change --- .../ui/src/axiosAPIs/airflowPipelineAPI.ts | 2 +- .../Ingestion/Ingestion.component.tsx | 34 +--- .../Ingestion/ingestion.interface.ts | 1 + .../IngestionModal.component.tsx | 171 +----------------- .../AddServiceModal/AddServiceModal.tsx | 92 +++++++--- .../common/description/Description.tsx | 2 +- .../resources/ui/src/constants/constants.ts | 2 + .../resources/ui/src/pages/service/index.tsx | 54 +++--- .../resources/ui/src/pages/services/index.tsx | 33 ++-- 9 files changed, 119 insertions(+), 272 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/airflowPipelineAPI.ts b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/airflowPipelineAPI.ts index 4cb5ec1622d..0397dab8c9e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/airflowPipelineAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/airflowPipelineAPI.ts @@ -24,7 +24,7 @@ export const getAirflowPipelines = ( serviceFilter?: string, paging?: string ): Promise => { - const service = `"service="${serviceFilter}`; + const service = `service=${serviceFilter}`; const url = `${getURLWithQueryFields( '/airflowPipeline', arrQueryFields, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx index c6a0163389d..76d56024f3a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx @@ -15,13 +15,7 @@ import classNames from 'classnames'; import cronstrue from 'cronstrue'; import { capitalize, isNil, lowerCase } from 'lodash'; import React, { Fragment, useCallback, useState } from 'react'; -import { Link } from 'react-router-dom'; -import { - getServiceDetailsPath, - TITLE_FOR_NON_ADMIN_ACTION, -} from '../../constants/constants'; -import { NoDataFoundPlaceHolder } from '../../constants/services.const'; -import { ServiceCategory } from '../../enums/service.enum'; +import { TITLE_FOR_NON_ADMIN_ACTION } from '../../constants/constants'; import { AirflowPipeline, ConfigObject, @@ -42,6 +36,7 @@ import { Props } from './ingestion.interface'; const Ingestion: React.FC = ({ serviceType = '', + serviceName, ingestionList, serviceList, deleteIngestion, @@ -183,13 +178,6 @@ const Ingestion: React.FC = ({ setIsConfirmationModalOpen(true); }; - const getServiceTypeFromName = (serviceName = ''): string => { - return ( - serviceList.find((service) => service.name === serviceName) - ?.serviceType || '' - ); - }; - const getSearchedIngestions = useCallback(() => { const sText = lowerCase(searchText); @@ -290,7 +278,6 @@ const Ingestion: React.FC = ({ Name Type - Service Schedule Recent Runs {/* Next Run */} @@ -307,17 +294,6 @@ const Ingestion: React.FC = ({ key={index}> {ingestion.name} {ingestion.pipelineType} - - - {ingestion.service.name} - - = ({ ) : (
- No Service -
-

{`No ingestion workflows found ${ searchText ? `for "${searchText}"` : '' @@ -436,7 +409,7 @@ const Ingestion: React.FC = ({ ingestionList={ingestionList} ingestionTypes={getAirflowPipelineTypeOption()} name="" - service="" + service={serviceName} serviceList={serviceList.map((s) => ({ name: s.name, serviceType: s.serviceType, @@ -452,6 +425,7 @@ const Ingestion: React.FC = ({ ingestionList={ingestionList} ingestionTypes={getAirflowPipelineTypes(serviceType) || []} selectedIngestion={updateSelection.ingestion} + service={serviceName} serviceList={serviceList.map((s) => ({ name: s.name, serviceType: s.serviceType, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts index e465b97246c..cb3f5e08a16 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/ingestion.interface.ts @@ -50,6 +50,7 @@ export interface IngestionData { export interface Props { serviceType?: string; + serviceName?: string; paging: Paging; ingestionList: Array; serviceList: Array; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx index 9fcd53ea842..95f91285fc7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/IngestionModal/IngestionModal.component.tsx @@ -27,7 +27,6 @@ import { getCurrentUserId, getSeparator, } from '../../utils/CommonUtils'; -import { getIngestionTypeList } from '../../utils/ServiceUtils'; import SVGIcons from '../../utils/SvgUtils'; import { Button } from '../buttons/Button/Button'; import CronEditor from '../common/CronEditor/CronEditor'; @@ -87,10 +86,6 @@ const PreviewSection = ({ ); }; -const getServiceName = (service: string) => { - return service.split('$$').splice(1).join('$$'); -}; - const getIngestionName = (name: string) => { const nameString = name.trim().replace(/\s+/g, '_'); @@ -100,7 +95,7 @@ const getIngestionName = (name: string) => { const IngestionModal: React.FC = ({ isUpdating, header, - serviceList = [], // TODO: remove default assignment after resolving prop validation warning + service, ingestionTypes, ingestionList, onCancel, @@ -121,14 +116,11 @@ const IngestionModal: React.FC = ({ ); const [ingestionName, setIngestionName] = useState( - selectedIngestion?.name || '' + selectedIngestion?.name || service || '' ); const [ingestionType, setIngestionType] = useState( selectedIngestion?.pipelineType || '' ); - const [ingestionService, setIngestionService] = useState( - selectedIngestion?.service.name || '' - ); const [pipelineConfig] = useState( (selectedIngestion?.pipelineConfig.config || {}) as ConfigObject ); @@ -171,16 +163,6 @@ const IngestionModal: React.FC = ({ isPipelineNameExists: false, }); - const isPipelineExists = () => { - return ingestionList.some( - (i) => - i.service.name === getServiceName(ingestionService) && - i.pipelineType === ingestionType && - i.service.name !== selectedIngestion?.name && - i.service.displayName === selectedIngestion?.pipelineType - ); - }; - const isPipeLineNameExists = () => { return ingestionList.some( (i) => @@ -196,17 +178,9 @@ const IngestionModal: React.FC = ({ const name = event.target.name; switch (name) { - case 'name': - setIngestionName(value); - - break; - case 'selectService': - setIngestionService(value); - setIngestionType(''); - - break; case 'ingestionType': setIngestionType(value); + setIngestionName(`${service}_${value}`); break; @@ -228,28 +202,14 @@ const IngestionModal: React.FC = ({ let isValid = false; switch (activeStep) { case 1: - isValid = Boolean( - ingestionName && ingestionType && !isPipelineExists() - ); + isValid = Boolean(ingestionName && ingestionType); setShowErrorMsg({ ...showErrorMsg, name: !ingestionName, ingestionType: !ingestionType, - selectService: !ingestionService, }); break; - // case 2: - // isValid = Boolean(username && password && host && database); - // setShowErrorMsg({ - // ...showErrorMsg, - // username: !username, - // password: !password, - // host: !host, - // database: !database, - // }); - - // break; case 2: isValid = Boolean(ingestionSchedule); setShowErrorMsg({ @@ -267,91 +227,6 @@ const IngestionModal: React.FC = ({ const getActiveStepFields = (activeStep: number) => { switch (activeStep) { - case 10: - return ( - - - - - {showErrorMsg.name && errorMsg('Ingestion Name is required')} - {showErrorMsg.isPipelineNameExists && - errorMsg(`Ingestion with similar name already exists.`)} - - - - - - {showErrorMsg.selectService && errorMsg('Service is required')} - - - - - {showErrorMsg.ingestionType && - errorMsg('Ingestion Type is required')} - {showErrorMsg.isPipelineExists && - errorMsg( - `Ingestion with service ${getServiceName( - ingestionService - )} and ingestion-type ${ingestionType} already exists ` - )} - - - ); - case 1: return ( @@ -360,17 +235,16 @@ const IngestionModal: React.FC = ({ {requiredField('Name:')} {showErrorMsg.name && errorMsg('Ingestion Name is required')} {showErrorMsg.isPipelineNameExists && @@ -400,12 +274,6 @@ const IngestionModal: React.FC = ({ {showErrorMsg.ingestionType && errorMsg('Ingestion Type is required')} - {showErrorMsg.isPipelineExists && - errorMsg( - `Ingestion with service ${getServiceName( - ingestionService - )} and ingestion-type ${ingestionType} already exists ` - )} @@ -679,27 +547,6 @@ const IngestionModal: React.FC = ({ }; const onSaveHandler = (triggerIngestion = false) => { - // const ingestionData = { - // ingestionType: ingestionType, - // displayName: ingestionName, - // name: getIngestionName(ingestionName), - // service: { name: getServiceName(ingestionService), id: '', type: '' }, - // startDate: startDate || getCurrentDate(), - // endDate: endDate || '', - // scheduleInterval: ingestionSchedule, - // forceDeploy: true, - // connectorConfig: { - // database: database, - // enableDataProfiler: excludeDataProfiler, - // excludeFilterPattern: excludeFilterPattern, - // host: host, - // includeFilterPattern: includeFilterPattern, - // includeViews: includeViews, - // password: password, - // username: username, - // }, - // }; - const ingestionObj: AirflowPipeline = { name: ingestionName, pipelineConfig: { @@ -762,7 +609,7 @@ const IngestionModal: React.FC = ({ // isPipelineExists: isPipelineExists(), isPipelineNameExists: isPipeLineNameExists(), }); - }, [ingestionType, ingestionService, ingestionName]); + }, [ingestionType, ingestionName]); useEffect(() => { if (endDate) { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx index 694b86008ae..077e2e9f8b6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx @@ -23,6 +23,7 @@ import React, { useRef, useState, } from 'react'; +import { ONLY_NUMBER_REGEX } from '../../../constants/constants'; import { serviceTypes } from '../../../constants/services.const'; import { DashboardServiceType, @@ -37,6 +38,7 @@ import { import { DatabaseService } from '../../../generated/entity/services/databaseService'; import { MessagingService } from '../../../generated/entity/services/messagingService'; import { PipelineService } from '../../../generated/entity/services/pipelineService'; +import { PipelineType } from '../../../generated/operations/pipelines/airflowPipeline'; import { useAuth } from '../../../hooks/authHooks'; import { errorMsg, @@ -146,6 +148,7 @@ type ErrorMsg = { selectService: boolean; name: boolean; url?: boolean; + port?: boolean; driverClass?: boolean; broker?: boolean; dashboardUrl?: boolean; @@ -296,7 +299,12 @@ export const AddServiceModal: FunctionComponent = ({ const [existingNames] = useState(generateName(serviceList)); const [selectService, setSelectService] = useState(data?.serviceType || ''); const [name, setName] = useState(data?.name || ''); - const [url, setUrl] = useState(data?.databaseConnection?.hostPort || ''); + const [url, setUrl] = useState( + data?.databaseConnection?.hostPort.split(':')[0] || '' + ); + const [port, setPort] = useState( + data?.databaseConnection?.hostPort.split(':')[1] || '' + ); const [database, setDatabase] = useState( data?.databaseConnection?.database || '' ); @@ -329,6 +337,7 @@ export const AddServiceModal: FunctionComponent = ({ selectService: false, name: false, url: false, + port: false, driverClass: false, broker: false, dashboardUrl: false, @@ -386,6 +395,14 @@ export const AddServiceModal: FunctionComponent = ({ case 'name': setName(value); + setIngestionTypeList( + ingestionTypeList?.map((d) => { + return { + ...d, + ingestionName: `${value}_${PipelineType.Metadata}`, + }; + }) + ); break; @@ -394,6 +411,13 @@ export const AddServiceModal: FunctionComponent = ({ break; + case 'port': + if (ONLY_NUMBER_REGEX.test(value) || value === '') { + setPort(value); + } + + break; + default: break; } @@ -452,7 +476,7 @@ export const AddServiceModal: FunctionComponent = ({ dataObj = { ...dataObj, databaseConnection: { - hostPort: url, + hostPort: `${url}:${port}`, connectionArguments: getKeyValueObject(connectionArguments), connectionOptions: getKeyValueObject(connectionOptions), database: database, @@ -556,7 +580,7 @@ export const AddServiceModal: FunctionComponent = ({ generateSampleData: value.ingestSampleData, enableDataProfiler: value.enableDataProfiler, schemaFilterPattern: - !isEmpty(schemaIncludePattern) && + !isEmpty(schemaIncludePattern) || !isEmpty(schemaExcludePattern) ? { includes: !isEmpty(schemaIncludePattern) @@ -572,7 +596,7 @@ export const AddServiceModal: FunctionComponent = ({ } : undefined, tableFilterPattern: - !isEmpty(tableIncludePattern) && + !isEmpty(tableIncludePattern) || !isEmpty(tableExcludePattern) ? { includes: !isEmpty(tableIncludePattern) @@ -625,10 +649,11 @@ export const AddServiceModal: FunctionComponent = ({ setMsg = { ...setMsg, url: !url, + port: !port, }; } - isValid = Boolean(url); + isValid = Boolean(url && port); break; case ServiceCategory.MESSAGING_SERVICES: @@ -759,21 +784,37 @@ export const AddServiceModal: FunctionComponent = ({ return ( <>

-
+
- {showErrorMsg.url && errorMsg('Host port is required')} + {showErrorMsg.url && errorMsg('Host name is required')} +
+
+ + + {showErrorMsg.port && errorMsg('Port is required')}
@@ -1271,9 +1312,13 @@ export const AddServiceModal: FunctionComponent = ({ case ServiceCategory.DATABASE_SERVICES: data = [ { - key: 'Host Port', + key: 'Host', value: url, }, + { + key: 'Port', + value: port, + }, ]; if (username) { @@ -1553,7 +1598,7 @@ export const AddServiceModal: FunctionComponent = ({ if (ingestionTypeList) { let noErrorListCount = 0; const newFormValue = ingestionTypeList.map((value) => { - if (isEmpty(value.ingestionName)) { + if (isEmpty(value.ingestionName) && value.isIngestionActive) { isValid = false; return { @@ -1707,11 +1752,9 @@ export const AddServiceModal: FunctionComponent = ({ {requiredField('Ingestion name:')} = ({ placeholder="Ingestion name" type="text" value={type.ingestionName} - onChange={(e) => { - const newFormValues = [...ingestionTypeList]; - newFormValues[id].ingestionName = e.target.value; - newFormValues[id].showError = Boolean(e.target.value); - setIngestionTypeList(newFormValues); - }} /> {type.showError && + type.isIngestionActive && isEmpty(type.ingestionName) && errorMsg('Ingestion Name is required')} - {/* {showErrorMsg.isPipelineNameExists && - errorMsg(`Ingestion with similar name already exists.`)} */} {getSeparator('Table Filter Pattern')} @@ -1994,9 +2030,9 @@ export const AddServiceModal: FunctionComponent = ({ {Boolean(ingestionTypeList && ingestionTypeList.length) && ( - {ingestionTypeList?.map((value) => { + {ingestionTypeList?.map((value, i) => { return value.isIngestionActive ? ( - <> + = ({ header="Schema Filter Patterns" /> )} - - ) : ( - <> - ); + + ) : null; })} )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/description/Description.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/description/Description.tsx index c946af50ae3..a77e313fd9e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/description/Description.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/description/Description.tsx @@ -39,7 +39,7 @@ const Description = ({ owner, hasEditAccess, onDescriptionEdit, - description, + description = '', isEdit, onCancel, onDescriptionUpdate, diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts index ab2f92e37d1..a99e49665c5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts @@ -51,6 +51,8 @@ const PLACEHOLDER_ROUTE_ENTITY_FQN = ':entityFQN'; export const pagingObject = { after: '', before: '' }; +export const ONLY_NUMBER_REGEX = /^[0-9\b]+$/; + /* eslint-disable @typescript-eslint/camelcase */ export const tiers = [ { key: 'Tier.Tier1', doc_count: 0 }, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx index 9d46723ec08..fc9a065e381 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx @@ -65,7 +65,6 @@ import { EntityReference } from '../../generated/type/entityReference'; import useToastContext from '../../hooks/useToastContext'; import { isEven } from '../../utils/CommonUtils'; import { - getFrequencyTime, getIsIngestionEnable, getServiceCategoryFromType, serviceTypeLogo, @@ -108,9 +107,23 @@ const ServicePage: FunctionComponent = () => { const [ingestionPaging, setIngestionPaging] = useState({} as Paging); const showToast = useToastContext(); + const getCountLabel = () => { + switch (serviceName) { + case ServiceCategory.DASHBOARD_SERVICES: + return 'Dashboards'; + case ServiceCategory.MESSAGING_SERVICES: + return 'Topics'; + case ServiceCategory.PIPELINE_SERVICES: + return 'Pipelines'; + case ServiceCategory.DATABASE_SERVICES: + default: + return 'Databases'; + } + }; + const tabs = [ { - name: 'Database', + name: getCountLabel(), icon: { alt: 'schema', name: 'icon-database', @@ -119,6 +132,7 @@ const ServicePage: FunctionComponent = () => { }, isProtected: false, position: 1, + count: instanceCount, }, { name: 'Ingestions', @@ -131,6 +145,7 @@ const ServicePage: FunctionComponent = () => { isHidden: !isIngestionEnable, isProtected: false, position: 2, + count: ingestions.length, }, ]; @@ -152,7 +167,7 @@ const ServicePage: FunctionComponent = () => { }; const getAllIngestionWorkflows = (paging?: string) => { - getAirflowPipelines(['owner, tags, status'], serviceName, paging) + getAirflowPipelines(['owner, tags, status'], serviceFQN, paging) .then((res) => { if (res.data.data) { setIngestions(res.data.data); @@ -412,19 +427,6 @@ const ServicePage: FunctionComponent = () => { const getOptionalFields = (): JSX.Element => { switch (serviceName) { - case ServiceCategory.DATABASE_SERVICES: { - return ( - - - Driver Class : - {' '} - - {serviceDetails?.jdbc?.driverClass || '--'} - - • - - ); - } case ServiceCategory.MESSAGING_SERVICES: { return ( <> @@ -613,6 +615,7 @@ const ServicePage: FunctionComponent = () => { ); + case ServiceCategory.DATABASE_SERVICES: default: { return <>; } @@ -841,20 +844,6 @@ const ServicePage: FunctionComponent = () => { getAllIngestionWorkflows(pagingString); }; - const getCountLabel = () => { - switch (serviceName) { - case ServiceCategory.DASHBOARD_SERVICES: - return 'Dashboards'; - case ServiceCategory.MESSAGING_SERVICES: - return 'Topics'; - case ServiceCategory.PIPELINE_SERVICES: - return 'Pipelines'; - case ServiceCategory.DATABASE_SERVICES: - default: - return 'Databases'; - } - }; - return ( <> {isLoading ? ( @@ -866,7 +855,7 @@ const ServicePage: FunctionComponent = () => {
{getOptionalFields()} - + {/* Ingestion : {' '} @@ -885,7 +874,7 @@ const ServicePage: FunctionComponent = () => { {getCountLabel()} : {' '} {instanceCount} - + */}
{ paging={ingestionPaging} pagingHandler={ingestionPagingHandler} serviceList={serviceList} + serviceName={serviceFQN} serviceType={serviceDetails?.serviceType} triggerIngestion={triggerIngestionById} updateIngestion={updateIngestion} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/services/index.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/services/index.tsx index 7d27f55b4e0..7272871a159 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/services/index.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/services/index.tsx @@ -58,6 +58,7 @@ import { import { DatabaseService } from '../../generated/entity/services/databaseService'; import { MessagingService } from '../../generated/entity/services/messagingService'; import { PipelineService } from '../../generated/entity/services/pipelineService'; +import { PipelineType } from '../../generated/operations/pipelines/airflowPipeline'; import { useAuth } from '../../hooks/authHooks'; import useToastContext from '../../hooks/useToastContext'; import { @@ -65,7 +66,6 @@ import { getCountBadge, getServiceLogo, } from '../../utils/CommonUtils'; -import { getFrequencyTime } from '../../utils/ServiceUtils'; import SVGIcons from '../../utils/SvgUtils'; type ServiceRecord = { @@ -261,13 +261,23 @@ const ServicesPage = () => { ...ingestion.service, id: serviceId, }, + pipelineType: PipelineType.Metadata, }); }); - Promise.allSettled(promises).then(() => { - setIsModalOpen(false); - setEditData(undefined); - }); + Promise.allSettled(promises).then( + (response: PromiseSettledResult[]) => { + response.map((data) => { + data.status === 'rejected' && + showToast({ + variant: 'error', + body: data.reason || 'Something went wrong!', + }); + }); + setIsModalOpen(false); + setEditData(undefined); + } + ); } else { setIsModalOpen(false); setEditData(undefined); @@ -551,18 +561,7 @@ const ServicesPage = () => { )}
{getOptionalFields(service)} -
- - - {service.ingestionSchedule?.repeatFrequency - ? getFrequencyTime( - service.ingestionSchedule.repeatFrequency - ) - : '--'} - -
+