Start Date: {new Date(r.startDate).toUTCString()}
+ ) : null} + {r.endDate ? ( +End Date: {new Date(r.endDate).toUTCString()}
+ ) : null} + + } + key={i} + position="bottom" + theme="light" + trigger="mouseenter"> + {i === lastFiveIngestions.length - 1 ? ( ++ {capitalize(r.state)} +
+ ) : ( + + )} +
+
+ {isAdding ? (
+
+
+ {getSearchedIngestions().length ? (
+
+ {searchText || getSearchedIngestions().length > 0 ? (
+
+
+
+
+
+
+
+
+
+
+ {Boolean(!isNil(paging.after) || !isNil(paging.before)) && (
+
+ ) : (
+ | Name | +Type | +Service | +Schedule | +Recent Runs | + {/*Next Run | */} +Actions | +
|---|---|---|---|---|---|---|
| {ingestion.displayName} | ++ {ingestion.ingestionType} + | ++ + {ingestion.service.name} + + | +
+ |
+
+ {getStatuses(ingestion)}
+ |
+ {/* + {ingestion.nextExecutionDate || '--'} + | */} +
+
+
+
+ handleTriggerIngestion(
+ ingestion.id as string,
+ ingestion.displayName
+ )
+ }>
+ {currTriggerId.id === ingestion.id ? (
+ currTriggerId.state === 'success' ? (
+
+ ) : (
+
+ handleUpdate(ingestion)}>
+ {updateSelection.id === ingestion.id ? (
+ updateSelection.state === 'success' ? (
+
+ ) : (
+
+ ConfirmDelete(
+ ingestion.id as string,
+ ingestion.displayName
+ )
+ }>
+ {deleteSelection.id === ingestion.id ? (
+ deleteSelection.state === 'success' ? (
+
+ ) : (
+
+ |
+
+
+ )}
+
+ 
+
+
+
+ + {`No ingestion workflows found ${ + searchText ? `for "${searchText}"` : '' + }`} +
+
+ {value}
+
+ );
+};
+
+const STEPS: Array{children}
;
+};
+
+const PreviewSection = ({
+ header,
+ data,
+ className,
+}: {
+ header: string;
+ data: Array<{ key: string; value: string }>;
+ className: string;
+}) => {
+ return (
+
+ {/*
*/} +
+ );
+};
+
+const getServiceName = (service: string) => {
+ return service.split('$$').splice(1).join('$$');
+};
+
+const getIngestionName = (name: string) => {
+ const nameString = name.trim().replace(/\s+/g, '_');
+
+ return nameString.toLowerCase();
+};
+
+const getCurrentDate = () => {
+ return `${new Date().toLocaleDateString('en-CA')}`;
+};
+
+const setService = (
+ serviceList: Array*/} +
{header}
+
+ {data.map((d, i) => (
+
+
+
+ ))}
+ + {d.key} +
+{d.value}
+
+
+
+
+
+
+
+
+ setIncludeViews(!includeViews)}>
+
+
+
+ setExcludeDataProfiler(!excludeDataProfiler)
+ }>
+
+
+
+
+
+
+ setIngestionSchedule(v)}
+ onError={setCronError}>
+
+
+ + ( + {ingestionSchedule} + in UTC Timezone ) +
+ {showErrorMsg.ingestionSchedule + ? errorMsg('Ingestion schedule is required') + : cronError && errorMsg(cronError.description)} +
+
+ + {cronstrue.toString(ingestionSchedule || '', { + use24HourTimeFormat: true, + verbose: true, + })} +
+
+ {steps.map((step, index) => (
+
+ {index > 0 && index < steps.length && (
+
+ )}
+
+ ))}
+
+ );
+};
+
+export default IngestionStepper;
diff --git a/catalog-rest-service/src/main/resources/ui/src/components/IngestionStepper/IngestionStepper.css b/catalog-rest-service/src/main/resources/ui/src/components/IngestionStepper/IngestionStepper.css
new file mode 100644
index 00000000000..2697e1e2589
--- /dev/null
+++ b/catalog-rest-service/src/main/resources/ui/src/components/IngestionStepper/IngestionStepper.css
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+.ingestion-content {
+ display: flex;
+ justify-content: space-around;
+}
+.ingestion-wrapper {
+ display: flex;
+}
+.ingestion-rounder {
+ display: block;
+ width: 18px;
+ height: 18px;
+ border: 3px solid #d9ceee;
+ background: white;
+ border-radius: 50%;
+ margin-top: 0.15rem;
+ z-index: 100;
+}
+.ingestion-rounder.active {
+ border-color: #7147e8;
+}
+.ingestion-rounder.completed {
+ background-color: #d9ceee;
+}
+
+.ingestion-rounder.completed::after {
+ content: '\2713';
+ display: block;
+ margin-top: -5px;
+ font-weight: 900;
+ color: #7147e8;
+}
+
+.ingestion-line::before {
+ top: 10px;
+ bottom: 0;
+ position: absolute;
+ content: ' ';
+ width: 78%;
+ height: 2px;
+ left: 64px;
+ background-color: #d9ceee;
+}
+.ingestion-line-se {
+ display: block;
+ width: 2px;
+ height: 15px;
+ background-color: #d9ceee;
+ transform: translate(8px, 0);
+}
diff --git a/catalog-rest-service/src/main/resources/ui/src/components/IngestionStepper/IngestionStepper.interface.ts b/catalog-rest-service/src/main/resources/ui/src/components/IngestionStepper/IngestionStepper.interface.ts
new file mode 100644
index 00000000000..90e3901a108
--- /dev/null
+++ b/catalog-rest-service/src/main/resources/ui/src/components/IngestionStepper/IngestionStepper.interface.ts
@@ -0,0 +1,4 @@
+export type Steps = {
+ name: string;
+ step: number;
+};
diff --git a/catalog-rest-service/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx b/catalog-rest-service/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx
index 3f3290cdd8f..41e668a43e4 100644
--- a/catalog-rest-service/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/components/Modals/AddServiceModal/AddServiceModal.tsx
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-import classNames from 'classnames';
+// import classNames from 'classnames';
import { ServiceTypes } from 'Models';
import React, { FunctionComponent, useEffect, useRef, useState } from 'react';
import { serviceTypes } from '../../../constants/services.const';
@@ -28,14 +28,14 @@ import {
import { DatabaseService } from '../../../generated/entity/services/databaseService';
import { MessagingService } from '../../../generated/entity/services/messagingService';
import { PipelineService } from '../../../generated/entity/services/pipelineService';
-import { fromISOString } from '../../../utils/ServiceUtils';
+// import { fromISOString } from '../../../utils/ServiceUtils';
import { Button } from '../../buttons/Button/Button';
import MarkdownWithPreview from '../../common/editor/MarkdownWithPreview';
// import { serviceType } from '../../../constants/services.const';
export type DataObj = {
description: string | undefined;
- ingestionSchedule:
+ ingestionSchedule?:
| {
repeatFrequency: string;
startDate: string;
@@ -128,15 +128,15 @@ const requiredField = (label: string) => (
);
-const generateOptions = (count: number, initialValue = 0) => {
- return Array(count)
- .fill(null)
- .map((_, i) => (
-
- ));
-};
+// const generateOptions = (count: number, initialValue = 0) => {
+// return Array(count)
+// .fill(null)
+// .map((_, i) => (
+//
+// ));
+// };
const generateName = (data: Array
+
+
+
+ {step.name}
+
+
+
+
+ {/*
- {ingestion && (
+
*/}
+ {/* {ingestion && (
= ({
onClick={() => setIngestion(!ingestion)}>
-
diff --git a/catalog-rest-service/src/main/resources/ui/src/components/Modals/ConfirmationModal/ConfirmationModal.tsx b/catalog-rest-service/src/main/resources/ui/src/components/Modals/ConfirmationModal/ConfirmationModal.tsx
index 65523b75257..e5d4287d657 100644
--- a/catalog-rest-service/src/main/resources/ui/src/components/Modals/ConfirmationModal/ConfirmationModal.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/components/Modals/ConfirmationModal/ConfirmationModal.tsx
@@ -1,9 +1,10 @@
import classNames from 'classnames';
import React from 'react';
+import { ReactNode } from 'react-markdown';
import { Button } from '../../buttons/Button/Button';
type Props = {
cancelText: string;
- confirmText: string;
+ confirmText: string | ReactNode;
bodyText: string;
header: string;
headerClassName?: string;
diff --git a/catalog-rest-service/src/main/resources/ui/src/components/common/CronEditor/CronEditor.component.tsx b/catalog-rest-service/src/main/resources/ui/src/components/common/CronEditor/CronEditor.component.tsx
new file mode 100644
index 00000000000..f485b489f65
--- /dev/null
+++ b/catalog-rest-service/src/main/resources/ui/src/components/common/CronEditor/CronEditor.component.tsx
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+import 'antd/dist/antd.css';
+import React, { useCallback, useEffect, useState } from 'react';
+import Cron, { CronError } from 'react-js-cron';
+
+type Props = {
+ defaultValue: string;
+ onChangeHandler?: (v: string) => void;
+ children?: React.ReactNode;
+ isReadOnly?: boolean;
+ className?: string;
+ onError?: (v: CronError) => void;
+};
+
+const CronEditor = ({
+ defaultValue,
+ onChangeHandler,
+ children,
+ isReadOnly = false,
+ className,
+ onError,
+}: Props) => {
+ const [value, setValue] = useState(defaultValue);
+ const customSetValue = useCallback((newValue) => {
+ setValue(newValue);
+ onChangeHandler?.(newValue);
+ }, []);
+
+ useEffect(() => {
+ setValue(defaultValue);
+ }, [defaultValue]);
+
+ return (
+ > = {
@@ -253,6 +254,7 @@ export const navLinkSettings = [
{ name: 'Tags', to: '/tags', disabled: false },
// { name: 'Store', to: '/store', disabled: false },
{ name: 'Services', to: '/services', disabled: false },
+ { name: 'Ingestions', to: '/ingestion', disabled: false },
// { name: 'Marketplace', to: '/marketplace', disabled: true },
// { name: 'Preferences', to: '/preference', disabled: true },
];
diff --git a/catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts b/catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts
index 6d73ce065e2..d84f083cc54 100644
--- a/catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts
+++ b/catalog-rest-service/src/main/resources/ui/src/enums/service.enum.ts
@@ -52,3 +52,18 @@ export enum PipelineServiceType {
AIRFLOW = 'Airflow',
PREFECT = 'Prefect',
}
+
+export enum IngestionType {
+ BIGQUERY = 'bigquery',
+ BIGQUERY_USAGE = 'bigquery-usage',
+ REDSHIFT = 'redshift',
+ REDSHIFT_USAGE = 'redshift-usage',
+ SNOWFLAKE = 'snowflake',
+ SNOWFLAKE_USAGE = 'snowflake-usage',
+ HIVE = 'hive',
+ MSSQL = 'mssql',
+ MYSQL = 'mysql',
+ POSTGRES = 'postgres',
+ TRINO = 'trino',
+ VERTICA = 'vertica',
+}
diff --git a/catalog-rest-service/src/main/resources/ui/src/pages/IngestionPage/IngestionPage.component.tsx b/catalog-rest-service/src/main/resources/ui/src/pages/IngestionPage/IngestionPage.component.tsx
new file mode 100644
index 00000000000..8d0806f0d70
--- /dev/null
+++ b/catalog-rest-service/src/main/resources/ui/src/pages/IngestionPage/IngestionPage.component.tsx
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+import { AxiosError, AxiosResponse } from 'axios';
+import { Operation } from 'fast-json-patch';
+import { Paging } from 'Models';
+import React, { useEffect, useState } from 'react';
+import {
+ addIngestionWorkflow,
+ deleteIngestionWorkflowsById,
+ getIngestionWorkflows,
+ patchIngestionWorkflowBtId,
+ triggerIngestionWorkflowsById,
+} from '../../axiosAPIs/ingestionWorkflowAPI';
+import { getServices } from '../../axiosAPIs/serviceAPI';
+import Ingestion from '../../components/Ingestion/Ingestion.component';
+import { IngestionData } from '../../components/Ingestion/ingestion.interface';
+import Loader from '../../components/Loader/Loader';
+import { DatabaseService } from '../../generated/entity/services/databaseService';
+import { EntityReference } from '../../generated/type/entityReference';
+import useToastContext from '../../hooks/useToastContext';
+import { getCurrentUserId } from '../../utils/CommonUtils';
+import { getOwnerFromId } from '../../utils/TableUtils';
+
+const IngestionPage = () => {
+ const showToast = useToastContext();
+ const [isLoading, setIsLoading] = useState(true);
+ const [ingestions, setIngestions] = useState([]);
+ const [serviceList, setServiceList] = useState>([]);
+ const [paging, setPaging] = useState({} as Paging);
+ const getDatabaseServices = () => {
+ getServices('databaseServices')
+ .then((res: AxiosResponse) => {
+ setServiceList(res.data.data);
+ })
+ .catch((err: AxiosError) => {
+ // eslint-disable-next-line
+ console.log(err);
+ });
+ };
+
+ const getAllIngestionWorkflows = (paging?: string) => {
+ getIngestionWorkflows(['owner, service, tags, status'], paging)
+ .then((res) => {
+ if (res.data.data) {
+ setIngestions(res.data.data);
+ setPaging(res.data.paging);
+ setIsLoading(false);
+ } else {
+ setPaging({} as Paging);
+ }
+ })
+ .catch((err: AxiosError) => {
+ const msg = err.message;
+ showToast({
+ variant: 'error',
+ body: msg ?? `Error while getting ingestion workflow`,
+ });
+ });
+ };
+
+ const triggerIngestionById = (
+ id: string,
+ displayName: string
+ ): Promise => {
+ return new Promise((resolve, reject) => {
+ triggerIngestionWorkflowsById(id)
+ .then((res) => {
+ if (res.data) {
+ resolve();
+ getAllIngestionWorkflows();
+ } else {
+ reject();
+ }
+ })
+ .catch((err: AxiosError) => {
+ const msg = err.message;
+ showToast({
+ variant: 'error',
+ body:
+ msg ?? `Error while triggering ingestion workflow ${displayName}`,
+ });
+ reject();
+ });
+ });
+ };
+
+ const deleteIngestionById = (
+ id: string,
+ displayName: string
+ ): Promise => {
+ return new Promise((resolve, reject) => {
+ deleteIngestionWorkflowsById(id)
+ .then(() => {
+ resolve();
+ getAllIngestionWorkflows();
+ })
+ .catch((err: AxiosError) => {
+ const msg = err.message;
+ showToast({
+ variant: 'error',
+ body:
+ msg ?? `Error while deleting ingestion workflow ${displayName}`,
+ });
+ reject();
+ });
+ });
+ };
+
+ const updateIngestionById = (
+ id: string,
+ displayName: string,
+ patch: Array,
+ triggerIngestion?: boolean
+ ): Promise => {
+ return new Promise((resolve, reject) => {
+ patchIngestionWorkflowBtId(id, patch)
+ .then(() => {
+ resolve();
+ getAllIngestionWorkflows();
+ if (triggerIngestion) {
+ triggerIngestionById(id, displayName).then();
+ }
+ })
+ .catch((err: AxiosError) => {
+ const msg = err.message;
+ showToast({
+ variant: 'error',
+ body:
+ msg ?? `Error while updating ingestion workflow ${displayName}`,
+ });
+ reject();
+ });
+ });
+ };
+
+ const addIngestionWorkflowHandler = (
+ data: IngestionData,
+ triggerIngestion?: boolean
+ ) => {
+ setIsLoading(true);
+ const service = serviceList.find((s) => s.name === data.service.name);
+ const owner = getOwnerFromId(getCurrentUserId());
+ const ingestionData = {
+ ...data,
+ service: {
+ id: service?.id,
+ type: 'databaseService',
+ name: data.service.name,
+ } as EntityReference,
+ owner: {
+ id: owner?.id as string,
+ name: owner?.name,
+ type: 'user',
+ },
+ };
+
+ addIngestionWorkflow(ingestionData)
+ .then((res: AxiosResponse) => {
+ const { id, displayName } = res.data;
+ setIsLoading(false);
+ getAllIngestionWorkflows();
+ if (triggerIngestion) {
+ triggerIngestionById(id, displayName).then();
+ }
+ })
+ .catch((err: AxiosError) => {
+ const msg = err.message;
+ showToast({
+ variant: 'error',
+ body: msg ?? `Something went wrong`,
+ });
+ setIsLoading(false);
+ });
+ };
+
+ const pagingHandler = (cursorType: string) => {
+ const pagingString = `&${cursorType}=${
+ paging[cursorType as keyof typeof paging]
+ }`;
+ getAllIngestionWorkflows(pagingString);
+ };
+
+ useEffect(() => {
+ getDatabaseServices();
+ getAllIngestionWorkflows();
+ }, []);
+
+ return (
+ <>
+ {isLoading ? (
+ = []) => {
return entityCounts;
};
+
+export const getIngestionTypeList = (
+ serviceType: string
+): Array | undefined => {
+ let ingestionType: Array | undefined;
+ switch (serviceType) {
+ case DatabaseServiceType.BIGQUERY:
+ ingestionType = [IngestionType.BIGQUERY, IngestionType.BIGQUERY_USAGE];
+
+ break;
+ case DatabaseServiceType.HIVE:
+ ingestionType = [IngestionType.HIVE];
+
+ break;
+
+ case DatabaseServiceType.MSSQL:
+ ingestionType = [IngestionType.MSSQL];
+
+ break;
+
+ case DatabaseServiceType.MYSQL:
+ ingestionType = [IngestionType.MYSQL];
+
+ break;
+
+ case DatabaseServiceType.POSTGRES:
+ ingestionType = [IngestionType.POSTGRES];
+
+ break;
+
+ case DatabaseServiceType.REDSHIFT:
+ ingestionType = [IngestionType.REDSHIFT, IngestionType.REDSHIFT_USAGE];
+
+ break;
+
+ case DatabaseServiceType.TRINO:
+ ingestionType = [IngestionType.TRINO];
+
+ break;
+
+ case DatabaseServiceType.SNOWFLAKE:
+ ingestionType = [IngestionType.SNOWFLAKE, IngestionType.SNOWFLAKE_USAGE];
+
+ break;
+
+ default:
+ break;
+ }
+
+ return ingestionType;
+};
diff --git a/catalog-rest-service/src/main/resources/ui/src/utils/SvgUtils.tsx b/catalog-rest-service/src/main/resources/ui/src/utils/SvgUtils.tsx
index 8c583dd2513..a1286ea7424 100644
--- a/catalog-rest-service/src/main/resources/ui/src/utils/SvgUtils.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/utils/SvgUtils.tsx
@@ -10,6 +10,7 @@ import IconConfig from '../assets/svg/config.svg';
import IconDashboardGrey from '../assets/svg/dashboard-grey.svg';
import IconDashboard from '../assets/svg/dashboard.svg';
import IconAsstest from '../assets/svg/data-assets.svg';
+import IconDeploy from '../assets/svg/deploy-icon.svg';
import IconDoc from '../assets/svg/doc.svg';
import IconError from '../assets/svg/error.svg';
import IconExternalLink from '../assets/svg/external-link.svg';
@@ -146,6 +147,7 @@ export const Icons = {
PIPELINE_GREY: 'pipeline-grey',
VERSION: 'icon-version',
VERSION_WHITE: 'icon-version-white',
+ ICON_DEPLOY: 'icon-deploy',
};
const SVGIcons: FunctionComponent = ({
@@ -435,6 +437,10 @@ const SVGIcons: FunctionComponent = ({
case Icons.VERSION_WHITE:
IconComponent = IconVersionWhite;
+ break;
+ case Icons.ICON_DEPLOY:
+ IconComponent = IconDeploy;
+
break;
default:
diff --git a/catalog-rest-service/src/main/resources/ui/tailwind.config.js b/catalog-rest-service/src/main/resources/ui/tailwind.config.js
index ffe07aba7fe..3559261df4a 100644
--- a/catalog-rest-service/src/main/resources/ui/tailwind.config.js
+++ b/catalog-rest-service/src/main/resources/ui/tailwind.config.js
@@ -33,6 +33,13 @@ const info = '#1890FF';
const warning = '#FFC34E';
const warningBG = '#FFC34E40';
+// status colors
+
+const statusSuccess = '#07A35A';
+const statusFailed = '#E54937';
+const statusRunning = '#276EF1';
+const statusQueued = '#777777';
+
// Background colors
const bodyBG = '#FCFBFE';
const bodyHoverBG = '#F9F8FD';
@@ -94,6 +101,10 @@ module.exports = {
error: error,
warning: warning,
'warning-lite': warningBG,
+ 'status-success': statusSuccess,
+ 'status-failed': statusFailed,
+ 'status-running': statusRunning,
+ 'status-queued': statusQueued,
info: info,
separator: mainSeparator,
},
+
+ );
+};
+
+export default CronEditor;
diff --git a/catalog-rest-service/src/main/resources/ui/src/constants/constants.ts b/catalog-rest-service/src/main/resources/ui/src/constants/constants.ts
index 1323daf0100..1c829074599 100644
--- a/catalog-rest-service/src/main/resources/ui/src/constants/constants.ts
+++ b/catalog-rest-service/src/main/resources/ui/src/constants/constants.ts
@@ -133,6 +133,7 @@ export const ROUTES = {
PIPELINE_DETAILS: `/pipeline/${PLACEHOLDER_ROUTE_PIPELINE_FQN}`,
PIPELINE_DETAILS_WITH_TAB: `/pipeline/${PLACEHOLDER_ROUTE_PIPELINE_FQN}/${PLACEHOLDER_ROUTE_TAB}`,
ONBOARDING: '/onboarding',
+ INGESTION: '/ingestion',
};
export const IN_PAGE_SEARCH_ROUTES: Record