✨ Ingestion Pipeline Logs from UI (#6155)

* Fix #4693 Ingestion Pipeline Logs from UI

* Fix eslint warning

* Increase the width

openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts

@@ -82,3 +82,9 @@ export const updateIngestionPipeline = (
 export const checkAirflowStatus = (): Promise<AxiosResponse> => {
   return APIClient.get('/services/ingestionPipelines/status');
 };
+
+export const getIngestionPipelineLogById = (
+  id: string
+): Promise<AxiosResponse> => {
+  return APIClient.get(`/services/ingestionPipelines/logs/${id}/last`);
+};

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx

@@ -43,6 +43,7 @@ import Searchbar from '../common/searchbar/Searchbar';
 import DropDownList from '../dropdown/DropDownList';
 import Loader from '../Loader/Loader';
 import EntityDeleteModal from '../Modals/EntityDeleteModal/EntityDeleteModal';
+import IngestionLogsModal from '../Modals/IngestionLogsModal/IngestionLogsModal';
 import { IngestionProps } from './ingestion.interface';
 
 const Ingestion: React.FC<IngestionProps> = ({
@@ -68,6 +69,8 @@ const Ingestion: React.FC<IngestionProps> = ({
   const [currTriggerId, setCurrTriggerId] = useState({ id: '', state: '' });
   const [currDeployId, setCurrDeployId] = useState({ id: '', state: '' });
   const [isConfirmationModalOpen, setIsConfirmationModalOpen] = useState(false);
+  const [isLogsModalOpen, setIsLogsModalOpen] = useState(false);
+  const [selectedPipeline, setSelectedPipeline] = useState<IngestionPipeline>();
   const [deleteSelection, setDeleteSelection] = useState({
     id: '',
     name: '',
@@ -271,6 +274,10 @@ const Ingestion: React.FC<IngestionProps> = ({
       : ingestionList;
   }, [searchText, ingestionList]);
 
+  const separator = (
+    <span className="tw-mx-1.5 tw-inline-block tw-text-gray-400">|</span>
+  );
+
   const getStatuses = (ingestion: IngestionPipeline) => {
     const lastFiveIngestions = ingestion.pipelineStatuses
       ?.sort((a, b) => {
@@ -326,7 +333,7 @@ const Ingestion: React.FC<IngestionProps> = ({
     if (ingestion.deployed) {
       return (
         <button
-          className="link-text tw-mr-2"
+          className="link-text"
           data-testid="run"
           onClick={() =>
             handleTriggerIngestion(ingestion.id as string, ingestion.name)
@@ -475,8 +482,9 @@ const Ingestion: React.FC<IngestionProps> = ({
                           {ingestion.enabled ? (
                             <Fragment>
                               {getTriggerDeployButton(ingestion)}
+                              {separator}
                               <button
-                                className="link-text tw-mr-2"
+                                className="link-text"
                                 data-testid="pause"
                                 disabled={!isRequiredDetailsAvailable}
                                 onClick={() =>
@@ -498,15 +506,17 @@ const Ingestion: React.FC<IngestionProps> = ({
                               Unpause
                             </button>
                           )}
+                          {separator}
                           <button
-                            className="link-text tw-mr-2"
+                            className="link-text"
                             data-testid="edit"
                             disabled={!isRequiredDetailsAvailable}
                             onClick={() => handleUpdate(ingestion)}>
                             Edit
                           </button>
+                          {separator}
                           <button
-                            className="link-text tw-mr-2"
+                            className="link-text"
                             data-testid="delete"
                             onClick={() =>
                               ConfirmDelete(
@@ -524,8 +534,33 @@ const Ingestion: React.FC<IngestionProps> = ({
                               'Delete'
                             )}
                           </button>
+                          {separator}
+                          <button
+                            className="link-text"
+                            data-testid="logs"
+                            disabled={!isRequiredDetailsAvailable}
+                            onClick={() => {
+                              setIsLogsModalOpen(true);
+                              setSelectedPipeline(ingestion);
+                            }}>
+                            Logs
+                          </button>
                         </div>
                       </NonAdminAction>
+                      {isLogsModalOpen &&
+                        selectedPipeline &&
+                        ingestion.id === selectedPipeline?.id && (
+                          <IngestionLogsModal
+                            isModalOpen={isLogsModalOpen}
+                            pipelinName={selectedPipeline.name}
+                            pipelineId={selectedPipeline.id as string}
+                            pipelineType={selectedPipeline.pipelineType}
+                            onClose={() => {
+                              setIsLogsModalOpen(false);
+                              setSelectedPipeline(undefined);
+                            }}
+                          />
+                        )}
                     </td>
                   </tr>
                 ))}

openmetadata-ui/src/main/resources/ui/src/components/Modals/IngestionLogsModal/IngestionLogsModal.test.tsx

@@ -0,0 +1,52 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { render, screen } from '@testing-library/react';
+import React from 'react';
+import { PipelineType } from '../../../generated/api/services/ingestionPipelines/createIngestionPipeline';
+import IngestionLogsModal from './IngestionLogsModal';
+
+const mockProps = {
+  pipelineId: 'bb2ee1a9-653f-4925-a70c-fdbb3abc2d2c',
+  pipelinName: 'MyUnsplash_Service_metadata',
+  pipelineType: PipelineType.Metadata,
+  isModalOpen: true,
+  onClose: jest.fn(),
+};
+
+jest.mock('../../../axiosAPIs/ingestionPipelineAPI', () => ({
+  getIngestionPipelineLogById: jest.fn().mockImplementation(() =>
+    // eslint-disable-next-line @typescript-eslint/camelcase
+    Promise.resolve({ data: { ingestion_task: 'logs' } })
+  ),
+}));
+
+jest.mock('../../buttons/CopyToClipboardButton/CopyToClipboardButton', () =>
+  jest.fn().mockReturnValue(<button data-testid="copy">copy</button>)
+);
+
+describe('Test Ingestion Logs Modal component', () => {
+  it('Should render the component', async () => {
+    render(<IngestionLogsModal {...mockProps} />);
+
+    const container = await screen.findByTestId('logs-modal');
+    const logsBody = await screen.findByTestId('logs-body');
+    const jumpToEndButton = await screen.findByTestId('jump-to-end-button');
+
+    expect(container).toBeInTheDocument();
+
+    expect(logsBody).toBeInTheDocument();
+
+    expect(jumpToEndButton).toBeInTheDocument();
+  });
+});

openmetadata-ui/src/main/resources/ui/src/components/Modals/IngestionLogsModal/IngestionLogsModal.tsx

@@ -0,0 +1,146 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { Viewer } from '@toast-ui/react-editor';
+import { Button, Empty, Modal } from 'antd';
+import { AxiosError, AxiosResponse } from 'axios';
+import classNames from 'classnames';
+import { isNil } from 'lodash';
+import React, { FC, Fragment, useEffect, useState } from 'react';
+import { getIngestionPipelineLogById } from '../../../axiosAPIs/ingestionPipelineAPI';
+import { PipelineType } from '../../../generated/entity/services/ingestionPipelines/ingestionPipeline';
+import { showErrorToast } from '../../../utils/ToastUtils';
+import CopyToClipboardButton from '../../buttons/CopyToClipboardButton/CopyToClipboardButton';
+import Loader from '../../Loader/Loader';
+
+interface IngestionLogsModalProps {
+  pipelineId: string;
+  pipelinName: string;
+  pipelineType: PipelineType;
+  isModalOpen: boolean;
+  onClose: () => void;
+}
+
+const IngestionLogsModal: FC<IngestionLogsModalProps> = ({
+  pipelineId,
+  pipelinName,
+  pipelineType,
+  isModalOpen,
+  onClose,
+}) => {
+  const [logs, setLogs] = useState<string>('');
+  const [isLoading, setIsLoading] = useState<boolean>(false);
+  const [isLogNotFound, setIsLogNotFound] = useState<boolean>(false);
+
+  const fetchLogs = (id: string) => {
+    setIsLoading(true);
+    getIngestionPipelineLogById(id)
+      .then((res: AxiosResponse) => {
+        switch (pipelineType) {
+          case PipelineType.Metadata:
+            setLogs(res.data?.ingestion_task || '');
+
+            break;
+          case PipelineType.Profiler:
+            setLogs(res.data?.profiler_task || '');
+
+            break;
+          case PipelineType.Usage:
+            setLogs(res.data?.usage_task || '');
+
+            break;
+
+          default:
+            setLogs('');
+
+            break;
+        }
+      })
+      .catch((err: AxiosError) => {
+        if (err.response?.status === 404) {
+          setIsLogNotFound(true);
+        } else {
+          showErrorToast(err);
+        }
+      })
+      .finally(() => {
+        setIsLoading(false);
+      });
+  };
+
+  const handleJumpToEnd = () => {
+    const logsBody = document.getElementById('logs-body') as HTMLElement;
+    if (!isNil(logsBody)) {
+      logsBody.scrollTop = logsBody.scrollHeight;
+    }
+  };
+
+  const modalTitle = (
+    <div className="tw-flex tw-justify-between tw-mr-8">
+      {`Logs for ${pipelinName}`} <CopyToClipboardButton copyText={logs} />
+    </div>
+  );
+
+  useEffect(() => {
+    fetchLogs(pipelineId);
+  }, [pipelineId]);
+
+  return (
+    <Modal
+      destroyOnClose
+      afterClose={() => setLogs('')}
+      data-testid="logs-modal"
+      footer={null}
+      title={modalTitle}
+      visible={isModalOpen}
+      width={1200}
+      onCancel={onClose}>
+      {isLoading ? (
+        <Loader />
+      ) : (
+        <Fragment>
+          {logs ? (
+            <Fragment>
+              <Button
+                className="tw-mb-2 ant-btn-primary-custom"
+                data-testid="jump-to-end-button"
+                type="primary"
+                onClick={handleJumpToEnd}>
+                Jump to end
+              </Button>
+              <div
+                className={classNames('tw-overflow-y-auto', {
+                  'tw-h-100': logs,
+                })}
+                data-testid="logs-body"
+                id="logs-body">
+                <Viewer initialValue={logs} />
+              </div>
+            </Fragment>
+          ) : (
+            <Empty
+              data-testid="empty-logs"
+              description={
+                isLogNotFound
+                  ? `No logs yet found for the latest execution of ${pipelinName}`
+                  : 'No logs data available'
+              }
+            />
+          )}
+        </Fragment>
+      )}
+    </Modal>
+  );
+};
+
+export default IngestionLogsModal;