fixed: #14435 support download option for ingestion logs (#17176)

* support download option for ingestion logs

* fix tests

openmetadata-ui/src/main/resources/ui/src/hooks/useDownloadProgressStore.ts

@@ -0,0 +1,31 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { create } from 'zustand';
+
+interface DownloadProgressStore {
+  progress: number;
+  updateProgress: (percentage: number) => void;
+  reset: () => void;
+}
+
+export const useDownloadProgressStore = create<DownloadProgressStore>()(
+  (set) => ({
+    progress: 0,
+    updateProgress: (percentage: number) => {
+      set({ progress: percentage });
+    },
+    reset: () => {
+      set({ progress: 0 });
+    },
+  })
+);

openmetadata-ui/src/main/resources/ui/src/pages/LogsViewerPage/LogsViewerPage.tsx

@@ -11,9 +11,10 @@
  *  limitations under the License.
  */
 
-import { Button, Col, Row, Space, Typography } from 'antd';
+import { DownloadOutlined } from '@ant-design/icons';
+import { Button, Col, Progress, Row, Space, Tooltip, Typography } from 'antd';
 import { AxiosError } from 'axios';
-import { isEmpty, isNil, isUndefined, toNumber } from 'lodash';
+import { isEmpty, isNil, isUndefined, round, toNumber } from 'lodash';
 import React, {
   Fragment,
   useCallback,
@@ -39,6 +40,7 @@ import {
 } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
 import { Include } from '../../generated/type/include';
 import { Paging } from '../../generated/type/paging';
+import { useDownloadProgressStore } from '../../hooks/useDownloadProgressStore';
 import { useFqn } from '../../hooks/useFqn';
 import {
   getApplicationByName,
@@ -50,6 +52,8 @@ import {
   getIngestionPipelineLogById,
 } from '../../rest/ingestionPipelineAPI';
 import { getEpochMillisForPastDays } from '../../utils/date-time/DateTimeUtils';
+import { getEntityName } from '../../utils/EntityUtils';
+import { downloadIngestionLog } from '../../utils/IngestionLogs/LogsUtils';
 import logsClassBase from '../../utils/LogsClassBase';
 import { showErrorToast } from '../../utils/ToastUtils';
 import './logs-viewer-page.style.less';
@@ -61,7 +65,7 @@ const LogsViewerPage = () => {
   const { fqn: ingestionName } = useFqn();
 
   const { t } = useTranslation();
-
+  const { progress, reset, updateProgress } = useDownloadProgressStore();
   const [isLoading, setIsLoading] = useState<boolean>(false);
   const [logs, setLogs] = useState<string>('');
   const [ingestionDetails, setIngestionDetails] = useState<IngestionPipeline>();
@@ -298,6 +302,37 @@ const LogsViewerPage = () => {
     };
   }, [ingestionDetails, appData, recentRuns]);
 
+  const handleIngestionDownloadClick = async () => {
+    try {
+      reset();
+      const progress = round(
+        (Number(paging?.after) * 100) / Number(paging?.total)
+      );
+
+      updateProgress(paging?.after ? progress : 1);
+
+      const logs = await downloadIngestionLog(
+        ingestionDetails?.id,
+        ingestionDetails?.pipelineType
+      );
+
+      const element = document.createElement('a');
+      const file = new Blob([logs || ''], { type: 'text/plain' });
+      element.href = URL.createObjectURL(file);
+      element.download = `${getEntityName(ingestionDetails)}-${
+        ingestionDetails?.pipelineType
+      }.log`;
+      document.body.appendChild(element);
+      element.click();
+      document.body.removeChild(element);
+    } catch (err) {
+      showErrorToast(err as AxiosError);
+    } finally {
+      setIsLoading(false);
+      reset();
+    }
+  };
+
   if (isLoading) {
     return <Loader />;
   }
@@ -339,6 +374,32 @@ const LogsViewerPage = () => {
                   <Col>
                     <CopyToClipboardButton copyText={logs} />
                   </Col>
+                  <Col>
+                    {progress ? (
+                      <Tooltip title={`${progress}%`}>
+                        <Progress
+                          className="h-8 m-l-md relative flex-center"
+                          percent={progress}
+                          strokeWidth={5}
+                          type="circle"
+                          width={32}
+                        />
+                      </Tooltip>
+                    ) : (
+                      <Button
+                        className="h-8 m-l-md relative flex-center"
+                        data-testid="download"
+                        icon={
+                          <DownloadOutlined
+                            data-testid="download-icon"
+                            width="16"
+                          />
+                        }
+                        type="text"
+                        onClick={handleIngestionDownloadClick}
+                      />
+                    )}
+                  </Col>
                 </Row>
               </Col>
               <Col

openmetadata-ui/src/main/resources/ui/src/pages/LogsViewerPage/LogsViewerPageSkeleton.component.tsx

@@ -21,6 +21,7 @@ const LogViewerPageSkeleton = () => {
         <div className="flex items-center gap-2 justify-end">
           <Skeleton.Button active />
           <Skeleton.Button active shape="circle" />
+          <Skeleton.Button active shape="circle" />
         </div>
         <Skeleton
           active

openmetadata-ui/src/main/resources/ui/src/utils/IngestionLogs/LogsUtils.test.ts

@@ -0,0 +1,199 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
+import { useDownloadProgressStore } from '../../hooks/useDownloadProgressStore';
+import { IngestionPipelineLogByIdInterface } from '../../pages/LogsViewerPage/LogsViewerPage.interfaces';
+import { getIngestionPipelineLogById } from '../../rest/ingestionPipelineAPI';
+import { showErrorToast } from '../ToastUtils';
+import {
+  downloadIngestionLog,
+  fetchLogsRecursively,
+  getLogsFromResponse,
+} from './LogsUtils';
+
+const mockUpdateProgress = jest.fn();
+
+jest.mock('../../rest/ingestionPipelineAPI');
+jest.mock('../ToastUtils');
+jest.mock('../../hooks/useDownloadProgressStore');
+jest.mock('../../hooks/useDownloadProgressStore', () => ({
+  useDownloadProgressStore: {
+    getState: jest.fn().mockImplementation(() => ({
+      updateProgress: mockUpdateProgress,
+    })),
+  },
+}));
+
+import * as utils from './LogsUtils';
+
+describe('LogsUtils', () => {
+  describe('getLogsFromResponse', () => {
+    it('should return the correct logs based on the pipeline type', () => {
+      const res: IngestionPipelineLogByIdInterface = {
+        ingestion_task: 'metadata_logs',
+        application_task: 'application_logs',
+        profiler_task: 'profiler_logs',
+        usage_task: 'usage_logs',
+        lineage_task: 'lineage_logs',
+        dbt_task: 'dbt_logs',
+        test_suite_task: 'test_suite_logs',
+        data_insight_task: 'data_insight_logs',
+        elasticsearch_reindex_task: 'elasticsearch_reindex_logs',
+      };
+
+      expect(getLogsFromResponse(res, PipelineType.Metadata)).toBe(
+        'metadata_logs'
+      );
+      expect(getLogsFromResponse(res, PipelineType.Application)).toBe(
+        'application_logs'
+      );
+      expect(getLogsFromResponse(res, PipelineType.Profiler)).toBe(
+        'profiler_logs'
+      );
+      expect(getLogsFromResponse(res, PipelineType.Usage)).toBe('usage_logs');
+      expect(getLogsFromResponse(res, PipelineType.Lineage)).toBe(
+        'lineage_logs'
+      );
+      expect(getLogsFromResponse(res, PipelineType.Dbt)).toBe('dbt_logs');
+      expect(getLogsFromResponse(res, PipelineType.TestSuite)).toBe(
+        'test_suite_logs'
+      );
+      expect(getLogsFromResponse(res, PipelineType.DataInsight)).toBe(
+        'data_insight_logs'
+      );
+      expect(getLogsFromResponse(res, PipelineType.ElasticSearchReindex)).toBe(
+        'elasticsearch_reindex_logs'
+      );
+      expect(getLogsFromResponse(res, 'unknown_pipeline')).toBe('');
+    });
+  });
+
+  describe('fetchLogsRecursively', () => {
+    let mockGetLogsFromResponse: jest.SpyInstance;
+
+    beforeAll(() => {
+      mockGetLogsFromResponse = jest.spyOn(utils, 'getLogsFromResponse');
+    });
+
+    afterAll(() => {
+      jest.clearAllMocks();
+    });
+
+    const ingestionId = '123';
+    const pipelineType = PipelineType.Metadata;
+    const after = '456';
+
+    it('should fetch logs recursively and return the concatenated logs', async () => {
+      const total = '100';
+      const afterCursor = '50';
+      const rest: IngestionPipelineLogByIdInterface = {
+        ingestion_task: 'metadata_logs_1',
+      };
+
+      (getIngestionPipelineLogById as jest.Mock).mockResolvedValueOnce({
+        data: { total, after: afterCursor, ...rest },
+      });
+
+      (getIngestionPipelineLogById as jest.Mock).mockResolvedValueOnce({
+        data: { total, after: null, ...rest },
+      });
+
+      const logs = await fetchLogsRecursively(ingestionId, pipelineType, after);
+
+      expect(getIngestionPipelineLogById).toHaveBeenCalledTimes(2);
+      expect(getIngestionPipelineLogById).toHaveBeenCalledWith(
+        ingestionId,
+        after
+      );
+      expect(mockGetLogsFromResponse).toHaveBeenCalledWith(rest, pipelineType);
+      expect(logs).toBe('metadata_logs_1metadata_logs_1');
+    });
+
+    it('should update the download progress when afterCursor and total are available', async () => {
+      const total = '100';
+      const afterCursor = '50';
+      const rest: IngestionPipelineLogByIdInterface = {
+        ingestion_task: 'metadata_logs_1',
+      };
+
+      (getIngestionPipelineLogById as jest.Mock).mockResolvedValueOnce({
+        data: { total, after: afterCursor, ...rest },
+      });
+
+      (getIngestionPipelineLogById as jest.Mock).mockResolvedValueOnce({
+        data: { total, after: null, ...rest },
+      });
+
+      await fetchLogsRecursively(ingestionId, pipelineType, after);
+
+      expect(
+        useDownloadProgressStore.getState().updateProgress
+      ).toHaveBeenCalledWith(50);
+    });
+  });
+
+  describe('downloadIngestionLog', () => {
+    const ingestionId = '123';
+    const pipelineType = PipelineType.Metadata;
+    let mockFetchLogsRecursively: jest.SpyInstance;
+
+    beforeAll(() => {
+      mockFetchLogsRecursively = jest.spyOn(utils, 'fetchLogsRecursively');
+    });
+
+    afterAll(() => {
+      jest.clearAllMocks();
+    });
+
+    it('should return the downloaded logs', async () => {
+      const logs = 'metadata_logs';
+
+      mockFetchLogsRecursively.mockResolvedValueOnce(logs);
+
+      const result = await downloadIngestionLog(ingestionId, pipelineType);
+
+      expect(mockFetchLogsRecursively).toHaveBeenCalledWith(
+        ingestionId,
+        pipelineType
+      );
+      expect(result).toBe(logs);
+    });
+
+    it('should show error toast and return empty string if an error occurs', async () => {
+      const error = new Error('Failed to fetch logs');
+
+      mockFetchLogsRecursively.mockRejectedValueOnce(error);
+
+      const result = await downloadIngestionLog(ingestionId, pipelineType);
+
+      expect(mockFetchLogsRecursively).toHaveBeenCalledWith(
+        ingestionId,
+        pipelineType
+      );
+      expect(showErrorToast).toHaveBeenCalledWith(error);
+      expect(result).toBe('');
+    });
+
+    it('should return empty string if ingestionId or pipelineType is not provided', async () => {
+      const result = await downloadIngestionLog(undefined, pipelineType);
+
+      expect(mockFetchLogsRecursively).not.toHaveBeenCalled();
+      expect(result).toBe('');
+
+      const result2 = await downloadIngestionLog(ingestionId, undefined);
+
+      expect(mockFetchLogsRecursively).not.toHaveBeenCalled();
+      expect(result2).toBe('');
+    });
+  });
+});

openmetadata-ui/src/main/resources/ui/src/utils/IngestionLogs/LogsUtils.ts

@@ -0,0 +1,96 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { AxiosError } from 'axios';
+import { round } from 'lodash';
+import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
+import { useDownloadProgressStore } from '../../hooks/useDownloadProgressStore';
+import { IngestionPipelineLogByIdInterface } from '../../pages/LogsViewerPage/LogsViewerPage.interfaces';
+import { getIngestionPipelineLogById } from '../../rest/ingestionPipelineAPI';
+import { showErrorToast } from '../ToastUtils';
+
+export const getLogsFromResponse = (
+  res: IngestionPipelineLogByIdInterface,
+  pipelineType: string
+) => {
+  switch (pipelineType) {
+    case PipelineType.Metadata:
+      return res.ingestion_task || '';
+
+    case PipelineType.Application:
+      return res.application_task || '';
+
+    case PipelineType.Profiler:
+      return res.profiler_task || '';
+
+    case PipelineType.Usage:
+      return res.usage_task || '';
+
+    case PipelineType.Lineage:
+      return res.lineage_task || '';
+
+    case PipelineType.Dbt:
+      return res.dbt_task || '';
+
+    case PipelineType.TestSuite:
+      return res.test_suite_task || '';
+
+    case PipelineType.DataInsight:
+      return res.data_insight_task || '';
+
+    case PipelineType.ElasticSearchReindex:
+      return res.elasticsearch_reindex_task || '';
+
+    default:
+      return '';
+  }
+};
+
+export const fetchLogsRecursively = async (
+  ingestionId: string,
+  pipelineType: string,
+  after?: string
+) => {
+  let logs = '';
+  const {
+    data: { total, after: afterCursor, ...rest },
+  } = await getIngestionPipelineLogById(ingestionId, after);
+
+  logs = logs.concat(getLogsFromResponse(rest, pipelineType));
+  if (afterCursor && total) {
+    const progress = round((Number(afterCursor) * 100) / Number(total));
+    useDownloadProgressStore.getState().updateProgress(progress);
+
+    logs = logs.concat(
+      await fetchLogsRecursively(ingestionId, pipelineType, afterCursor)
+    );
+  }
+
+  return logs;
+};
+
+export const downloadIngestionLog = async (
+  ingestionId?: string,
+  pipelineType?: PipelineType
+) => {
+  if (!ingestionId || !pipelineType) {
+    return '';
+  }
+
+  try {
+    return await fetchLogsRecursively(ingestionId, pipelineType);
+  } catch (err) {
+    showErrorToast(err as AxiosError);
+
+    return '';
+  }
+};