feat(frontend/ingestion): Support flagged / warning / connection failure statuses; add recipe (#8920)

datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx

@@ -2,6 +2,7 @@ import { DownloadOutlined } from '@ant-design/icons';
 import { Button, message, Modal, Typography } from 'antd';
 import React, { useEffect, useState } from 'react';
 import styled from 'styled-components';
+import YAML from 'yamljs';
 import { useGetIngestionExecutionRequestQuery } from '../../../../graphql/ingestion.generated';
 import { ANTD_GRAY } from '../../../entity/shared/constants';
 import { downloadFile } from '../../../search/utils/csvUtils';
@@ -65,6 +66,13 @@ const IngestedAssetsSection = styled.div`
     padding-right: 30px;
 `;
 
+const RecipeSection = styled.div`
+    border-top: 1px solid ${ANTD_GRAY[4]};
+    padding-top: 16px;
+    padding-left: 30px;
+    padding-right: 30px;
+`;
+
 const LogsSection = styled.div`
     padding-top: 16px;
     padding-left: 30px;
@@ -91,6 +99,8 @@ type Props = {
 
 export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
     const [showExpandedLogs, setShowExpandedLogs] = useState(false);
+    const [showExpandedRecipe, setShowExpandedRecipe] = useState(false);
+
     const { data, loading, error, refetch } = useGetIngestionExecutionRequestQuery({ variables: { urn } });
     const output = data?.executionRequest?.result?.report || 'No output found.';
 
@@ -120,7 +130,18 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
     const resultSummaryText =
         (result && <Typography.Text type="secondary">{getExecutionRequestSummaryText(result)}</Typography.Text>) ||
         undefined;
-    const isOutputExpandable = output.length > 100;
+
+    const recipeJson = data?.executionRequest?.input.arguments?.find((arg) => arg.key === 'recipe')?.value;
+    let recipeYaml: string;
+    try {
+        recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
+    } catch (e) {
+        recipeYaml = '';
+    }
+    const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n').slice(0, 1).join('\n');
+
+    const areLogsExpandable = output.length > 100;
+    const isRecipeExpandable = recipeYaml?.includes('\n');
 
     return (
         <Modal
@@ -161,14 +182,32 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
                         </Button>
                     </SectionSubHeader>
                     <Typography.Paragraph ellipsis>
-                        <pre>{`${logs}${!showExpandedLogs && isOutputExpandable ? '...' : ''}`}</pre>
+                        <pre>{`${logs}${!showExpandedLogs && areLogsExpandable ? '...' : ''}`}</pre>
-                        {isOutputExpandable && (
+                        {areLogsExpandable && (
                             <ShowMoreButton type="link" onClick={() => setShowExpandedLogs(!showExpandedLogs)}>
                                 {showExpandedLogs ? 'Hide' : 'Show More'}
                             </ShowMoreButton>
                         )}
                     </Typography.Paragraph>
                 </LogsSection>
+                {recipe && (
+                    <RecipeSection>
+                        <SectionHeader level={5}>Recipe</SectionHeader>
+                        <SectionSubHeader>
+                            <SubHeaderParagraph type="secondary">
+                                The recipe used for this ingestion run.
+                            </SubHeaderParagraph>
+                        </SectionSubHeader>
+                        <Typography.Paragraph ellipsis>
+                            <pre>{`${recipe}${!showExpandedRecipe && isRecipeExpandable ? '\n...' : ''}`}</pre>
+                        </Typography.Paragraph>
+                        {isRecipeExpandable && (
+                            <ShowMoreButton type="link" onClick={() => setShowExpandedRecipe((v) => !v)}>
+                                {showExpandedRecipe ? 'Hide' : 'Show More'}
+                            </ShowMoreButton>
+                        )}
+                    </RecipeSection>
+                )}
             </Section>
         </Modal>
     );

datahub-web-react/src/app/ingest/source/utils.ts

@@ -1,17 +1,19 @@
-import YAML from 'yamljs';
 import {
     CheckCircleOutlined,
     ClockCircleOutlined,
     CloseCircleOutlined,
+    ExclamationCircleOutlined,
     LoadingOutlined,
+    StopOutlined,
     WarningOutlined,
 } from '@ant-design/icons';
-import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants';
+import YAML from 'yamljs';
-import { EntityType, FacetMetadata } from '../../../types.generated';
-import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil';
-import EntityRegistry from '../../entity/EntityRegistry';
-import { SourceConfig } from './builder/types';
 import { ListIngestionSourcesDocument, ListIngestionSourcesQuery } from '../../../graphql/ingestion.generated';
+import { EntityType, FacetMetadata } from '../../../types.generated';
+import EntityRegistry from '../../entity/EntityRegistry';
+import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants';
+import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil';
+import { SourceConfig } from './builder/types';
 
 export const getSourceConfigs = (ingestionSources: SourceConfig[], sourceType: string) => {
     const sourceConfigs = ingestionSources.find((source) => source.name === sourceType);
@@ -40,7 +42,9 @@ export function getPlaceholderRecipe(ingestionSources: SourceConfig[], type?: st
 
 export const RUNNING = 'RUNNING';
 export const SUCCESS = 'SUCCESS';
+export const WARNING = 'WARNING';
 export const FAILURE = 'FAILURE';
+export const CONNECTION_FAILURE = 'CONNECTION_FAILURE';
 export const CANCELLED = 'CANCELLED';
 export const UP_FOR_RETRY = 'UP_FOR_RETRY';
 export const ROLLING_BACK = 'ROLLING_BACK';
@@ -56,8 +60,10 @@ export const getExecutionRequestStatusIcon = (status: string) => {
     return (
         (status === RUNNING && LoadingOutlined) ||
         (status === SUCCESS && CheckCircleOutlined) ||
+        (status === WARNING && ExclamationCircleOutlined) ||
         (status === FAILURE && CloseCircleOutlined) ||
-        (status === CANCELLED && CloseCircleOutlined) ||
+        (status === CONNECTION_FAILURE && CloseCircleOutlined) ||
+        (status === CANCELLED && StopOutlined) ||
         (status === UP_FOR_RETRY && ClockCircleOutlined) ||
         (status === ROLLED_BACK && WarningOutlined) ||
         (status === ROLLING_BACK && LoadingOutlined) ||
@@ -70,7 +76,9 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
     return (
         (status === RUNNING && 'Running') ||
         (status === SUCCESS && 'Succeeded') ||
+        (status === WARNING && 'Completed') ||
         (status === FAILURE && 'Failed') ||
+        (status === CONNECTION_FAILURE && 'Connection Failed') ||
         (status === CANCELLED && 'Cancelled') ||
         (status === UP_FOR_RETRY && 'Up for Retry') ||
         (status === ROLLED_BACK && 'Rolled Back') ||
@@ -83,21 +91,25 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
 export const getExecutionRequestSummaryText = (status: string) => {
     switch (status) {
         case RUNNING:
-            return 'Ingestion is running';
+            return 'Ingestion is running...';
         case SUCCESS:
-            return 'Ingestion successfully completed';
+            return 'Ingestion succeeded with no errors or suspected missing data.';
+        case WARNING:
+            return 'Ingestion completed with minor or intermittent errors.';
         case FAILURE:
-            return 'Ingestion completed with errors';
+            return 'Ingestion failed to complete, or completed with serious errors.';
+        case CONNECTION_FAILURE:
+            return 'Ingestion failed due to network, authentication, or permission issues.';
         case CANCELLED:
-            return 'Ingestion was cancelled';
+            return 'Ingestion was cancelled.';
         case ROLLED_BACK:
-            return 'Ingestion was rolled back';
+            return 'Ingestion was rolled back.';
         case ROLLING_BACK:
-            return 'Ingestion is in the process of rolling back';
+            return 'Ingestion is in the process of rolling back.';
         case ROLLBACK_FAILED:
-            return 'Ingestion rollback failed';
+            return 'Ingestion rollback failed.';
         default:
-            return 'Ingestion status not recognized';
+            return 'Ingestion status not recognized.';
     }
 };
 
@@ -105,7 +117,9 @@ export const getExecutionRequestStatusDisplayColor = (status: string) => {
     return (
         (status === RUNNING && REDESIGN_COLORS.BLUE) ||
         (status === SUCCESS && 'green') ||
+        (status === WARNING && 'orangered') ||
         (status === FAILURE && 'red') ||
+        (status === CONNECTION_FAILURE && 'crimson') ||
         (status === UP_FOR_RETRY && 'orange') ||
         (status === CANCELLED && ANTD_GRAY[9]) ||
         (status === ROLLED_BACK && 'orange') ||

datahub-web-react/src/graphql/ingestion.graphql

@@ -90,6 +90,10 @@ query getIngestionExecutionRequest($urn: String!) {
             source {
                 type
             }
+            arguments {
+                key
+                value
+            }
         }
         result {
             status