yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-31 12:52:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(frontend/ingestion): Support flagged / warning / connection failure statuses; add recipe (#8920)

Browse Source
This commit is contained in:
Andrew Sikowitz 2023-10-18 13:42:03 -04:00 committed by GitHub
parent bd5c4e0d70
commit b3ac42b1e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 75 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
View File

@ -2,6 +2,7 @@ import { DownloadOutlined } from '@ant-design/icons';
import { Button, message, Modal, Typography } from 'antd';
import React, { useEffect, useState } from 'react';
import styled from 'styled-components';
import YAML from 'yamljs';
import { useGetIngestionExecutionRequestQuery } from '../../../../graphql/ingestion.generated';
import { ANTD_GRAY } from '../../../entity/shared/constants';
import { downloadFile } from '../../../search/utils/csvUtils';
@ -65,6 +66,13 @@ const IngestedAssetsSection = styled.div`
padding-right: 30px;
`;
const RecipeSection = styled.div`
border-top: 1px solid ${ANTD_GRAY[4]};
padding-top: 16px;
padding-left: 30px;
padding-right: 30px;
`;
const LogsSection = styled.div`
padding-top: 16px;
padding-left: 30px;
@ -91,6 +99,8 @@ type Props = {
export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
const [showExpandedLogs, setShowExpandedLogs] = useState(false);
const [showExpandedRecipe, setShowExpandedRecipe] = useState(false);
const { data, loading, error, refetch } = useGetIngestionExecutionRequestQuery({ variables: { urn } });
const output = data?.executionRequest?.result?.report || 'No output found.';
@ -120,7 +130,18 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
const resultSummaryText =
(result && <Typography.Text type="secondary">{getExecutionRequestSummaryText(result)}</Typography.Text>) ||
undefined;
const isOutputExpandable = output.length > 100;
const recipeJson = data?.executionRequest?.input.arguments?.find((arg) => arg.key === 'recipe')?.value;
let recipeYaml: string;
try {
recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
} catch (e) {
recipeYaml = '';
}
const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n').slice(0, 1).join('\n');
const areLogsExpandable = output.length > 100;
const isRecipeExpandable = recipeYaml?.includes('\n');
return (
<Modal
@ -161,14 +182,32 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
</Button>
</SectionSubHeader>
<Typography.Paragraph ellipsis>
<pre>{`${logs}${!showExpandedLogs && isOutputExpandable ? '...' : ''}`}</pre>
{isOutputExpandable && (
<pre>{`${logs}${!showExpandedLogs && areLogsExpandable ? '...' : ''}`}</pre>
{areLogsExpandable && (
<ShowMoreButton type="link" onClick={() => setShowExpandedLogs(!showExpandedLogs)}>
{showExpandedLogs ? 'Hide' : 'Show More'}
</ShowMoreButton>
)}
</Typography.Paragraph>
</LogsSection>
{recipe && (
<RecipeSection>
<SectionHeader level={5}>Recipe</SectionHeader>
<SectionSubHeader>
<SubHeaderParagraph type="secondary">
The recipe used for this ingestion run.
</SubHeaderParagraph>
</SectionSubHeader>
<Typography.Paragraph ellipsis>
<pre>{`${recipe}${!showExpandedRecipe && isRecipeExpandable ? '\n...' : ''}`}</pre>
</Typography.Paragraph>
{isRecipeExpandable && (
<ShowMoreButton type="link" onClick={() => setShowExpandedRecipe((v) => !v)}>
{showExpandedRecipe ? 'Hide' : 'Show More'}
</ShowMoreButton>
)}
</RecipeSection>
)}
</Section>
</Modal>
);

44
datahub-web-react/src/app/ingest/source/utils.ts
View File

@ -1,17 +1,19 @@
import YAML from 'yamljs';
import {
CheckCircleOutlined,
ClockCircleOutlined,
CloseCircleOutlined,
ExclamationCircleOutlined,
LoadingOutlined,
StopOutlined,
WarningOutlined,
} from '@ant-design/icons';
import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants';
import { EntityType, FacetMetadata } from '../../../types.generated';
import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil';
import EntityRegistry from '../../entity/EntityRegistry';
import { SourceConfig } from './builder/types';
import YAML from 'yamljs';
import { ListIngestionSourcesDocument, ListIngestionSourcesQuery } from '../../../graphql/ingestion.generated';
import { EntityType, FacetMetadata } from '../../../types.generated';
import EntityRegistry from '../../entity/EntityRegistry';
import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants';
import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil';
import { SourceConfig } from './builder/types';
export const getSourceConfigs = (ingestionSources: SourceConfig[], sourceType: string) => {
const sourceConfigs = ingestionSources.find((source) => source.name === sourceType);
@ -40,7 +42,9 @@ export function getPlaceholderRecipe(ingestionSources: SourceConfig[], type?: st
export const RUNNING = 'RUNNING';
export const SUCCESS = 'SUCCESS';
export const WARNING = 'WARNING';
export const FAILURE = 'FAILURE';
export const CONNECTION_FAILURE = 'CONNECTION_FAILURE';
export const CANCELLED = 'CANCELLED';
export const UP_FOR_RETRY = 'UP_FOR_RETRY';
export const ROLLING_BACK = 'ROLLING_BACK';
@ -56,8 +60,10 @@ export const getExecutionRequestStatusIcon = (status: string) => {
return (
(status === RUNNING && LoadingOutlined) ||
(status === SUCCESS && CheckCircleOutlined) ||
(status === WARNING && ExclamationCircleOutlined) ||
(status === FAILURE && CloseCircleOutlined) ||
(status === CANCELLED && CloseCircleOutlined) ||
(status === CONNECTION_FAILURE && CloseCircleOutlined) ||
(status === CANCELLED && StopOutlined) ||
(status === UP_FOR_RETRY && ClockCircleOutlined) ||
(status === ROLLED_BACK && WarningOutlined) ||
(status === ROLLING_BACK && LoadingOutlined) ||
@ -70,7 +76,9 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
return (
(status === RUNNING && 'Running') ||
(status === SUCCESS && 'Succeeded') ||
(status === WARNING && 'Completed') ||
(status === FAILURE && 'Failed') ||
(status === CONNECTION_FAILURE && 'Connection Failed') ||
(status === CANCELLED && 'Cancelled') ||
(status === UP_FOR_RETRY && 'Up for Retry') ||
(status === ROLLED_BACK && 'Rolled Back') ||
@ -83,21 +91,25 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
export const getExecutionRequestSummaryText = (status: string) => {
switch (status) {
case RUNNING:
return 'Ingestion is running';
return 'Ingestion is running...';
case SUCCESS:
return 'Ingestion successfully completed';
return 'Ingestion succeeded with no errors or suspected missing data.';
case WARNING:
return 'Ingestion completed with minor or intermittent errors.';
case FAILURE:
return 'Ingestion completed with errors';
return 'Ingestion failed to complete, or completed with serious errors.';
case CONNECTION_FAILURE:
return 'Ingestion failed due to network, authentication, or permission issues.';
case CANCELLED:
return 'Ingestion was cancelled';
return 'Ingestion was cancelled.';
case ROLLED_BACK:
return 'Ingestion was rolled back';
return 'Ingestion was rolled back.';
case ROLLING_BACK:
return 'Ingestion is in the process of rolling back';
return 'Ingestion is in the process of rolling back.';
case ROLLBACK_FAILED:
return 'Ingestion rollback failed';
return 'Ingestion rollback failed.';
default:
return 'Ingestion status not recognized';
return 'Ingestion status not recognized.';
}
};
@ -105,7 +117,9 @@ export const getExecutionRequestStatusDisplayColor = (status: string) => {
return (
(status === RUNNING && REDESIGN_COLORS.BLUE) ||
(status === SUCCESS && 'green') ||
(status === WARNING && 'orangered') ||
(status === FAILURE && 'red') ||
(status === CONNECTION_FAILURE && 'crimson') ||
(status === UP_FOR_RETRY && 'orange') ||
(status === CANCELLED && ANTD_GRAY[9]) ||
(status === ROLLED_BACK && 'orange') ||

4
datahub-web-react/src/graphql/ingestion.graphql
View File

@ -90,6 +90,10 @@ query getIngestionExecutionRequest($urn: String!) {
source {
type
}
arguments {
key
value
}
}
result {
status