mirror of
https://github.com/datahub-project/datahub.git
synced 2025-08-08 09:18:06 +00:00
feat(ingest): change asset list (#14073)
This commit is contained in:
Aseem Bansal
GitHub
parent
b8aac8b4e3
commit
e539e33e10
@ -16,7 +16,14 @@
|
||||
"Bash(ruff:*)",
|
||||
"Bash(python -m mypy:*)",
|
||||
"Bash(python -m ruff:*)",
|
||||
"Bash(python -m pytest:*)"
|
||||
"Bash(python -m pytest:*)",
|
||||
"Bash(yarn format:*)",
|
||||
"Bash(yarn lint:*)",
|
||||
"Bash(yarn type-check:*)",
|
||||
"Bash(yarn test:*)",
|
||||
"Bash(yarn generate:*)",
|
||||
"Bash(./gradlew :datahub-web-react:yarnLintFix)",
|
||||
"Bash(./gradlew :datahub-web-react:yarnLint)"
|
||||
],
|
||||
"deny": []
|
||||
}
|
||||
|
||||
@ -174,8 +174,19 @@ yarn generate
|
||||
# Run formatting
|
||||
yarn format
|
||||
|
||||
# Run linting
|
||||
yarn lint
|
||||
# Run linting on all files
|
||||
../gradlew yarnLint
|
||||
|
||||
# Run lint-fix on a single file
|
||||
../gradlew -x yarnInstall -x yarnGenerate yarnLintFix -Pfile=src/path/to/file.tsx
|
||||
|
||||
# Run linting on a single file
|
||||
# This does not run full type-check when we run for a single file
|
||||
# that should be run at the end of all changes before commit
|
||||
../gradlew -x yarnInstall -x yarnGenerate yarnLint -Pfile=src/path/to/file.tsx
|
||||
|
||||
# Run lint-fix on all files
|
||||
../gradlew yarnLintFix
|
||||
|
||||
# Run type checking
|
||||
yarn type-check
|
||||
|
||||
@ -105,12 +105,38 @@ task yarnTest(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||
}
|
||||
|
||||
task yarnLint(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||
def targetFile = project.findProperty('file') ?: ''
|
||||
|
||||
if (targetFile.isEmpty()) {
|
||||
// Run on all files
|
||||
args = ['run', 'lint']
|
||||
} else {
|
||||
// Run on specific file - run lint, format-check, and type-check
|
||||
doFirst {
|
||||
// Run lint-file first
|
||||
exec {
|
||||
workingDir projectDir
|
||||
commandLine 'yarn', 'run', 'lint-file', targetFile
|
||||
}
|
||||
// Then run format-check-file
|
||||
exec {
|
||||
workingDir projectDir
|
||||
commandLine 'yarn', 'run', 'format-check-file', targetFile
|
||||
}
|
||||
// This does not run full type-check when we run for a single file
|
||||
// as that would take too long
|
||||
}
|
||||
outputs.cacheIf { false }
|
||||
}
|
||||
}
|
||||
|
||||
test.dependsOn([yarnInstall, yarnTest, yarnLint])
|
||||
|
||||
task yarnLintFix(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||
def targetFile = project.findProperty('file') ?: ''
|
||||
|
||||
if (targetFile.isEmpty()) {
|
||||
// Run on all files
|
||||
args = ['run', 'lint-fix']
|
||||
def lint_sentinel = "${buildDir}/.yarn-lint-sentinel"
|
||||
outputs.file(lint_sentinel)
|
||||
@ -125,6 +151,22 @@ task yarnLintFix(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||
}
|
||||
}
|
||||
outputs.cacheIf { true }
|
||||
} else {
|
||||
// Run on specific file - need to run both lint-fix and format
|
||||
doFirst {
|
||||
// Run lint-fix-file first
|
||||
exec {
|
||||
workingDir projectDir
|
||||
commandLine 'yarn', 'run', 'lint-fix-file', targetFile
|
||||
}
|
||||
// Then run format-file
|
||||
exec {
|
||||
workingDir projectDir
|
||||
commandLine 'yarn', 'run', 'format-file', targetFile
|
||||
}
|
||||
}
|
||||
outputs.cacheIf { false }
|
||||
}
|
||||
}
|
||||
|
||||
task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||
|
||||
@ -115,8 +115,12 @@
|
||||
"generate": "NODE_OPTIONS='--max-old-space-size=5120 --openssl-legacy-provider' graphql-codegen --config codegen.yml",
|
||||
"lint": "eslint . --ext .ts,.tsx --quiet && yarn format-check && yarn type-check",
|
||||
"lint-fix": "eslint . --ext .ts,.tsx --quiet --fix && yarn format",
|
||||
"lint-file": "eslint --ext .ts,.tsx --quiet",
|
||||
"format-check-file": "prettier --check",
|
||||
"lint-fix-file": "eslint --ext .ts,.tsx --quiet --fix",
|
||||
"format-check": "prettier --check src",
|
||||
"format": "prettier --write src",
|
||||
"format-file": "prettier --write",
|
||||
"type-check": "tsc --noEmit",
|
||||
"type-watch": "tsc -w --noEmit",
|
||||
"storybook": "storybook dev -p 6006",
|
||||
|
||||
@ -42,6 +42,12 @@ const HeaderContainer = styled.div<{ hasChildren: boolean }>`
|
||||
flex-direction: column;
|
||||
`;
|
||||
|
||||
const TitleRow = styled.div`
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
`;
|
||||
|
||||
const ButtonsContainer = styled.div`
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
@ -59,6 +65,7 @@ export interface ModalProps {
|
||||
buttons: ModalButton[];
|
||||
title: string;
|
||||
subtitle?: string;
|
||||
titlePill?: React.ReactNode;
|
||||
children?: React.ReactNode;
|
||||
onCancel: () => void;
|
||||
dataTestId?: string;
|
||||
@ -68,6 +75,7 @@ export function Modal({
|
||||
buttons,
|
||||
title,
|
||||
subtitle,
|
||||
titlePill,
|
||||
children,
|
||||
onCancel,
|
||||
dataTestId,
|
||||
@ -83,9 +91,12 @@ export function Modal({
|
||||
data-testid={dataTestId}
|
||||
title={
|
||||
<HeaderContainer hasChildren={!!children}>
|
||||
<TitleRow>
|
||||
<Heading type="h1" color="gray" colorLevel={600} weight="bold" size="lg">
|
||||
{title}
|
||||
</Heading>
|
||||
{titlePill}
|
||||
</TitleRow>
|
||||
{!!subtitle && (
|
||||
<Text type="span" color="gray" colorLevel={1700} weight="medium">
|
||||
{subtitle}
|
||||
|
||||
@ -90,6 +90,19 @@ const meta = {
|
||||
getCurrentUrl: {
|
||||
description: 'A custom function to get the current URL. Defaults to window.location.pathname',
|
||||
},
|
||||
scrollToTopOnChange: {
|
||||
description: 'Whether to scroll to the top of the tabs container when switching tabs',
|
||||
control: { type: 'boolean' },
|
||||
},
|
||||
maxHeight: {
|
||||
description:
|
||||
'Maximum height of the scrollable tabs container (only applies when scrollToTopOnChange is true)',
|
||||
control: { type: 'text' },
|
||||
},
|
||||
stickyHeader: {
|
||||
description: 'Whether to make the tab headers sticky when scrolling',
|
||||
control: { type: 'boolean' },
|
||||
},
|
||||
},
|
||||
|
||||
// Args for the story
|
||||
@ -136,3 +149,77 @@ export const urlAware: Story = {
|
||||
tags: ['dev'],
|
||||
render: () => <UrlAwareTabsDemo />,
|
||||
};
|
||||
|
||||
const StickyHeaderTabsDemo = () => {
|
||||
const [selectedTab, setSelectedTab] = React.useState('tab1');
|
||||
|
||||
const stickyTabs = [
|
||||
{
|
||||
key: 'tab1',
|
||||
name: 'Tab One',
|
||||
component: (
|
||||
<div style={{ padding: '20px' }}>
|
||||
<h3>Tab One with Sticky Header</h3>
|
||||
{Array.from({ length: 40 }, (_, i) => (
|
||||
<p key={i}>
|
||||
This is paragraph {i + 1}. The tab header will stick to the top when you scroll down. Lorem
|
||||
ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut
|
||||
labore et dolore magna aliqua.
|
||||
</p>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: 'tab2',
|
||||
name: 'Tab Two',
|
||||
component: (
|
||||
<div style={{ padding: '20px' }}>
|
||||
<h3>Tab Two Content</h3>
|
||||
{Array.from({ length: 35 }, (_, i) => (
|
||||
<div key={i} style={{ marginBottom: '15px', padding: '15px', backgroundColor: '#f0f0f0' }}>
|
||||
Content block {i + 1} - Notice how the tab header remains visible at the top while
|
||||
scrolling.
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: 'tab3',
|
||||
name: 'Tab Three',
|
||||
component: (
|
||||
<div style={{ padding: '20px' }}>
|
||||
<h3>Tab Three Content</h3>
|
||||
{Array.from({ length: 50 }, (_, i) => (
|
||||
<p key={i}>
|
||||
Line {i + 1}: The sticky header allows you to easily switch between tabs even when scrolled
|
||||
deep into the content. This is particularly useful for long content like logs or detailed
|
||||
configuration files.
|
||||
</p>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<div style={{ width: '700px', height: '500px' }}>
|
||||
<h4>Sticky Header & Scroll to Top on Tab Change Demo</h4>
|
||||
<p>Scroll down within the tab content to see the sticky header behavior and switch between tabs</p>
|
||||
<Tabs
|
||||
tabs={stickyTabs}
|
||||
selectedTab={selectedTab}
|
||||
onChange={setSelectedTab}
|
||||
scrollToTopOnChange
|
||||
maxHeight="400px"
|
||||
stickyHeader
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export const stickyHeader: Story = {
|
||||
tags: ['dev'],
|
||||
render: () => <StickyHeaderTabsDemo />,
|
||||
};
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Tabs as AntTabs } from 'antd';
|
||||
import React, { useEffect } from 'react';
|
||||
import React, { useEffect, useRef } from 'react';
|
||||
import styled from 'styled-components';
|
||||
|
||||
import { Pill } from '@components/components/Pills';
|
||||
@ -7,9 +7,21 @@ import { Tooltip } from '@components/components/Tooltip';
|
||||
|
||||
import { colors } from '@src/alchemy-components/theme';
|
||||
|
||||
const StyledTabs = styled(AntTabs)<{ $addPaddingLeft?: boolean; $hideTabsHeader: boolean }>`
|
||||
flex: 1;
|
||||
overflow: hidden;
|
||||
const ScrollableTabsContainer = styled.div<{ $maxHeight?: string }>`
|
||||
max-height: ${({ $maxHeight }) => $maxHeight || '100%'};
|
||||
overflow-y: auto;
|
||||
height: 100%;
|
||||
position: relative;
|
||||
`;
|
||||
|
||||
const StyledTabs = styled(AntTabs)<{
|
||||
$addPaddingLeft?: boolean;
|
||||
$hideTabsHeader: boolean;
|
||||
$scrollable?: boolean;
|
||||
$stickyHeader?: boolean;
|
||||
}>`
|
||||
${({ $scrollable }) => !$scrollable && 'flex: 1;'}
|
||||
${({ $scrollable }) => !$scrollable && 'overflow: hidden;'}
|
||||
|
||||
.ant-tabs-tab {
|
||||
padding: 8px 0;
|
||||
@ -38,6 +50,17 @@ const StyledTabs = styled(AntTabs)<{ $addPaddingLeft?: boolean; $hideTabsHeader:
|
||||
}
|
||||
`}
|
||||
|
||||
${({ $stickyHeader }) =>
|
||||
$stickyHeader &&
|
||||
`
|
||||
.ant-tabs-nav {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
background-color: white;
|
||||
}
|
||||
`}
|
||||
|
||||
.ant-tabs-tab-active .ant-tabs-tab-btn {
|
||||
color: ${(props) => props.theme.styles['primary-color']};
|
||||
font-weight: 600;
|
||||
@ -58,6 +81,37 @@ const StyledTabs = styled(AntTabs)<{ $addPaddingLeft?: boolean; $hideTabsHeader:
|
||||
.ant-tabs-nav {
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
${({ $stickyHeader }) =>
|
||||
$stickyHeader &&
|
||||
`
|
||||
.ant-tabs-nav::before {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.ant-tabs-nav-wrap::before {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.ant-tabs-nav-list::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 1px;
|
||||
background-color: ${colors.gray[200]};
|
||||
}
|
||||
`}
|
||||
|
||||
${({ $addPaddingLeft, $stickyHeader }) =>
|
||||
$addPaddingLeft &&
|
||||
$stickyHeader &&
|
||||
`
|
||||
.ant-tabs-nav-list::after {
|
||||
left: 16px;
|
||||
}
|
||||
`}
|
||||
`;
|
||||
|
||||
const TabViewWrapper = styled.div<{ $disabled?: boolean }>`
|
||||
@ -100,6 +154,9 @@ export interface Props {
|
||||
getCurrentUrl?: () => string;
|
||||
addPaddingLeft?: boolean;
|
||||
hideTabsHeader?: boolean;
|
||||
scrollToTopOnChange?: boolean;
|
||||
maxHeight?: string;
|
||||
stickyHeader?: boolean;
|
||||
}
|
||||
|
||||
export function Tabs({
|
||||
@ -112,8 +169,19 @@ export function Tabs({
|
||||
getCurrentUrl = () => window.location.pathname,
|
||||
addPaddingLeft,
|
||||
hideTabsHeader,
|
||||
scrollToTopOnChange = false,
|
||||
maxHeight = '100%',
|
||||
stickyHeader = false,
|
||||
}: Props) {
|
||||
const { TabPane } = AntTabs;
|
||||
const tabsContainerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Scroll to top when selectedTab changes if scrollToTopOnChange is enabled
|
||||
useEffect(() => {
|
||||
if (scrollToTopOnChange && tabsContainerRef.current) {
|
||||
tabsContainerRef.current.scrollTo({ top: 0, behavior: 'smooth' });
|
||||
}
|
||||
}, [selectedTab, scrollToTopOnChange]);
|
||||
|
||||
// Create reverse mapping from URLs to tab keys if urlMap is provided
|
||||
const urlToTabMap = React.useMemo(() => {
|
||||
@ -140,23 +208,19 @@ export function Tabs({
|
||||
}
|
||||
}, [getCurrentUrl, onChange, onUrlChange, selectedTab, urlMap, urlToTabMap, defaultTab]);
|
||||
|
||||
function handleTabClick(key: string) {
|
||||
onChange?.(key);
|
||||
const newTab = tabs.find((t) => t.key === key);
|
||||
newTab?.onSelectTab?.();
|
||||
|
||||
// Update URL if urlMap is provided
|
||||
if (urlMap && urlMap[key]) {
|
||||
onUrlChange(urlMap[key]);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
const tabsContent = (
|
||||
<StyledTabs
|
||||
activeKey={selectedTab}
|
||||
onChange={handleTabClick}
|
||||
onChange={(key) => {
|
||||
if (onChange) onChange(key);
|
||||
if (urlMap && onUrlChange && urlMap[key]) {
|
||||
onUrlChange(urlMap[key]);
|
||||
}
|
||||
}}
|
||||
$addPaddingLeft={addPaddingLeft}
|
||||
$hideTabsHeader={!!hideTabsHeader}
|
||||
$scrollable={scrollToTopOnChange}
|
||||
$stickyHeader={stickyHeader}
|
||||
>
|
||||
{tabs.map((tab) => {
|
||||
return (
|
||||
@ -167,4 +231,14 @@ export function Tabs({
|
||||
})}
|
||||
</StyledTabs>
|
||||
);
|
||||
|
||||
if (scrollToTopOnChange) {
|
||||
return (
|
||||
<ScrollableTabsContainer ref={tabsContainerRef} $maxHeight={maxHeight}>
|
||||
{tabsContent}
|
||||
</ScrollableTabsContainer>
|
||||
);
|
||||
}
|
||||
|
||||
return tabsContent;
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Button, PageTitle, Tabs, Tooltip } from '@components';
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { useHistory } from 'react-router';
|
||||
import styled from 'styled-components';
|
||||
|
||||
@ -112,16 +112,6 @@ export const ManageIngestionPage = () => {
|
||||
}
|
||||
}, [loadedAppConfig, loadedPlatformPrivileges, showIngestionTab, showSecretsTab, selectedTab]);
|
||||
|
||||
const onSwitchTab = (newTab: string, options?: { clearQueryParams: boolean }) => {
|
||||
const preserveParams = shouldPreserveParams.current;
|
||||
const matchingTab = Object.values(TabType).find((tab) => tab === newTab);
|
||||
if (!preserveParams && options?.clearQueryParams) {
|
||||
history.push({ search: '' });
|
||||
}
|
||||
setSelectedTab(matchingTab || selectedTab);
|
||||
shouldPreserveParams.current = false;
|
||||
};
|
||||
|
||||
const tabs: Tab[] = [
|
||||
showIngestionTab && {
|
||||
component: (
|
||||
@ -160,9 +150,14 @@ export const ManageIngestionPage = () => {
|
||||
},
|
||||
].filter((tab): tab is Tab => Boolean(tab));
|
||||
|
||||
const onUrlChange = (tabPath: string) => {
|
||||
history.push(tabPath);
|
||||
};
|
||||
const onUrlChange = useCallback(
|
||||
(tabPath: string) => {
|
||||
history.push({ pathname: tabPath, search: '' });
|
||||
},
|
||||
[history],
|
||||
);
|
||||
|
||||
const getCurrentUrl = useCallback(() => window.location.pathname, []);
|
||||
|
||||
const handleCreateSource = () => {
|
||||
setShowCreateSourceModal(true);
|
||||
@ -228,11 +223,11 @@ export const ManageIngestionPage = () => {
|
||||
<Tabs
|
||||
tabs={tabs}
|
||||
selectedTab={selectedTab}
|
||||
onChange={(tab) => onSwitchTab(tab, { clearQueryParams: true })}
|
||||
onChange={(tab) => setSelectedTab(tab as TabType)}
|
||||
urlMap={tabUrlMap}
|
||||
onUrlChange={onUrlChange}
|
||||
defaultTab={TabType.Sources}
|
||||
getCurrentUrl={() => window.location.pathname}
|
||||
getCurrentUrl={getCurrentUrl}
|
||||
/>
|
||||
</PageContentContainer>
|
||||
</PageContainer>
|
||||
|
||||
@ -0,0 +1,14 @@
|
||||
import { Typography } from 'antd';
|
||||
import styled from 'styled-components';
|
||||
|
||||
export const SectionBase = styled.div`
|
||||
padding: 16px 30px 0;
|
||||
`;
|
||||
|
||||
export const SectionHeader = styled(Typography.Title)`
|
||||
&&&& {
|
||||
padding: 0px;
|
||||
margin: 0px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
`;
|
||||
@ -1,111 +1,30 @@
|
||||
import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons';
|
||||
import { LoadingOutlined } from '@ant-design/icons';
|
||||
import { Icon, Modal, Pill } from '@components';
|
||||
import { Button, Typography, message } from 'antd';
|
||||
import { message } from 'antd';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import styled from 'styled-components';
|
||||
import YAML from 'yamljs';
|
||||
|
||||
import { ANTD_GRAY } from '@app/entity/shared/constants';
|
||||
import { StructuredReport } from '@app/ingestV2/executions/components/reporting/StructuredReport';
|
||||
import {
|
||||
EXECUTION_REQUEST_STATUS_LOADING,
|
||||
EXECUTION_REQUEST_STATUS_RUNNING,
|
||||
EXECUTION_REQUEST_STATUS_SUCCEEDED_WITH_WARNINGS,
|
||||
EXECUTION_REQUEST_STATUS_SUCCESS,
|
||||
} from '@app/ingestV2/executions/constants';
|
||||
import { Tab, Tabs } from '@components/components/Tabs/Tabs';
|
||||
|
||||
import { LogsTab } from '@app/ingestV2/executions/components/LogsTab';
|
||||
import { RecipeTab } from '@app/ingestV2/executions/components/RecipeTab';
|
||||
import { SummaryTab } from '@app/ingestV2/executions/components/SummaryTab';
|
||||
import { EXECUTION_REQUEST_STATUS_LOADING, EXECUTION_REQUEST_STATUS_RUNNING } from '@app/ingestV2/executions/constants';
|
||||
import { TabType } from '@app/ingestV2/executions/types';
|
||||
import {
|
||||
getExecutionRequestStatusDisplayColor,
|
||||
getExecutionRequestStatusDisplayText,
|
||||
getExecutionRequestStatusIcon,
|
||||
getExecutionRequestSummaryText,
|
||||
} from '@app/ingestV2/executions/utils';
|
||||
import IngestedAssets from '@app/ingestV2/source/IngestedAssets';
|
||||
import { getIngestionSourceStatus, getStructuredReport } from '@app/ingestV2/source/utils';
|
||||
import { downloadFile } from '@app/search/utils/csvUtils';
|
||||
import { getIngestionSourceStatus } from '@app/ingestV2/source/utils';
|
||||
import { Message } from '@app/shared/Message';
|
||||
|
||||
import { useGetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
||||
import { ExecutionRequestResult } from '@types';
|
||||
|
||||
const Section = styled.div`
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding-bottom: 12px;
|
||||
`;
|
||||
|
||||
const SectionHeader = styled(Typography.Title)`
|
||||
&&&& {
|
||||
padding: 0px;
|
||||
margin: 0px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
`;
|
||||
|
||||
const SectionSubHeader = styled.div`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
`;
|
||||
|
||||
const SubHeaderParagraph = styled(Typography.Paragraph)`
|
||||
margin-bottom: 0px;
|
||||
`;
|
||||
|
||||
const StatusSection = styled.div`
|
||||
border-bottom: 1px solid ${ANTD_GRAY[4]};
|
||||
padding: 16px;
|
||||
padding-left: 30px;
|
||||
padding-right: 30px;
|
||||
`;
|
||||
|
||||
const ResultText = styled.div`
|
||||
margin-bottom: 4px;
|
||||
`;
|
||||
|
||||
const IngestedAssetsSection = styled.div`
|
||||
border-bottom: 1px solid ${ANTD_GRAY[4]};
|
||||
padding: 16px;
|
||||
padding-left: 30px;
|
||||
padding-right: 30px;
|
||||
`;
|
||||
|
||||
const RecipeSection = styled.div`
|
||||
border-top: 1px solid ${ANTD_GRAY[4]};
|
||||
padding-top: 16px;
|
||||
padding-left: 30px;
|
||||
padding-right: 30px;
|
||||
`;
|
||||
|
||||
const LogsSection = styled.div`
|
||||
padding-top: 16px;
|
||||
padding-left: 30px;
|
||||
padding-right: 30px;
|
||||
`;
|
||||
|
||||
const ShowMoreButton = styled(Button)`
|
||||
padding: 0px;
|
||||
`;
|
||||
|
||||
const DetailsContainer = styled.div<DetailsContainerProps>`
|
||||
margin-bottom: -25px;
|
||||
${(props) =>
|
||||
props.areDetailsExpandable &&
|
||||
!props.showExpandedDetails &&
|
||||
`
|
||||
-webkit-mask-image: linear-gradient(to bottom, rgba(0,0,0,1) 50%, rgba(255,0,0,0.5) 60%, rgba(255,0,0,0) 90% );
|
||||
mask-image: linear-gradient(to bottom, rgba(0,0,0,1) 50%, rgba(255,0,0,0.5) 60%, rgba(255,0,0,0) 90%);
|
||||
`}
|
||||
`;
|
||||
|
||||
const modalBodyStyle = {
|
||||
padding: 0,
|
||||
};
|
||||
|
||||
type DetailsContainerProps = {
|
||||
showExpandedDetails: boolean;
|
||||
areDetailsExpandable: boolean;
|
||||
};
|
||||
|
||||
type Props = {
|
||||
urn: string;
|
||||
open: boolean;
|
||||
@ -113,33 +32,14 @@ type Props = {
|
||||
};
|
||||
|
||||
export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
|
||||
const [showExpandedLogs, setShowExpandedLogs] = useState(false);
|
||||
const [showExpandedRecipe, setShowExpandedRecipe] = useState(false);
|
||||
|
||||
const { data, loading, error, refetch } = useGetIngestionExecutionRequestQuery({ variables: { urn } });
|
||||
const output = data?.executionRequest?.result?.report || 'No output found.';
|
||||
|
||||
const downloadLogs = () => {
|
||||
downloadFile(output, `exec-${urn}.log`);
|
||||
};
|
||||
|
||||
const logs = (showExpandedLogs && output) || output?.split('\n')?.slice(0, 5)?.join('\n');
|
||||
const result = data?.executionRequest?.result as Partial<ExecutionRequestResult>;
|
||||
const status = getIngestionSourceStatus(result);
|
||||
|
||||
useEffect(() => {
|
||||
const interval = setInterval(() => {
|
||||
if (status === EXECUTION_REQUEST_STATUS_RUNNING) refetch();
|
||||
}, 2000);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
});
|
||||
const [selectedTab, setSelectedTab] = useState<TabType>(TabType.Summary);
|
||||
|
||||
const ResultIcon = status && getExecutionRequestStatusIcon(status);
|
||||
const resultColor = status ? getExecutionRequestStatusDisplayColor(status) : 'gray';
|
||||
const resultText = status && (
|
||||
<Typography.Text style={{ color: resultColor, fontSize: 14 }}>
|
||||
{ResultIcon && (
|
||||
const titlePill = status && ResultIcon && (
|
||||
<Pill
|
||||
customIconRenderer={() =>
|
||||
status === EXECUTION_REQUEST_STATUS_LOADING || status === EXECUTION_REQUEST_STATUS_RUNNING ? (
|
||||
@ -152,100 +52,64 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
|
||||
color={resultColor}
|
||||
size="md"
|
||||
/>
|
||||
)}
|
||||
</Typography.Text>
|
||||
);
|
||||
|
||||
const structuredReport = result && getStructuredReport(result);
|
||||
useEffect(() => {
|
||||
const interval = setInterval(() => {
|
||||
if (status === EXECUTION_REQUEST_STATUS_RUNNING) refetch();
|
||||
}, 2000);
|
||||
|
||||
const resultSummaryText =
|
||||
(status && <Typography.Text type="secondary">{getExecutionRequestSummaryText(status)}</Typography.Text>) ||
|
||||
undefined;
|
||||
return () => clearInterval(interval);
|
||||
});
|
||||
|
||||
const recipeJson = data?.executionRequest?.input?.arguments?.find((arg) => arg.key === 'recipe')?.value;
|
||||
let recipeYaml: string;
|
||||
try {
|
||||
recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
|
||||
} catch (e) {
|
||||
recipeYaml = '';
|
||||
}
|
||||
const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n')?.slice(0, 5)?.join('\n');
|
||||
|
||||
const areLogsExpandable = output?.split(/\r\n|\r|\n/)?.length > 5;
|
||||
const isRecipeExpandable = recipeYaml?.split(/\r\n|\r|\n/)?.length > 5;
|
||||
const tabs: Tab[] = [
|
||||
{
|
||||
component: (
|
||||
<SummaryTab
|
||||
urn={urn}
|
||||
status={status}
|
||||
result={result}
|
||||
data={data}
|
||||
onTabChange={(tab: TabType) => setSelectedTab(tab)}
|
||||
/>
|
||||
),
|
||||
key: TabType.Summary,
|
||||
name: TabType.Summary,
|
||||
},
|
||||
{
|
||||
component: <LogsTab urn={urn} data={data} />,
|
||||
key: TabType.Logs,
|
||||
name: TabType.Logs,
|
||||
},
|
||||
{
|
||||
component: <RecipeTab data={data} />,
|
||||
key: TabType.Recipe,
|
||||
name: TabType.Recipe,
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<Modal
|
||||
width={800}
|
||||
width="1400px"
|
||||
bodyStyle={modalBodyStyle}
|
||||
title="Execution Run Details"
|
||||
title="Status Details"
|
||||
titlePill={titlePill}
|
||||
open={open}
|
||||
onCancel={onClose}
|
||||
buttons={[{ text: 'Close', variant: 'outline', onClick: onClose }]}
|
||||
>
|
||||
{!data && loading && <Message type="loading" content="Loading execution run details..." />}
|
||||
{error && message.error('Failed to load execution run details :(')}
|
||||
<Section>
|
||||
<StatusSection>
|
||||
<Typography.Title level={5}>Status</Typography.Title>
|
||||
<ResultText>{resultText}</ResultText>
|
||||
<SubHeaderParagraph>{resultSummaryText}</SubHeaderParagraph>
|
||||
{structuredReport ? <StructuredReport report={structuredReport} /> : null}
|
||||
</StatusSection>
|
||||
{(status === EXECUTION_REQUEST_STATUS_SUCCESS ||
|
||||
status === EXECUTION_REQUEST_STATUS_SUCCEEDED_WITH_WARNINGS) && (
|
||||
<IngestedAssetsSection>
|
||||
{data?.executionRequest?.id && (
|
||||
<IngestedAssets executionResult={result} id={data?.executionRequest?.id} />
|
||||
)}
|
||||
</IngestedAssetsSection>
|
||||
)}
|
||||
<LogsSection>
|
||||
<SectionHeader level={5}>Logs</SectionHeader>
|
||||
<SectionSubHeader>
|
||||
<SubHeaderParagraph type="secondary">
|
||||
View logs that were collected during the sync.
|
||||
</SubHeaderParagraph>
|
||||
<Button type="text" onClick={downloadLogs}>
|
||||
<DownloadOutlined />
|
||||
Download
|
||||
</Button>
|
||||
</SectionSubHeader>
|
||||
<DetailsContainer areDetailsExpandable={areLogsExpandable} showExpandedDetails={showExpandedLogs}>
|
||||
<Typography.Paragraph ellipsis>
|
||||
<pre>{`${logs}${!showExpandedLogs && areLogsExpandable ? '...' : ''}`}</pre>
|
||||
</Typography.Paragraph>
|
||||
</DetailsContainer>
|
||||
{areLogsExpandable && (
|
||||
<ShowMoreButton type="link" onClick={() => setShowExpandedLogs(!showExpandedLogs)}>
|
||||
{showExpandedLogs ? 'Hide' : 'Show More'}
|
||||
</ShowMoreButton>
|
||||
)}
|
||||
</LogsSection>
|
||||
{recipe && (
|
||||
<RecipeSection>
|
||||
<SectionHeader level={5}>Recipe</SectionHeader>
|
||||
<SectionSubHeader>
|
||||
<SubHeaderParagraph type="secondary">
|
||||
The configurations used for this sync with the data source.
|
||||
</SubHeaderParagraph>
|
||||
</SectionSubHeader>
|
||||
<DetailsContainer
|
||||
areDetailsExpandable={isRecipeExpandable}
|
||||
showExpandedDetails={showExpandedRecipe}
|
||||
>
|
||||
<Typography.Paragraph ellipsis>
|
||||
<pre>{`${recipe}${!showExpandedRecipe && isRecipeExpandable ? '...' : ''}`}</pre>
|
||||
</Typography.Paragraph>
|
||||
</DetailsContainer>
|
||||
{isRecipeExpandable && (
|
||||
<ShowMoreButton type="link" onClick={() => setShowExpandedRecipe((v) => !v)}>
|
||||
{showExpandedRecipe ? 'Hide' : 'Show More'}
|
||||
</ShowMoreButton>
|
||||
)}
|
||||
</RecipeSection>
|
||||
)}
|
||||
</Section>
|
||||
<Tabs
|
||||
tabs={tabs}
|
||||
selectedTab={selectedTab}
|
||||
onChange={(tab) => setSelectedTab(tab as TabType)}
|
||||
getCurrentUrl={() => window.location.pathname}
|
||||
scrollToTopOnChange
|
||||
maxHeight="80vh"
|
||||
stickyHeader
|
||||
addPaddingLeft
|
||||
/>
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
|
||||
@ -0,0 +1,49 @@
|
||||
import { DownloadOutlined } from '@ant-design/icons';
|
||||
import { Button, Typography } from 'antd';
|
||||
import React from 'react';
|
||||
import styled from 'styled-components';
|
||||
|
||||
import { SectionHeader } from '@app/ingestV2/executions/components/BaseTab';
|
||||
import { downloadFile } from '@app/search/utils/csvUtils';
|
||||
|
||||
import { GetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
||||
|
||||
const SectionSubHeader = styled.div`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
`;
|
||||
|
||||
const SubHeaderParagraph = styled(Typography.Paragraph)`
|
||||
margin-bottom: 0px;
|
||||
`;
|
||||
|
||||
const LogsSection = styled.div`
|
||||
padding-top: 16px;
|
||||
padding-left: 30px;
|
||||
padding-right: 30px;
|
||||
`;
|
||||
|
||||
export const LogsTab = ({ urn, data }: { urn: string; data: GetIngestionExecutionRequestQuery | undefined }) => {
|
||||
const output = data?.executionRequest?.result?.report || 'No output found.';
|
||||
|
||||
const downloadLogs = () => {
|
||||
downloadFile(output, `exec-${urn}.log`);
|
||||
};
|
||||
|
||||
return (
|
||||
<LogsSection>
|
||||
<SectionHeader level={5}>Logs</SectionHeader>
|
||||
<SectionSubHeader>
|
||||
<SubHeaderParagraph type="secondary">View logs that were collected during the sync.</SubHeaderParagraph>
|
||||
<Button type="text" onClick={downloadLogs}>
|
||||
<DownloadOutlined />
|
||||
Download
|
||||
</Button>
|
||||
</SectionSubHeader>
|
||||
<Typography.Paragraph ellipsis>
|
||||
<pre>{output}</pre>
|
||||
</Typography.Paragraph>
|
||||
</LogsSection>
|
||||
);
|
||||
};
|
||||
@ -0,0 +1,47 @@
|
||||
import { Typography } from 'antd';
|
||||
import React from 'react';
|
||||
import styled from 'styled-components';
|
||||
import YAML from 'yamljs';
|
||||
|
||||
import { SectionBase, SectionHeader } from '@app/ingestV2/executions/components/BaseTab';
|
||||
import colors from '@src/alchemy-components/theme/foundations/colors';
|
||||
|
||||
import { GetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
||||
|
||||
const SectionSubHeader = styled.div`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
`;
|
||||
|
||||
const SubHeaderParagraph = styled(Typography.Paragraph)`
|
||||
margin-bottom: 0px;
|
||||
`;
|
||||
|
||||
const RecipeSection = styled(SectionBase)`
|
||||
border-top: 1px solid ${colors.gray[1400]};
|
||||
`;
|
||||
|
||||
export const RecipeTab = ({ data }: { data: GetIngestionExecutionRequestQuery | undefined }) => {
|
||||
const recipeJson = data?.executionRequest?.input?.arguments?.find((arg) => arg.key === 'recipe')?.value;
|
||||
let recipeYaml: string;
|
||||
try {
|
||||
recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
|
||||
} catch (e) {
|
||||
recipeYaml = '';
|
||||
}
|
||||
|
||||
return (
|
||||
<RecipeSection>
|
||||
<SectionHeader level={5}>Recipe</SectionHeader>
|
||||
<SectionSubHeader>
|
||||
<SubHeaderParagraph type="secondary">
|
||||
The configurations used for this sync with the data source.
|
||||
</SubHeaderParagraph>
|
||||
</SectionSubHeader>
|
||||
<Typography.Paragraph ellipsis>
|
||||
<pre>{recipeYaml || 'No recipe found.'}</pre>
|
||||
</Typography.Paragraph>
|
||||
</RecipeSection>
|
||||
);
|
||||
};
|
||||
@ -0,0 +1,187 @@
|
||||
import { DownloadOutlined } from '@ant-design/icons';
|
||||
import { Button, Typography } from 'antd';
|
||||
import React, { useState } from 'react';
|
||||
import styled from 'styled-components';
|
||||
import YAML from 'yamljs';
|
||||
|
||||
import { SectionBase, SectionHeader } from '@app/ingestV2/executions/components/BaseTab';
|
||||
import { StructuredReport, hasSomethingToShow } from '@app/ingestV2/executions/components/reporting/StructuredReport';
|
||||
import { EXECUTION_REQUEST_STATUS_SUCCESS } from '@app/ingestV2/executions/constants';
|
||||
import { TabType } from '@app/ingestV2/executions/types';
|
||||
import { getExecutionRequestSummaryText } from '@app/ingestV2/executions/utils';
|
||||
import IngestedAssets from '@app/ingestV2/source/IngestedAssets';
|
||||
import { getStructuredReport } from '@app/ingestV2/source/utils';
|
||||
import { downloadFile } from '@app/search/utils/csvUtils';
|
||||
import colors from '@src/alchemy-components/theme/foundations/colors';
|
||||
|
||||
import { GetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
||||
import { ExecutionRequestResult } from '@types';
|
||||
|
||||
const Section = styled.div`
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding-bottom: 12px;
|
||||
`;
|
||||
|
||||
const SectionSubHeader = styled.div`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
`;
|
||||
|
||||
const ButtonGroup = styled.div`
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
`;
|
||||
|
||||
const SubHeaderParagraph = styled(Typography.Paragraph)`
|
||||
margin-bottom: 0px;
|
||||
`;
|
||||
|
||||
const StatusSection = styled.div`
|
||||
border-bottom: 1px solid ${colors.gray[1400]};
|
||||
padding: 16px;
|
||||
padding-left: 30px;
|
||||
padding-right: 30px;
|
||||
`;
|
||||
|
||||
const IngestedAssetsSection = styled.div<{ isFirstSection?: boolean }>`
|
||||
border-bottom: 1px solid ${colors.gray[1400]};
|
||||
${({ isFirstSection }) => !isFirstSection && `border-top: 1px solid ${colors.gray[1400]};`}
|
||||
padding: 16px;
|
||||
padding-left: 30px;
|
||||
padding-right: 30px;
|
||||
`;
|
||||
|
||||
const RecipeSection = styled(SectionBase)``;
|
||||
|
||||
const LogsSection = styled(SectionBase)``;
|
||||
|
||||
const ShowMoreButton = styled(Button)`
|
||||
padding: 0px;
|
||||
`;
|
||||
|
||||
const DetailsContainer = styled.div`
|
||||
margin-top: 12px;
|
||||
|
||||
pre {
|
||||
background-color: ${colors.gray[1500]};
|
||||
border: 1px solid ${colors.gray[1400]};
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
margin: 0;
|
||||
color: ${colors.gray[1700]};
|
||||
}
|
||||
`;
|
||||
|
||||
export const SummaryTab = ({
|
||||
urn,
|
||||
status,
|
||||
result,
|
||||
data,
|
||||
onTabChange,
|
||||
}: {
|
||||
urn: string;
|
||||
status: string | undefined;
|
||||
result: Partial<ExecutionRequestResult>;
|
||||
data: GetIngestionExecutionRequestQuery | undefined;
|
||||
onTabChange: (tab: TabType) => void;
|
||||
}) => {
|
||||
const [showExpandedLogs] = useState(false);
|
||||
const [showExpandedRecipe] = useState(false);
|
||||
|
||||
const output = data?.executionRequest?.result?.report || 'No output found.';
|
||||
|
||||
const downloadLogs = () => {
|
||||
downloadFile(output, `exec-${urn}.log`);
|
||||
};
|
||||
|
||||
const logs = (showExpandedLogs && output) || output?.split('\n')?.slice(0, 14)?.join('\n');
|
||||
const structuredReport = result && getStructuredReport(result);
|
||||
const resultSummaryText =
|
||||
(status && status !== EXECUTION_REQUEST_STATUS_SUCCESS && (
|
||||
<Typography.Text type="secondary">{getExecutionRequestSummaryText(status)}</Typography.Text>
|
||||
)) ||
|
||||
undefined;
|
||||
const recipeJson = data?.executionRequest?.input?.arguments?.find((arg) => arg.key === 'recipe')?.value;
|
||||
let recipeYaml: string;
|
||||
try {
|
||||
recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
|
||||
} catch (e) {
|
||||
recipeYaml = '';
|
||||
}
|
||||
const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n')?.slice(0, 14)?.join('\n');
|
||||
const areLogsExpandable = output?.split(/\r\n|\r|\n/)?.length > 14;
|
||||
const isRecipeExpandable = recipeYaml?.split(/\r\n|\r|\n/)?.length > 14;
|
||||
|
||||
const downloadRecipe = () => {
|
||||
downloadFile(recipeYaml, `recipe-${urn}.yaml`);
|
||||
};
|
||||
|
||||
return (
|
||||
<Section>
|
||||
{(resultSummaryText || (structuredReport && hasSomethingToShow(structuredReport))) && (
|
||||
<StatusSection>
|
||||
<SubHeaderParagraph>{resultSummaryText}</SubHeaderParagraph>
|
||||
{structuredReport && structuredReport ? <StructuredReport report={structuredReport} /> : null}
|
||||
</StatusSection>
|
||||
)}
|
||||
<IngestedAssetsSection
|
||||
isFirstSection={!resultSummaryText && !(structuredReport && hasSomethingToShow(structuredReport))}
|
||||
>
|
||||
{data?.executionRequest?.id && (
|
||||
<IngestedAssets executionResult={result} id={data?.executionRequest?.id} />
|
||||
)}
|
||||
</IngestedAssetsSection>
|
||||
<LogsSection>
|
||||
<SectionHeader level={5}>Logs</SectionHeader>
|
||||
<SectionSubHeader>
|
||||
<SubHeaderParagraph type="secondary">
|
||||
View logs that were collected during the sync.
|
||||
</SubHeaderParagraph>
|
||||
<ButtonGroup>
|
||||
{areLogsExpandable && (
|
||||
<ShowMoreButton type="text" onClick={() => onTabChange(TabType.Logs)}>
|
||||
View More
|
||||
</ShowMoreButton>
|
||||
)}
|
||||
<Button type="text" onClick={downloadLogs}>
|
||||
<DownloadOutlined />
|
||||
</Button>
|
||||
</ButtonGroup>
|
||||
</SectionSubHeader>
|
||||
<DetailsContainer>
|
||||
<Typography.Paragraph ellipsis>
|
||||
<pre>{`${logs}${areLogsExpandable ? '...' : ''}`}</pre>
|
||||
</Typography.Paragraph>
|
||||
</DetailsContainer>
|
||||
</LogsSection>
|
||||
{recipe && (
|
||||
<RecipeSection>
|
||||
<SectionHeader level={5}>Recipe</SectionHeader>
|
||||
<SectionSubHeader>
|
||||
<SubHeaderParagraph type="secondary">
|
||||
The configurations used for this sync with the data source.
|
||||
</SubHeaderParagraph>
|
||||
<ButtonGroup>
|
||||
{isRecipeExpandable && (
|
||||
<ShowMoreButton type="text" onClick={() => onTabChange(TabType.Recipe)}>
|
||||
View More
|
||||
</ShowMoreButton>
|
||||
)}
|
||||
<Button type="text" onClick={downloadRecipe}>
|
||||
<DownloadOutlined />
|
||||
</Button>
|
||||
</ButtonGroup>
|
||||
</SectionSubHeader>
|
||||
<DetailsContainer>
|
||||
<Typography.Paragraph ellipsis>
|
||||
<pre>{`${recipe}${!showExpandedRecipe && isRecipeExpandable ? '...' : ''}`}</pre>
|
||||
</Typography.Paragraph>
|
||||
</DetailsContainer>
|
||||
</RecipeSection>
|
||||
)}
|
||||
</Section>
|
||||
);
|
||||
};
|
||||
@ -23,6 +23,13 @@ interface Props {
|
||||
report: StructuredReportType;
|
||||
}
|
||||
|
||||
export function hasSomethingToShow(report: StructuredReportType): boolean {
|
||||
const warnings = report.items.filter((item) => item.level === StructuredReportItemLevel.WARN);
|
||||
const errors = report.items.filter((item) => item.level === StructuredReportItemLevel.ERROR);
|
||||
const infos = report.items.filter((item) => item.level === StructuredReportItemLevel.INFO);
|
||||
return warnings.length > 0 || errors.length > 0 || infos.length > 0;
|
||||
}
|
||||
|
||||
export function StructuredReport({ report }: Props) {
|
||||
if (!report.items.length) {
|
||||
return null;
|
||||
@ -34,7 +41,12 @@ export function StructuredReport({ report }: Props) {
|
||||
return (
|
||||
<Container>
|
||||
{errors.length ? (
|
||||
<StructuredReportItemList items={errors} color={ERROR_COLOR} icon={CloseCircleOutlined} />
|
||||
<StructuredReportItemList
|
||||
items={errors}
|
||||
color={ERROR_COLOR}
|
||||
icon={CloseCircleOutlined}
|
||||
defaultActiveKey="0"
|
||||
/>
|
||||
) : null}
|
||||
{warnings.length ? (
|
||||
<StructuredReportItemList items={warnings} color={WARNING_COLOR} icon={ExclamationCircleOutlined} />
|
||||
|
||||
@ -55,12 +55,13 @@ interface Props {
|
||||
item: StructuredReportLogEntry;
|
||||
color: string;
|
||||
icon?: React.ComponentType<any>;
|
||||
defaultActiveKey?: string;
|
||||
}
|
||||
|
||||
export function StructuredReportItem({ item, color, icon }: Props) {
|
||||
export function StructuredReportItem({ item, color, icon, defaultActiveKey }: Props) {
|
||||
const Icon = icon;
|
||||
return (
|
||||
<StyledCollapse color={color}>
|
||||
<StyledCollapse color={color} defaultActiveKey={defaultActiveKey}>
|
||||
<Collapse.Panel
|
||||
header={
|
||||
<Item>
|
||||
|
||||
@ -16,9 +16,10 @@ interface Props {
|
||||
color: string;
|
||||
icon?: React.ComponentType<any>;
|
||||
pageSize?: number;
|
||||
defaultActiveKey?: string;
|
||||
}
|
||||
|
||||
export function StructuredReportItemList({ items, color, icon, pageSize = 3 }: Props) {
|
||||
export function StructuredReportItemList({ items, color, icon, pageSize = 3, defaultActiveKey }: Props) {
|
||||
const [visibleCount, setVisibleCount] = useState(pageSize);
|
||||
const visibleItems = items.slice(0, visibleCount);
|
||||
const totalCount = items.length;
|
||||
@ -31,6 +32,7 @@ export function StructuredReportItemList({ items, color, icon, pageSize = 3 }: P
|
||||
item={item}
|
||||
color={color}
|
||||
icon={icon}
|
||||
defaultActiveKey={defaultActiveKey}
|
||||
key={`${item.message}-${item.context}`}
|
||||
/>
|
||||
))}
|
||||
|
||||
@ -20,3 +20,9 @@ export interface ExecutionCancelInfo {
|
||||
executionUrn: string;
|
||||
sourceUrn: string;
|
||||
}
|
||||
|
||||
export const enum TabType {
|
||||
Summary = 'Summary',
|
||||
Logs = 'Logs',
|
||||
Recipe = 'Recipe',
|
||||
}
|
||||
|
||||
@ -1,59 +1,129 @@
|
||||
import { Button, Typography } from 'antd';
|
||||
import React, { useState } from 'react';
|
||||
import { Button } from 'antd';
|
||||
import React, { useMemo, useState } from 'react';
|
||||
import styled from 'styled-components';
|
||||
|
||||
import { EmbeddedListSearchModal } from '@app/entity/shared/components/styled/search/EmbeddedListSearchModal';
|
||||
import { ANTD_GRAY } from '@app/entity/shared/constants';
|
||||
import {
|
||||
extractEntityTypeCountsFromFacets,
|
||||
getEntitiesIngestedByType,
|
||||
getIngestionContents,
|
||||
getOtherIngestionContents,
|
||||
getTotalEntitiesIngested,
|
||||
} from '@app/ingestV2/source/utils';
|
||||
import { UnionType } from '@app/search/utils/constants';
|
||||
import { Message } from '@app/shared/Message';
|
||||
import { formatNumber } from '@app/shared/formatNumber';
|
||||
import { capitalizeFirstLetterOnly } from '@app/shared/textUtil';
|
||||
import { useEntityRegistry } from '@app/useEntityRegistry';
|
||||
import { Heading, Pill, Text } from '@src/alchemy-components';
|
||||
import colors from '@src/alchemy-components/theme/foundations/colors';
|
||||
import { ExecutionRequestResult, Maybe } from '@src/types.generated';
|
||||
|
||||
import { useGetSearchResultsForMultipleQuery } from '@graphql/search.generated';
|
||||
|
||||
const HeaderContainer = styled.div`
|
||||
// Base flex container with common spacing
|
||||
const FlexContainer = styled.div`
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
`;
|
||||
|
||||
const TitleContainer = styled.div``;
|
||||
|
||||
const TotalContainer = styled.div`
|
||||
// Base card styling
|
||||
const BaseCard = styled.div`
|
||||
display: flex;
|
||||
padding: 12px;
|
||||
background-color: white;
|
||||
border: 1px solid ${colors.gray[1400]};
|
||||
border-radius: 12px;
|
||||
box-shadow: 0px 4px 8px 0px rgba(33, 23, 95, 0.04);
|
||||
min-height: 60px;
|
||||
`;
|
||||
|
||||
const MainContainer = styled(FlexContainer)`
|
||||
align-items: stretch;
|
||||
margin-top: 16px;
|
||||
`;
|
||||
|
||||
const CardContainer = styled(BaseCard)`
|
||||
flex-direction: column;
|
||||
justify-content: right;
|
||||
align-items: end;
|
||||
justify-content: center;
|
||||
align-items: flex-start;
|
||||
flex: 1 0 0;
|
||||
min-height: 80px;
|
||||
max-height: 80px;
|
||||
`;
|
||||
|
||||
const TotalText = styled(Typography.Text)`
|
||||
font-size: 16px;
|
||||
color: ${ANTD_GRAY[8]};
|
||||
`;
|
||||
|
||||
const EntityCountsContainer = styled.div`
|
||||
display: flex;
|
||||
justify-content: left;
|
||||
const TotalContainer = styled(BaseCard)`
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
max-width: 100%;
|
||||
flex-wrap: wrap;
|
||||
justify-content: space-between;
|
||||
flex: 1 0 0;
|
||||
min-height: 80px;
|
||||
max-height: 80px;
|
||||
`;
|
||||
|
||||
const EntityCount = styled.div`
|
||||
margin-right: 40px;
|
||||
const TotalInfo = styled.div`
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
`;
|
||||
|
||||
const ViewAllButton = styled(Button)`
|
||||
padding: 0px;
|
||||
margin-top: 4px;
|
||||
const TypesSection = styled.div`
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
width: 100%;
|
||||
`;
|
||||
|
||||
const IngestionBoxesContainer = styled(FlexContainer)`
|
||||
width: 100%;
|
||||
`;
|
||||
|
||||
const EntityCountsContainer = styled(FlexContainer)`
|
||||
flex: 1;
|
||||
width: 100%;
|
||||
align-items: stretch;
|
||||
justify-content: flex-start;
|
||||
flex-wrap: wrap;
|
||||
`;
|
||||
|
||||
const TypesHeaderContainer = styled.div`
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
margin-top: 16px;
|
||||
position: relative;
|
||||
`;
|
||||
|
||||
const TypesHeader = styled(Text)`
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: calc(50% + 33px);
|
||||
margin-bottom: 0;
|
||||
z-index: 1;
|
||||
`;
|
||||
|
||||
const VerticalDivider = styled.div`
|
||||
width: 2px;
|
||||
background-color: ${colors.gray[1400]};
|
||||
height: 80px;
|
||||
align-self: center;
|
||||
`;
|
||||
|
||||
const IngestionContentsContainer = styled.div`
|
||||
margin-top: 20px;
|
||||
`;
|
||||
|
||||
const IngestionBoxTopRow = styled.div`
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
justify-content: flex-start;
|
||||
width: 100%;
|
||||
`;
|
||||
|
||||
const IngestionRowCount = styled(Text)`
|
||||
margin-right: 10px;
|
||||
`;
|
||||
|
||||
type Props = {
|
||||
@ -64,6 +134,42 @@ type Props = {
|
||||
const ENTITY_FACET_NAME = 'entity';
|
||||
const TYPE_NAMES_FACET_NAME = 'typeNames';
|
||||
|
||||
type IngestionContentItem = {
|
||||
title?: string;
|
||||
type?: string;
|
||||
count: number;
|
||||
percent: string;
|
||||
};
|
||||
|
||||
type RenderIngestionContentsProps = {
|
||||
items: IngestionContentItem[];
|
||||
getKey: (item: IngestionContentItem) => string;
|
||||
getLabel: (item: IngestionContentItem) => string;
|
||||
};
|
||||
|
||||
const IngestionContents: React.FC<RenderIngestionContentsProps> = ({ items, getKey, getLabel }) => (
|
||||
<IngestionBoxesContainer>
|
||||
{items.map((item) => (
|
||||
<CardContainer key={getKey(item)}>
|
||||
<IngestionBoxTopRow>
|
||||
<IngestionRowCount size="xl" weight="bold" color="gray" colorLevel={800}>
|
||||
{formatNumber(item.count)}
|
||||
</IngestionRowCount>
|
||||
<Pill
|
||||
size="sm"
|
||||
variant="filled"
|
||||
color="gray"
|
||||
label={item.count === 0 ? 'Missing' : `${item.percent} of Total`}
|
||||
/>
|
||||
</IngestionBoxTopRow>
|
||||
<Text size="md" color="gray" colorLevel={600}>
|
||||
{getLabel(item)}
|
||||
</Text>
|
||||
</CardContainer>
|
||||
))}
|
||||
</IngestionBoxesContainer>
|
||||
);
|
||||
|
||||
export default function IngestedAssets({ id, executionResult }: Props) {
|
||||
const entityRegistry = useEntityRegistry();
|
||||
|
||||
@ -113,44 +219,99 @@ export default function IngestedAssets({ id, executionResult }: Props) {
|
||||
// The total number of assets ingested
|
||||
const total = totalEntitiesIngested ?? data?.searchAcrossEntities?.total ?? 0;
|
||||
|
||||
const ingestionContents = useMemo(
|
||||
() => executionResult && getIngestionContents(executionResult),
|
||||
[executionResult],
|
||||
);
|
||||
const otherIngestionContents = useMemo(
|
||||
() => executionResult && getOtherIngestionContents(executionResult),
|
||||
[executionResult],
|
||||
);
|
||||
|
||||
return (
|
||||
<>
|
||||
{error && <Message type="error" content="" />}
|
||||
<HeaderContainer>
|
||||
<TitleContainer>
|
||||
<Typography.Title level={5}>Ingested Assets</Typography.Title>
|
||||
{(loading && <Typography.Text type="secondary">Loading...</Typography.Text>) || (
|
||||
<Heading type="h4" size="lg" weight="bold">
|
||||
Assets
|
||||
</Heading>
|
||||
{loading && (
|
||||
<Text color="gray" colorLevel={600}>
|
||||
Loading...
|
||||
</Text>
|
||||
)}
|
||||
{!loading && total === 0 && <Text>No assets were ingested.</Text>}
|
||||
{!loading && total > 0 && (
|
||||
<>
|
||||
{(total > 0 && (
|
||||
<Typography.Paragraph type="secondary">
|
||||
The following asset types were ingested during this run.
|
||||
</Typography.Paragraph>
|
||||
)) || <Typography.Text>No assets were ingested.</Typography.Text>}
|
||||
</>
|
||||
)}
|
||||
</TitleContainer>
|
||||
{!loading && (
|
||||
<TypesHeaderContainer>
|
||||
<Text color="gray" colorLevel={600}>
|
||||
Types and counts for this ingestion run.
|
||||
</Text>
|
||||
<TypesHeader size="sm" color="gray" colorLevel={600} weight="bold">
|
||||
Types
|
||||
</TypesHeader>
|
||||
</TypesHeaderContainer>
|
||||
<MainContainer>
|
||||
<TotalContainer>
|
||||
<Typography.Text type="secondary">Total</Typography.Text>
|
||||
<TotalText style={{ fontSize: 16, color: ANTD_GRAY[8] }}>
|
||||
<b>{formatNumber(total)}</b> assets
|
||||
</TotalText>
|
||||
<TotalInfo>
|
||||
<Text size="xl" weight="bold" color="gray" colorLevel={800}>
|
||||
{formatNumber(total)}
|
||||
</Text>
|
||||
<Text size="md" color="gray" colorLevel={600} style={{ marginTop: 2 }}>
|
||||
Total Assets Ingested
|
||||
</Text>
|
||||
</TotalInfo>
|
||||
<Button type="link" onClick={() => setShowAssetSearch(true)}>
|
||||
View All
|
||||
</Button>
|
||||
</TotalContainer>
|
||||
)}
|
||||
</HeaderContainer>
|
||||
<VerticalDivider />
|
||||
<TypesSection>
|
||||
<EntityCountsContainer>
|
||||
{countsByEntityType.map((entityCount) => (
|
||||
<EntityCount>
|
||||
<Typography.Text style={{ paddingLeft: 2, fontSize: 18, color: ANTD_GRAY[8] }}>
|
||||
<b>{formatNumber(entityCount.count)}</b>
|
||||
</Typography.Text>
|
||||
<Typography.Text type="secondary">{entityCount.displayName}</Typography.Text>
|
||||
</EntityCount>
|
||||
<CardContainer key={entityCount.displayName}>
|
||||
<Text size="xl" weight="bold" color="gray" colorLevel={800}>
|
||||
{formatNumber(entityCount.count)}
|
||||
</Text>
|
||||
<Text size="md" color="gray" colorLevel={600}>
|
||||
{capitalizeFirstLetterOnly(entityCount.displayName)}
|
||||
</Text>
|
||||
</CardContainer>
|
||||
))}
|
||||
</EntityCountsContainer>
|
||||
<ViewAllButton type="link" onClick={() => setShowAssetSearch(true)}>
|
||||
View All
|
||||
</ViewAllButton>
|
||||
</TypesSection>
|
||||
</MainContainer>
|
||||
{ingestionContents && (
|
||||
<IngestionContentsContainer>
|
||||
<Heading type="h5" size="lg" weight="bold">
|
||||
Coverage
|
||||
</Heading>
|
||||
<Text color="gray" colorLevel={600}>
|
||||
Additional metadata collected during this ingestion run.
|
||||
</Text>
|
||||
<Text weight="semiBold" size="md">
|
||||
Lineage
|
||||
</Text>
|
||||
<IngestionContents
|
||||
items={ingestionContents}
|
||||
getKey={(item) => item.title || ''}
|
||||
getLabel={(item) => item.title || ''}
|
||||
/>
|
||||
{otherIngestionContents && (
|
||||
<>
|
||||
<Text weight="semiBold" size="md">
|
||||
Statistics
|
||||
</Text>
|
||||
<IngestionContents
|
||||
items={otherIngestionContents}
|
||||
getKey={(item) => item.type || ''}
|
||||
getLabel={(item) => item.type || ''}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</IngestionContentsContainer>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{showAssetSearch && (
|
||||
<EmbeddedListSearchModal
|
||||
title="View Ingested Assets"
|
||||
|
||||
@ -15,6 +15,8 @@ import {
|
||||
capitalizeMonthsAndDays,
|
||||
formatTimezone,
|
||||
getEntitiesIngestedByType,
|
||||
getIngestionContents,
|
||||
getOtherIngestionContents,
|
||||
getSortInput,
|
||||
getSourceStatus,
|
||||
getTotalEntitiesIngested,
|
||||
@ -412,6 +414,430 @@ describe('getSourceStatus', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIngestionContents', () => {
|
||||
test('returns null when structured report is not available', () => {
|
||||
const result = getIngestionContents({} as Partial<ExecutionRequestResult>);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null when aspects_by_subtypes is empty', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('processes dataset subtypes with lineage information correctly', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
container: {
|
||||
containerProperties: 156,
|
||||
container: 117,
|
||||
},
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 10,
|
||||
upstreamLineage: 5,
|
||||
datasetProfile: 10,
|
||||
},
|
||||
View: {
|
||||
status: 20,
|
||||
upstreamLineage: 10,
|
||||
datasetProfile: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
title: 'Table',
|
||||
count: 10,
|
||||
percent: '50%',
|
||||
},
|
||||
{
|
||||
title: 'View',
|
||||
count: 20,
|
||||
percent: '50%',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('filters out subtypes with 0% lineage', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 10,
|
||||
upstreamLineage: 0,
|
||||
},
|
||||
View: {
|
||||
status: 20,
|
||||
upstreamLineage: 5,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
title: 'View',
|
||||
count: 20,
|
||||
percent: '25%',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('filters out subtypes with status count of 0', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 0,
|
||||
upstreamLineage: 5,
|
||||
},
|
||||
View: {
|
||||
status: 20,
|
||||
upstreamLineage: 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
title: 'View',
|
||||
count: 20,
|
||||
percent: '50%',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('handles missing upstreamLineage property', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 10,
|
||||
// upstreamLineage is missing
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('handles missing status property', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
// status is missing
|
||||
upstreamLineage: 5,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('calculates percentage correctly and rounds to nearest integer', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 7,
|
||||
upstreamLineage: 2, // 2/7 = 28.57...% rounds to 29%
|
||||
},
|
||||
View: {
|
||||
status: 3,
|
||||
upstreamLineage: 1, // 1/3 = 33.33...% rounds to 33%
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
title: 'Table',
|
||||
count: 7,
|
||||
percent: '29%',
|
||||
},
|
||||
{
|
||||
title: 'View',
|
||||
count: 3,
|
||||
percent: '33%',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getOtherIngestionContents', () => {
|
||||
test('returns null when structured report is not available', () => {
|
||||
const result = getOtherIngestionContents({} as Partial<ExecutionRequestResult>);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null when aspects_by_subtypes is empty', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('processes multiple dataset subtypes and aggregates profiling and usage entries', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 10,
|
||||
datasetProfile: 5,
|
||||
datasetUsageStatistics: 3,
|
||||
},
|
||||
View: {
|
||||
status: 20,
|
||||
datasetProfile: 8,
|
||||
datasetUsageStatistics: 12,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
type: 'Profiling',
|
||||
count: 13, // 5 + 8
|
||||
percent: '43%', // (13 / 30) * 100 = 43.33...% rounds to 43%
|
||||
},
|
||||
{
|
||||
type: 'Usage',
|
||||
count: 15, // 3 + 12
|
||||
percent: '50%', // (15 / 30) * 100 = 50%
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('filters out subtypes with zero profiling and usage counts', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 10,
|
||||
datasetProfile: 0,
|
||||
datasetUsageStatistics: 0,
|
||||
},
|
||||
View: {
|
||||
status: 20,
|
||||
datasetProfile: 8,
|
||||
datasetUsageStatistics: 12,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
type: 'Profiling',
|
||||
count: 8,
|
||||
percent: '27%', // (8 / 30) * 100 = 26.66...% rounds to 27%
|
||||
},
|
||||
{
|
||||
type: 'Usage',
|
||||
count: 12,
|
||||
percent: '40%', // (12 / 30) * 100 = 40%
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('filters out subtypes with zero status count', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 0,
|
||||
datasetProfile: 5,
|
||||
datasetUsageStatistics: 3,
|
||||
},
|
||||
View: {
|
||||
status: 20,
|
||||
datasetProfile: 8,
|
||||
datasetUsageStatistics: 12,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
type: 'Profiling',
|
||||
count: 8,
|
||||
percent: '40%', // (8 / 20) * 100 = 40%
|
||||
},
|
||||
{
|
||||
type: 'Usage',
|
||||
count: 12,
|
||||
percent: '60%', // (12 / 20) * 100 = 60%
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('handles missing datasetProfile and datasetUsageStatistics properties', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 10,
|
||||
// datasetProfile and datasetUsageStatistics are missing
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
count: 0,
|
||||
percent: '0%',
|
||||
type: 'Usage',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('ignores non-dataset entity types', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
container: {
|
||||
Container: {
|
||||
status: 10,
|
||||
datasetProfile: 5,
|
||||
datasetUsageStatistics: 3,
|
||||
},
|
||||
},
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 20,
|
||||
datasetProfile: 8,
|
||||
datasetUsageStatistics: 12,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
type: 'Profiling',
|
||||
count: 8,
|
||||
percent: '40%', // (8 / 20) * 100 = 40%
|
||||
},
|
||||
{
|
||||
type: 'Usage',
|
||||
count: 12,
|
||||
percent: '60%', // (12 / 20) * 100 = 60%
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('calculates percentage correctly and rounds to nearest integer', () => {
|
||||
const structuredReport = {
|
||||
source: {
|
||||
report: {
|
||||
aspects_by_subtypes: {
|
||||
dataset: {
|
||||
Table: {
|
||||
status: 7,
|
||||
datasetProfile: 2, // 2/7 = 28.57...% rounds to 29%
|
||||
datasetUsageStatistics: 1, // 1/7 = 14.28...% rounds to 14%
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||
expect(result).toEqual([
|
||||
{
|
||||
type: 'Profiling',
|
||||
count: 2,
|
||||
percent: '29%',
|
||||
},
|
||||
{
|
||||
type: 'Usage',
|
||||
count: 1,
|
||||
percent: '14%',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildOwnerEntities', () => {
|
||||
const entityUrn = 'urn:li:entity:123';
|
||||
const ownerUrn = 'urn:li:user:123';
|
||||
|
||||
@ -303,6 +303,118 @@ export const getTotalEntitiesIngested = (result: Partial<ExecutionRequestResult>
|
||||
return entityTypeCounts.reduce((total, entityType) => total + entityType.count, 0);
|
||||
};
|
||||
|
||||
export const getOtherIngestionContents = (executionResult: Partial<ExecutionRequestResult>) => {
|
||||
const structuredReportObject = extractStructuredReportPOJO(executionResult);
|
||||
if (!structuredReportObject) {
|
||||
return null;
|
||||
}
|
||||
const aspectsBySubtypes = structuredReportObject?.source?.report?.aspects_by_subtypes;
|
||||
|
||||
if (!aspectsBySubtypes || Object.keys(aspectsBySubtypes).length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let totalStatusCount = 0;
|
||||
let totalDatasetProfileCount = 0;
|
||||
let totalDatasetUsageStatisticsCount = 0;
|
||||
|
||||
Object.entries(aspectsBySubtypes).forEach(([entityType, subtypes]) => {
|
||||
if (entityType !== 'dataset') {
|
||||
// temporary for now - we have not decided on the design for non dataset entity types
|
||||
return;
|
||||
}
|
||||
Object.entries(subtypes as Record<string, any>).forEach(([_, aspects]) => {
|
||||
const statusCount = (aspects as any)?.status || 0;
|
||||
if (statusCount === 0) {
|
||||
return;
|
||||
}
|
||||
const dataSetProfileCount = (aspects as any)?.datasetProfile || 0;
|
||||
const dataSetUsageStatisticsCount = (aspects as any)?.datasetUsageStatistics || 0;
|
||||
|
||||
totalStatusCount += statusCount;
|
||||
totalDatasetProfileCount += dataSetProfileCount;
|
||||
totalDatasetUsageStatisticsCount += dataSetUsageStatisticsCount;
|
||||
});
|
||||
});
|
||||
|
||||
if (totalStatusCount === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const result: Array<{ type: string; count: number; percent: string }> = [];
|
||||
|
||||
if (totalDatasetProfileCount > 0) {
|
||||
const datasetProfilePercent = `${((totalDatasetProfileCount / totalStatusCount) * 100).toFixed(0)}%`;
|
||||
result.push({
|
||||
type: 'Profiling',
|
||||
count: totalDatasetProfileCount,
|
||||
percent: datasetProfilePercent,
|
||||
});
|
||||
}
|
||||
|
||||
if (totalDatasetUsageStatisticsCount > 0) {
|
||||
const datasetUsageStatisticsPercent = `${((totalDatasetUsageStatisticsCount / totalStatusCount) * 100).toFixed(0)}%`;
|
||||
result.push({
|
||||
type: 'Usage',
|
||||
count: totalDatasetUsageStatisticsCount,
|
||||
percent: datasetUsageStatisticsPercent,
|
||||
});
|
||||
} else {
|
||||
result.push({
|
||||
type: 'Usage',
|
||||
count: 0,
|
||||
percent: '0%',
|
||||
});
|
||||
}
|
||||
|
||||
if (result.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
export const getIngestionContents = (executionResult: Partial<ExecutionRequestResult>) => {
|
||||
const structuredReportObject = extractStructuredReportPOJO(executionResult);
|
||||
if (!structuredReportObject) {
|
||||
return null;
|
||||
}
|
||||
const aspectsBySubtypes = structuredReportObject.source.report.aspects_by_subtypes;
|
||||
|
||||
if (!aspectsBySubtypes || Object.keys(aspectsBySubtypes).length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const result: Array<{ title: string; count: number; percent: string }> = [];
|
||||
Object.entries(aspectsBySubtypes).forEach(([entityType, subtypes]) => {
|
||||
if (entityType !== 'dataset') {
|
||||
// temporary for now - we have not decided on the design for non dataset entity types
|
||||
return;
|
||||
}
|
||||
Object.entries(subtypes as Record<string, any>).forEach(([subtype, aspects]) => {
|
||||
const statusCount = (aspects as any)?.status || 0;
|
||||
const upstreamLineage = (aspects as any)?.upstreamLineage || 0;
|
||||
if (statusCount === 0) {
|
||||
return;
|
||||
}
|
||||
const percent = `${((upstreamLineage / statusCount) * 100).toFixed(0)}%`;
|
||||
if (percent === '0%') {
|
||||
return;
|
||||
}
|
||||
result.push({
|
||||
title: subtype,
|
||||
count: statusCount,
|
||||
percent,
|
||||
});
|
||||
});
|
||||
});
|
||||
if (result.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
export const getIngestionSourceStatus = (result?: Partial<ExecutionRequestResult> | null) => {
|
||||
if (!result) {
|
||||
return undefined;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user