mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-05 15:13:21 +00:00
feat(ingest): change asset list (#14073)
This commit is contained in:
parent
b8aac8b4e3
commit
e539e33e10
@ -16,7 +16,14 @@
|
|||||||
"Bash(ruff:*)",
|
"Bash(ruff:*)",
|
||||||
"Bash(python -m mypy:*)",
|
"Bash(python -m mypy:*)",
|
||||||
"Bash(python -m ruff:*)",
|
"Bash(python -m ruff:*)",
|
||||||
"Bash(python -m pytest:*)"
|
"Bash(python -m pytest:*)",
|
||||||
|
"Bash(yarn format:*)",
|
||||||
|
"Bash(yarn lint:*)",
|
||||||
|
"Bash(yarn type-check:*)",
|
||||||
|
"Bash(yarn test:*)",
|
||||||
|
"Bash(yarn generate:*)",
|
||||||
|
"Bash(./gradlew :datahub-web-react:yarnLintFix)",
|
||||||
|
"Bash(./gradlew :datahub-web-react:yarnLint)"
|
||||||
],
|
],
|
||||||
"deny": []
|
"deny": []
|
||||||
}
|
}
|
||||||
|
|||||||
@ -174,8 +174,19 @@ yarn generate
|
|||||||
# Run formatting
|
# Run formatting
|
||||||
yarn format
|
yarn format
|
||||||
|
|
||||||
# Run linting
|
# Run linting on all files
|
||||||
yarn lint
|
../gradlew yarnLint
|
||||||
|
|
||||||
|
# Run lint-fix on a single file
|
||||||
|
../gradlew -x yarnInstall -x yarnGenerate yarnLintFix -Pfile=src/path/to/file.tsx
|
||||||
|
|
||||||
|
# Run linting on a single file
|
||||||
|
# This does not run full type-check when we run for a single file
|
||||||
|
# that should be run at the end of all changes before commit
|
||||||
|
../gradlew -x yarnInstall -x yarnGenerate yarnLint -Pfile=src/path/to/file.tsx
|
||||||
|
|
||||||
|
# Run lint-fix on all files
|
||||||
|
../gradlew yarnLintFix
|
||||||
|
|
||||||
# Run type checking
|
# Run type checking
|
||||||
yarn type-check
|
yarn type-check
|
||||||
|
|||||||
@ -105,12 +105,38 @@ task yarnTest(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
task yarnLint(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
task yarnLint(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||||
|
def targetFile = project.findProperty('file') ?: ''
|
||||||
|
|
||||||
|
if (targetFile.isEmpty()) {
|
||||||
|
// Run on all files
|
||||||
args = ['run', 'lint']
|
args = ['run', 'lint']
|
||||||
|
} else {
|
||||||
|
// Run on specific file - run lint, format-check, and type-check
|
||||||
|
doFirst {
|
||||||
|
// Run lint-file first
|
||||||
|
exec {
|
||||||
|
workingDir projectDir
|
||||||
|
commandLine 'yarn', 'run', 'lint-file', targetFile
|
||||||
|
}
|
||||||
|
// Then run format-check-file
|
||||||
|
exec {
|
||||||
|
workingDir projectDir
|
||||||
|
commandLine 'yarn', 'run', 'format-check-file', targetFile
|
||||||
|
}
|
||||||
|
// This does not run full type-check when we run for a single file
|
||||||
|
// as that would take too long
|
||||||
|
}
|
||||||
|
outputs.cacheIf { false }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
test.dependsOn([yarnInstall, yarnTest, yarnLint])
|
test.dependsOn([yarnInstall, yarnTest, yarnLint])
|
||||||
|
|
||||||
task yarnLintFix(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
task yarnLintFix(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||||
|
def targetFile = project.findProperty('file') ?: ''
|
||||||
|
|
||||||
|
if (targetFile.isEmpty()) {
|
||||||
|
// Run on all files
|
||||||
args = ['run', 'lint-fix']
|
args = ['run', 'lint-fix']
|
||||||
def lint_sentinel = "${buildDir}/.yarn-lint-sentinel"
|
def lint_sentinel = "${buildDir}/.yarn-lint-sentinel"
|
||||||
outputs.file(lint_sentinel)
|
outputs.file(lint_sentinel)
|
||||||
@ -125,6 +151,22 @@ task yarnLintFix(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
outputs.cacheIf { true }
|
outputs.cacheIf { true }
|
||||||
|
} else {
|
||||||
|
// Run on specific file - need to run both lint-fix and format
|
||||||
|
doFirst {
|
||||||
|
// Run lint-fix-file first
|
||||||
|
exec {
|
||||||
|
workingDir projectDir
|
||||||
|
commandLine 'yarn', 'run', 'lint-fix-file', targetFile
|
||||||
|
}
|
||||||
|
// Then run format-file
|
||||||
|
exec {
|
||||||
|
workingDir projectDir
|
||||||
|
commandLine 'yarn', 'run', 'format-file', targetFile
|
||||||
|
}
|
||||||
|
}
|
||||||
|
outputs.cacheIf { false }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
|
||||||
|
|||||||
@ -115,8 +115,12 @@
|
|||||||
"generate": "NODE_OPTIONS='--max-old-space-size=5120 --openssl-legacy-provider' graphql-codegen --config codegen.yml",
|
"generate": "NODE_OPTIONS='--max-old-space-size=5120 --openssl-legacy-provider' graphql-codegen --config codegen.yml",
|
||||||
"lint": "eslint . --ext .ts,.tsx --quiet && yarn format-check && yarn type-check",
|
"lint": "eslint . --ext .ts,.tsx --quiet && yarn format-check && yarn type-check",
|
||||||
"lint-fix": "eslint . --ext .ts,.tsx --quiet --fix && yarn format",
|
"lint-fix": "eslint . --ext .ts,.tsx --quiet --fix && yarn format",
|
||||||
|
"lint-file": "eslint --ext .ts,.tsx --quiet",
|
||||||
|
"format-check-file": "prettier --check",
|
||||||
|
"lint-fix-file": "eslint --ext .ts,.tsx --quiet --fix",
|
||||||
"format-check": "prettier --check src",
|
"format-check": "prettier --check src",
|
||||||
"format": "prettier --write src",
|
"format": "prettier --write src",
|
||||||
|
"format-file": "prettier --write",
|
||||||
"type-check": "tsc --noEmit",
|
"type-check": "tsc --noEmit",
|
||||||
"type-watch": "tsc -w --noEmit",
|
"type-watch": "tsc -w --noEmit",
|
||||||
"storybook": "storybook dev -p 6006",
|
"storybook": "storybook dev -p 6006",
|
||||||
|
|||||||
@ -42,6 +42,12 @@ const HeaderContainer = styled.div<{ hasChildren: boolean }>`
|
|||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
const TitleRow = styled.div`
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
`;
|
||||||
|
|
||||||
const ButtonsContainer = styled.div`
|
const ButtonsContainer = styled.div`
|
||||||
display: flex;
|
display: flex;
|
||||||
gap: 16px;
|
gap: 16px;
|
||||||
@ -59,6 +65,7 @@ export interface ModalProps {
|
|||||||
buttons: ModalButton[];
|
buttons: ModalButton[];
|
||||||
title: string;
|
title: string;
|
||||||
subtitle?: string;
|
subtitle?: string;
|
||||||
|
titlePill?: React.ReactNode;
|
||||||
children?: React.ReactNode;
|
children?: React.ReactNode;
|
||||||
onCancel: () => void;
|
onCancel: () => void;
|
||||||
dataTestId?: string;
|
dataTestId?: string;
|
||||||
@ -68,6 +75,7 @@ export function Modal({
|
|||||||
buttons,
|
buttons,
|
||||||
title,
|
title,
|
||||||
subtitle,
|
subtitle,
|
||||||
|
titlePill,
|
||||||
children,
|
children,
|
||||||
onCancel,
|
onCancel,
|
||||||
dataTestId,
|
dataTestId,
|
||||||
@ -83,9 +91,12 @@ export function Modal({
|
|||||||
data-testid={dataTestId}
|
data-testid={dataTestId}
|
||||||
title={
|
title={
|
||||||
<HeaderContainer hasChildren={!!children}>
|
<HeaderContainer hasChildren={!!children}>
|
||||||
|
<TitleRow>
|
||||||
<Heading type="h1" color="gray" colorLevel={600} weight="bold" size="lg">
|
<Heading type="h1" color="gray" colorLevel={600} weight="bold" size="lg">
|
||||||
{title}
|
{title}
|
||||||
</Heading>
|
</Heading>
|
||||||
|
{titlePill}
|
||||||
|
</TitleRow>
|
||||||
{!!subtitle && (
|
{!!subtitle && (
|
||||||
<Text type="span" color="gray" colorLevel={1700} weight="medium">
|
<Text type="span" color="gray" colorLevel={1700} weight="medium">
|
||||||
{subtitle}
|
{subtitle}
|
||||||
|
|||||||
@ -90,6 +90,19 @@ const meta = {
|
|||||||
getCurrentUrl: {
|
getCurrentUrl: {
|
||||||
description: 'A custom function to get the current URL. Defaults to window.location.pathname',
|
description: 'A custom function to get the current URL. Defaults to window.location.pathname',
|
||||||
},
|
},
|
||||||
|
scrollToTopOnChange: {
|
||||||
|
description: 'Whether to scroll to the top of the tabs container when switching tabs',
|
||||||
|
control: { type: 'boolean' },
|
||||||
|
},
|
||||||
|
maxHeight: {
|
||||||
|
description:
|
||||||
|
'Maximum height of the scrollable tabs container (only applies when scrollToTopOnChange is true)',
|
||||||
|
control: { type: 'text' },
|
||||||
|
},
|
||||||
|
stickyHeader: {
|
||||||
|
description: 'Whether to make the tab headers sticky when scrolling',
|
||||||
|
control: { type: 'boolean' },
|
||||||
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
// Args for the story
|
// Args for the story
|
||||||
@ -136,3 +149,77 @@ export const urlAware: Story = {
|
|||||||
tags: ['dev'],
|
tags: ['dev'],
|
||||||
render: () => <UrlAwareTabsDemo />,
|
render: () => <UrlAwareTabsDemo />,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const StickyHeaderTabsDemo = () => {
|
||||||
|
const [selectedTab, setSelectedTab] = React.useState('tab1');
|
||||||
|
|
||||||
|
const stickyTabs = [
|
||||||
|
{
|
||||||
|
key: 'tab1',
|
||||||
|
name: 'Tab One',
|
||||||
|
component: (
|
||||||
|
<div style={{ padding: '20px' }}>
|
||||||
|
<h3>Tab One with Sticky Header</h3>
|
||||||
|
{Array.from({ length: 40 }, (_, i) => (
|
||||||
|
<p key={i}>
|
||||||
|
This is paragraph {i + 1}. The tab header will stick to the top when you scroll down. Lorem
|
||||||
|
ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut
|
||||||
|
labore et dolore magna aliqua.
|
||||||
|
</p>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'tab2',
|
||||||
|
name: 'Tab Two',
|
||||||
|
component: (
|
||||||
|
<div style={{ padding: '20px' }}>
|
||||||
|
<h3>Tab Two Content</h3>
|
||||||
|
{Array.from({ length: 35 }, (_, i) => (
|
||||||
|
<div key={i} style={{ marginBottom: '15px', padding: '15px', backgroundColor: '#f0f0f0' }}>
|
||||||
|
Content block {i + 1} - Notice how the tab header remains visible at the top while
|
||||||
|
scrolling.
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'tab3',
|
||||||
|
name: 'Tab Three',
|
||||||
|
component: (
|
||||||
|
<div style={{ padding: '20px' }}>
|
||||||
|
<h3>Tab Three Content</h3>
|
||||||
|
{Array.from({ length: 50 }, (_, i) => (
|
||||||
|
<p key={i}>
|
||||||
|
Line {i + 1}: The sticky header allows you to easily switch between tabs even when scrolled
|
||||||
|
deep into the content. This is particularly useful for long content like logs or detailed
|
||||||
|
configuration files.
|
||||||
|
</p>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ width: '700px', height: '500px' }}>
|
||||||
|
<h4>Sticky Header & Scroll to Top on Tab Change Demo</h4>
|
||||||
|
<p>Scroll down within the tab content to see the sticky header behavior and switch between tabs</p>
|
||||||
|
<Tabs
|
||||||
|
tabs={stickyTabs}
|
||||||
|
selectedTab={selectedTab}
|
||||||
|
onChange={setSelectedTab}
|
||||||
|
scrollToTopOnChange
|
||||||
|
maxHeight="400px"
|
||||||
|
stickyHeader
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const stickyHeader: Story = {
|
||||||
|
tags: ['dev'],
|
||||||
|
render: () => <StickyHeaderTabsDemo />,
|
||||||
|
};
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
import { Tabs as AntTabs } from 'antd';
|
import { Tabs as AntTabs } from 'antd';
|
||||||
import React, { useEffect } from 'react';
|
import React, { useEffect, useRef } from 'react';
|
||||||
import styled from 'styled-components';
|
import styled from 'styled-components';
|
||||||
|
|
||||||
import { Pill } from '@components/components/Pills';
|
import { Pill } from '@components/components/Pills';
|
||||||
@ -7,9 +7,21 @@ import { Tooltip } from '@components/components/Tooltip';
|
|||||||
|
|
||||||
import { colors } from '@src/alchemy-components/theme';
|
import { colors } from '@src/alchemy-components/theme';
|
||||||
|
|
||||||
const StyledTabs = styled(AntTabs)<{ $addPaddingLeft?: boolean; $hideTabsHeader: boolean }>`
|
const ScrollableTabsContainer = styled.div<{ $maxHeight?: string }>`
|
||||||
flex: 1;
|
max-height: ${({ $maxHeight }) => $maxHeight || '100%'};
|
||||||
overflow: hidden;
|
overflow-y: auto;
|
||||||
|
height: 100%;
|
||||||
|
position: relative;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StyledTabs = styled(AntTabs)<{
|
||||||
|
$addPaddingLeft?: boolean;
|
||||||
|
$hideTabsHeader: boolean;
|
||||||
|
$scrollable?: boolean;
|
||||||
|
$stickyHeader?: boolean;
|
||||||
|
}>`
|
||||||
|
${({ $scrollable }) => !$scrollable && 'flex: 1;'}
|
||||||
|
${({ $scrollable }) => !$scrollable && 'overflow: hidden;'}
|
||||||
|
|
||||||
.ant-tabs-tab {
|
.ant-tabs-tab {
|
||||||
padding: 8px 0;
|
padding: 8px 0;
|
||||||
@ -38,6 +50,17 @@ const StyledTabs = styled(AntTabs)<{ $addPaddingLeft?: boolean; $hideTabsHeader:
|
|||||||
}
|
}
|
||||||
`}
|
`}
|
||||||
|
|
||||||
|
${({ $stickyHeader }) =>
|
||||||
|
$stickyHeader &&
|
||||||
|
`
|
||||||
|
.ant-tabs-nav {
|
||||||
|
position: sticky;
|
||||||
|
top: 0;
|
||||||
|
z-index: 10;
|
||||||
|
background-color: white;
|
||||||
|
}
|
||||||
|
`}
|
||||||
|
|
||||||
.ant-tabs-tab-active .ant-tabs-tab-btn {
|
.ant-tabs-tab-active .ant-tabs-tab-btn {
|
||||||
color: ${(props) => props.theme.styles['primary-color']};
|
color: ${(props) => props.theme.styles['primary-color']};
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
@ -58,6 +81,37 @@ const StyledTabs = styled(AntTabs)<{ $addPaddingLeft?: boolean; $hideTabsHeader:
|
|||||||
.ant-tabs-nav {
|
.ant-tabs-nav {
|
||||||
margin-bottom: 24px;
|
margin-bottom: 24px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
${({ $stickyHeader }) =>
|
||||||
|
$stickyHeader &&
|
||||||
|
`
|
||||||
|
.ant-tabs-nav::before {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.ant-tabs-nav-wrap::before {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.ant-tabs-nav-list::after {
|
||||||
|
content: '';
|
||||||
|
position: absolute;
|
||||||
|
bottom: 0;
|
||||||
|
left: 0;
|
||||||
|
right: 0;
|
||||||
|
height: 1px;
|
||||||
|
background-color: ${colors.gray[200]};
|
||||||
|
}
|
||||||
|
`}
|
||||||
|
|
||||||
|
${({ $addPaddingLeft, $stickyHeader }) =>
|
||||||
|
$addPaddingLeft &&
|
||||||
|
$stickyHeader &&
|
||||||
|
`
|
||||||
|
.ant-tabs-nav-list::after {
|
||||||
|
left: 16px;
|
||||||
|
}
|
||||||
|
`}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const TabViewWrapper = styled.div<{ $disabled?: boolean }>`
|
const TabViewWrapper = styled.div<{ $disabled?: boolean }>`
|
||||||
@ -100,6 +154,9 @@ export interface Props {
|
|||||||
getCurrentUrl?: () => string;
|
getCurrentUrl?: () => string;
|
||||||
addPaddingLeft?: boolean;
|
addPaddingLeft?: boolean;
|
||||||
hideTabsHeader?: boolean;
|
hideTabsHeader?: boolean;
|
||||||
|
scrollToTopOnChange?: boolean;
|
||||||
|
maxHeight?: string;
|
||||||
|
stickyHeader?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function Tabs({
|
export function Tabs({
|
||||||
@ -112,8 +169,19 @@ export function Tabs({
|
|||||||
getCurrentUrl = () => window.location.pathname,
|
getCurrentUrl = () => window.location.pathname,
|
||||||
addPaddingLeft,
|
addPaddingLeft,
|
||||||
hideTabsHeader,
|
hideTabsHeader,
|
||||||
|
scrollToTopOnChange = false,
|
||||||
|
maxHeight = '100%',
|
||||||
|
stickyHeader = false,
|
||||||
}: Props) {
|
}: Props) {
|
||||||
const { TabPane } = AntTabs;
|
const { TabPane } = AntTabs;
|
||||||
|
const tabsContainerRef = useRef<HTMLDivElement>(null);
|
||||||
|
|
||||||
|
// Scroll to top when selectedTab changes if scrollToTopOnChange is enabled
|
||||||
|
useEffect(() => {
|
||||||
|
if (scrollToTopOnChange && tabsContainerRef.current) {
|
||||||
|
tabsContainerRef.current.scrollTo({ top: 0, behavior: 'smooth' });
|
||||||
|
}
|
||||||
|
}, [selectedTab, scrollToTopOnChange]);
|
||||||
|
|
||||||
// Create reverse mapping from URLs to tab keys if urlMap is provided
|
// Create reverse mapping from URLs to tab keys if urlMap is provided
|
||||||
const urlToTabMap = React.useMemo(() => {
|
const urlToTabMap = React.useMemo(() => {
|
||||||
@ -140,23 +208,19 @@ export function Tabs({
|
|||||||
}
|
}
|
||||||
}, [getCurrentUrl, onChange, onUrlChange, selectedTab, urlMap, urlToTabMap, defaultTab]);
|
}, [getCurrentUrl, onChange, onUrlChange, selectedTab, urlMap, urlToTabMap, defaultTab]);
|
||||||
|
|
||||||
function handleTabClick(key: string) {
|
const tabsContent = (
|
||||||
onChange?.(key);
|
|
||||||
const newTab = tabs.find((t) => t.key === key);
|
|
||||||
newTab?.onSelectTab?.();
|
|
||||||
|
|
||||||
// Update URL if urlMap is provided
|
|
||||||
if (urlMap && urlMap[key]) {
|
|
||||||
onUrlChange(urlMap[key]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<StyledTabs
|
<StyledTabs
|
||||||
activeKey={selectedTab}
|
activeKey={selectedTab}
|
||||||
onChange={handleTabClick}
|
onChange={(key) => {
|
||||||
|
if (onChange) onChange(key);
|
||||||
|
if (urlMap && onUrlChange && urlMap[key]) {
|
||||||
|
onUrlChange(urlMap[key]);
|
||||||
|
}
|
||||||
|
}}
|
||||||
$addPaddingLeft={addPaddingLeft}
|
$addPaddingLeft={addPaddingLeft}
|
||||||
$hideTabsHeader={!!hideTabsHeader}
|
$hideTabsHeader={!!hideTabsHeader}
|
||||||
|
$scrollable={scrollToTopOnChange}
|
||||||
|
$stickyHeader={stickyHeader}
|
||||||
>
|
>
|
||||||
{tabs.map((tab) => {
|
{tabs.map((tab) => {
|
||||||
return (
|
return (
|
||||||
@ -167,4 +231,14 @@ export function Tabs({
|
|||||||
})}
|
})}
|
||||||
</StyledTabs>
|
</StyledTabs>
|
||||||
);
|
);
|
||||||
|
|
||||||
|
if (scrollToTopOnChange) {
|
||||||
|
return (
|
||||||
|
<ScrollableTabsContainer ref={tabsContainerRef} $maxHeight={maxHeight}>
|
||||||
|
{tabsContent}
|
||||||
|
</ScrollableTabsContainer>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return tabsContent;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
import { Button, PageTitle, Tabs, Tooltip } from '@components';
|
import { Button, PageTitle, Tabs, Tooltip } from '@components';
|
||||||
import React, { useEffect, useRef, useState } from 'react';
|
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||||
import { useHistory } from 'react-router';
|
import { useHistory } from 'react-router';
|
||||||
import styled from 'styled-components';
|
import styled from 'styled-components';
|
||||||
|
|
||||||
@ -112,16 +112,6 @@ export const ManageIngestionPage = () => {
|
|||||||
}
|
}
|
||||||
}, [loadedAppConfig, loadedPlatformPrivileges, showIngestionTab, showSecretsTab, selectedTab]);
|
}, [loadedAppConfig, loadedPlatformPrivileges, showIngestionTab, showSecretsTab, selectedTab]);
|
||||||
|
|
||||||
const onSwitchTab = (newTab: string, options?: { clearQueryParams: boolean }) => {
|
|
||||||
const preserveParams = shouldPreserveParams.current;
|
|
||||||
const matchingTab = Object.values(TabType).find((tab) => tab === newTab);
|
|
||||||
if (!preserveParams && options?.clearQueryParams) {
|
|
||||||
history.push({ search: '' });
|
|
||||||
}
|
|
||||||
setSelectedTab(matchingTab || selectedTab);
|
|
||||||
shouldPreserveParams.current = false;
|
|
||||||
};
|
|
||||||
|
|
||||||
const tabs: Tab[] = [
|
const tabs: Tab[] = [
|
||||||
showIngestionTab && {
|
showIngestionTab && {
|
||||||
component: (
|
component: (
|
||||||
@ -160,9 +150,14 @@ export const ManageIngestionPage = () => {
|
|||||||
},
|
},
|
||||||
].filter((tab): tab is Tab => Boolean(tab));
|
].filter((tab): tab is Tab => Boolean(tab));
|
||||||
|
|
||||||
const onUrlChange = (tabPath: string) => {
|
const onUrlChange = useCallback(
|
||||||
history.push(tabPath);
|
(tabPath: string) => {
|
||||||
};
|
history.push({ pathname: tabPath, search: '' });
|
||||||
|
},
|
||||||
|
[history],
|
||||||
|
);
|
||||||
|
|
||||||
|
const getCurrentUrl = useCallback(() => window.location.pathname, []);
|
||||||
|
|
||||||
const handleCreateSource = () => {
|
const handleCreateSource = () => {
|
||||||
setShowCreateSourceModal(true);
|
setShowCreateSourceModal(true);
|
||||||
@ -228,11 +223,11 @@ export const ManageIngestionPage = () => {
|
|||||||
<Tabs
|
<Tabs
|
||||||
tabs={tabs}
|
tabs={tabs}
|
||||||
selectedTab={selectedTab}
|
selectedTab={selectedTab}
|
||||||
onChange={(tab) => onSwitchTab(tab, { clearQueryParams: true })}
|
onChange={(tab) => setSelectedTab(tab as TabType)}
|
||||||
urlMap={tabUrlMap}
|
urlMap={tabUrlMap}
|
||||||
onUrlChange={onUrlChange}
|
onUrlChange={onUrlChange}
|
||||||
defaultTab={TabType.Sources}
|
defaultTab={TabType.Sources}
|
||||||
getCurrentUrl={() => window.location.pathname}
|
getCurrentUrl={getCurrentUrl}
|
||||||
/>
|
/>
|
||||||
</PageContentContainer>
|
</PageContentContainer>
|
||||||
</PageContainer>
|
</PageContainer>
|
||||||
|
|||||||
@ -0,0 +1,14 @@
|
|||||||
|
import { Typography } from 'antd';
|
||||||
|
import styled from 'styled-components';
|
||||||
|
|
||||||
|
export const SectionBase = styled.div`
|
||||||
|
padding: 16px 30px 0;
|
||||||
|
`;
|
||||||
|
|
||||||
|
export const SectionHeader = styled(Typography.Title)`
|
||||||
|
&&&& {
|
||||||
|
padding: 0px;
|
||||||
|
margin: 0px;
|
||||||
|
margin-bottom: 12px;
|
||||||
|
}
|
||||||
|
`;
|
||||||
@ -1,111 +1,30 @@
|
|||||||
import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons';
|
import { LoadingOutlined } from '@ant-design/icons';
|
||||||
import { Icon, Modal, Pill } from '@components';
|
import { Icon, Modal, Pill } from '@components';
|
||||||
import { Button, Typography, message } from 'antd';
|
import { message } from 'antd';
|
||||||
import React, { useEffect, useState } from 'react';
|
import React, { useEffect, useState } from 'react';
|
||||||
import styled from 'styled-components';
|
|
||||||
import YAML from 'yamljs';
|
|
||||||
|
|
||||||
import { ANTD_GRAY } from '@app/entity/shared/constants';
|
import { Tab, Tabs } from '@components/components/Tabs/Tabs';
|
||||||
import { StructuredReport } from '@app/ingestV2/executions/components/reporting/StructuredReport';
|
|
||||||
import {
|
import { LogsTab } from '@app/ingestV2/executions/components/LogsTab';
|
||||||
EXECUTION_REQUEST_STATUS_LOADING,
|
import { RecipeTab } from '@app/ingestV2/executions/components/RecipeTab';
|
||||||
EXECUTION_REQUEST_STATUS_RUNNING,
|
import { SummaryTab } from '@app/ingestV2/executions/components/SummaryTab';
|
||||||
EXECUTION_REQUEST_STATUS_SUCCEEDED_WITH_WARNINGS,
|
import { EXECUTION_REQUEST_STATUS_LOADING, EXECUTION_REQUEST_STATUS_RUNNING } from '@app/ingestV2/executions/constants';
|
||||||
EXECUTION_REQUEST_STATUS_SUCCESS,
|
import { TabType } from '@app/ingestV2/executions/types';
|
||||||
} from '@app/ingestV2/executions/constants';
|
|
||||||
import {
|
import {
|
||||||
getExecutionRequestStatusDisplayColor,
|
getExecutionRequestStatusDisplayColor,
|
||||||
getExecutionRequestStatusDisplayText,
|
getExecutionRequestStatusDisplayText,
|
||||||
getExecutionRequestStatusIcon,
|
getExecutionRequestStatusIcon,
|
||||||
getExecutionRequestSummaryText,
|
|
||||||
} from '@app/ingestV2/executions/utils';
|
} from '@app/ingestV2/executions/utils';
|
||||||
import IngestedAssets from '@app/ingestV2/source/IngestedAssets';
|
import { getIngestionSourceStatus } from '@app/ingestV2/source/utils';
|
||||||
import { getIngestionSourceStatus, getStructuredReport } from '@app/ingestV2/source/utils';
|
|
||||||
import { downloadFile } from '@app/search/utils/csvUtils';
|
|
||||||
import { Message } from '@app/shared/Message';
|
import { Message } from '@app/shared/Message';
|
||||||
|
|
||||||
import { useGetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
import { useGetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
||||||
import { ExecutionRequestResult } from '@types';
|
import { ExecutionRequestResult } from '@types';
|
||||||
|
|
||||||
const Section = styled.div`
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
padding-bottom: 12px;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const SectionHeader = styled(Typography.Title)`
|
|
||||||
&&&& {
|
|
||||||
padding: 0px;
|
|
||||||
margin: 0px;
|
|
||||||
margin-bottom: 12px;
|
|
||||||
}
|
|
||||||
`;
|
|
||||||
|
|
||||||
const SectionSubHeader = styled.div`
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
align-items: center;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const SubHeaderParagraph = styled(Typography.Paragraph)`
|
|
||||||
margin-bottom: 0px;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const StatusSection = styled.div`
|
|
||||||
border-bottom: 1px solid ${ANTD_GRAY[4]};
|
|
||||||
padding: 16px;
|
|
||||||
padding-left: 30px;
|
|
||||||
padding-right: 30px;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const ResultText = styled.div`
|
|
||||||
margin-bottom: 4px;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const IngestedAssetsSection = styled.div`
|
|
||||||
border-bottom: 1px solid ${ANTD_GRAY[4]};
|
|
||||||
padding: 16px;
|
|
||||||
padding-left: 30px;
|
|
||||||
padding-right: 30px;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const RecipeSection = styled.div`
|
|
||||||
border-top: 1px solid ${ANTD_GRAY[4]};
|
|
||||||
padding-top: 16px;
|
|
||||||
padding-left: 30px;
|
|
||||||
padding-right: 30px;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const LogsSection = styled.div`
|
|
||||||
padding-top: 16px;
|
|
||||||
padding-left: 30px;
|
|
||||||
padding-right: 30px;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const ShowMoreButton = styled(Button)`
|
|
||||||
padding: 0px;
|
|
||||||
`;
|
|
||||||
|
|
||||||
const DetailsContainer = styled.div<DetailsContainerProps>`
|
|
||||||
margin-bottom: -25px;
|
|
||||||
${(props) =>
|
|
||||||
props.areDetailsExpandable &&
|
|
||||||
!props.showExpandedDetails &&
|
|
||||||
`
|
|
||||||
-webkit-mask-image: linear-gradient(to bottom, rgba(0,0,0,1) 50%, rgba(255,0,0,0.5) 60%, rgba(255,0,0,0) 90% );
|
|
||||||
mask-image: linear-gradient(to bottom, rgba(0,0,0,1) 50%, rgba(255,0,0,0.5) 60%, rgba(255,0,0,0) 90%);
|
|
||||||
`}
|
|
||||||
`;
|
|
||||||
|
|
||||||
const modalBodyStyle = {
|
const modalBodyStyle = {
|
||||||
padding: 0,
|
padding: 0,
|
||||||
};
|
};
|
||||||
|
|
||||||
type DetailsContainerProps = {
|
|
||||||
showExpandedDetails: boolean;
|
|
||||||
areDetailsExpandable: boolean;
|
|
||||||
};
|
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
urn: string;
|
urn: string;
|
||||||
open: boolean;
|
open: boolean;
|
||||||
@ -113,33 +32,14 @@ type Props = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
|
export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
|
||||||
const [showExpandedLogs, setShowExpandedLogs] = useState(false);
|
|
||||||
const [showExpandedRecipe, setShowExpandedRecipe] = useState(false);
|
|
||||||
|
|
||||||
const { data, loading, error, refetch } = useGetIngestionExecutionRequestQuery({ variables: { urn } });
|
const { data, loading, error, refetch } = useGetIngestionExecutionRequestQuery({ variables: { urn } });
|
||||||
const output = data?.executionRequest?.result?.report || 'No output found.';
|
|
||||||
|
|
||||||
const downloadLogs = () => {
|
|
||||||
downloadFile(output, `exec-${urn}.log`);
|
|
||||||
};
|
|
||||||
|
|
||||||
const logs = (showExpandedLogs && output) || output?.split('\n')?.slice(0, 5)?.join('\n');
|
|
||||||
const result = data?.executionRequest?.result as Partial<ExecutionRequestResult>;
|
const result = data?.executionRequest?.result as Partial<ExecutionRequestResult>;
|
||||||
const status = getIngestionSourceStatus(result);
|
const status = getIngestionSourceStatus(result);
|
||||||
|
const [selectedTab, setSelectedTab] = useState<TabType>(TabType.Summary);
|
||||||
useEffect(() => {
|
|
||||||
const interval = setInterval(() => {
|
|
||||||
if (status === EXECUTION_REQUEST_STATUS_RUNNING) refetch();
|
|
||||||
}, 2000);
|
|
||||||
|
|
||||||
return () => clearInterval(interval);
|
|
||||||
});
|
|
||||||
|
|
||||||
const ResultIcon = status && getExecutionRequestStatusIcon(status);
|
const ResultIcon = status && getExecutionRequestStatusIcon(status);
|
||||||
const resultColor = status ? getExecutionRequestStatusDisplayColor(status) : 'gray';
|
const resultColor = status ? getExecutionRequestStatusDisplayColor(status) : 'gray';
|
||||||
const resultText = status && (
|
const titlePill = status && ResultIcon && (
|
||||||
<Typography.Text style={{ color: resultColor, fontSize: 14 }}>
|
|
||||||
{ResultIcon && (
|
|
||||||
<Pill
|
<Pill
|
||||||
customIconRenderer={() =>
|
customIconRenderer={() =>
|
||||||
status === EXECUTION_REQUEST_STATUS_LOADING || status === EXECUTION_REQUEST_STATUS_RUNNING ? (
|
status === EXECUTION_REQUEST_STATUS_LOADING || status === EXECUTION_REQUEST_STATUS_RUNNING ? (
|
||||||
@ -152,100 +52,64 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
|
|||||||
color={resultColor}
|
color={resultColor}
|
||||||
size="md"
|
size="md"
|
||||||
/>
|
/>
|
||||||
)}
|
|
||||||
</Typography.Text>
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const structuredReport = result && getStructuredReport(result);
|
useEffect(() => {
|
||||||
|
const interval = setInterval(() => {
|
||||||
|
if (status === EXECUTION_REQUEST_STATUS_RUNNING) refetch();
|
||||||
|
}, 2000);
|
||||||
|
|
||||||
const resultSummaryText =
|
return () => clearInterval(interval);
|
||||||
(status && <Typography.Text type="secondary">{getExecutionRequestSummaryText(status)}</Typography.Text>) ||
|
});
|
||||||
undefined;
|
|
||||||
|
|
||||||
const recipeJson = data?.executionRequest?.input?.arguments?.find((arg) => arg.key === 'recipe')?.value;
|
const tabs: Tab[] = [
|
||||||
let recipeYaml: string;
|
{
|
||||||
try {
|
component: (
|
||||||
recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
|
<SummaryTab
|
||||||
} catch (e) {
|
urn={urn}
|
||||||
recipeYaml = '';
|
status={status}
|
||||||
}
|
result={result}
|
||||||
const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n')?.slice(0, 5)?.join('\n');
|
data={data}
|
||||||
|
onTabChange={(tab: TabType) => setSelectedTab(tab)}
|
||||||
const areLogsExpandable = output?.split(/\r\n|\r|\n/)?.length > 5;
|
/>
|
||||||
const isRecipeExpandable = recipeYaml?.split(/\r\n|\r|\n/)?.length > 5;
|
),
|
||||||
|
key: TabType.Summary,
|
||||||
|
name: TabType.Summary,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
component: <LogsTab urn={urn} data={data} />,
|
||||||
|
key: TabType.Logs,
|
||||||
|
name: TabType.Logs,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
component: <RecipeTab data={data} />,
|
||||||
|
key: TabType.Recipe,
|
||||||
|
name: TabType.Recipe,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Modal
|
<Modal
|
||||||
width={800}
|
width="1400px"
|
||||||
bodyStyle={modalBodyStyle}
|
bodyStyle={modalBodyStyle}
|
||||||
title="Execution Run Details"
|
title="Status Details"
|
||||||
|
titlePill={titlePill}
|
||||||
open={open}
|
open={open}
|
||||||
onCancel={onClose}
|
onCancel={onClose}
|
||||||
buttons={[{ text: 'Close', variant: 'outline', onClick: onClose }]}
|
buttons={[{ text: 'Close', variant: 'outline', onClick: onClose }]}
|
||||||
>
|
>
|
||||||
{!data && loading && <Message type="loading" content="Loading execution run details..." />}
|
{!data && loading && <Message type="loading" content="Loading execution run details..." />}
|
||||||
{error && message.error('Failed to load execution run details :(')}
|
{error && message.error('Failed to load execution run details :(')}
|
||||||
<Section>
|
<Tabs
|
||||||
<StatusSection>
|
tabs={tabs}
|
||||||
<Typography.Title level={5}>Status</Typography.Title>
|
selectedTab={selectedTab}
|
||||||
<ResultText>{resultText}</ResultText>
|
onChange={(tab) => setSelectedTab(tab as TabType)}
|
||||||
<SubHeaderParagraph>{resultSummaryText}</SubHeaderParagraph>
|
getCurrentUrl={() => window.location.pathname}
|
||||||
{structuredReport ? <StructuredReport report={structuredReport} /> : null}
|
scrollToTopOnChange
|
||||||
</StatusSection>
|
maxHeight="80vh"
|
||||||
{(status === EXECUTION_REQUEST_STATUS_SUCCESS ||
|
stickyHeader
|
||||||
status === EXECUTION_REQUEST_STATUS_SUCCEEDED_WITH_WARNINGS) && (
|
addPaddingLeft
|
||||||
<IngestedAssetsSection>
|
/>
|
||||||
{data?.executionRequest?.id && (
|
|
||||||
<IngestedAssets executionResult={result} id={data?.executionRequest?.id} />
|
|
||||||
)}
|
|
||||||
</IngestedAssetsSection>
|
|
||||||
)}
|
|
||||||
<LogsSection>
|
|
||||||
<SectionHeader level={5}>Logs</SectionHeader>
|
|
||||||
<SectionSubHeader>
|
|
||||||
<SubHeaderParagraph type="secondary">
|
|
||||||
View logs that were collected during the sync.
|
|
||||||
</SubHeaderParagraph>
|
|
||||||
<Button type="text" onClick={downloadLogs}>
|
|
||||||
<DownloadOutlined />
|
|
||||||
Download
|
|
||||||
</Button>
|
|
||||||
</SectionSubHeader>
|
|
||||||
<DetailsContainer areDetailsExpandable={areLogsExpandable} showExpandedDetails={showExpandedLogs}>
|
|
||||||
<Typography.Paragraph ellipsis>
|
|
||||||
<pre>{`${logs}${!showExpandedLogs && areLogsExpandable ? '...' : ''}`}</pre>
|
|
||||||
</Typography.Paragraph>
|
|
||||||
</DetailsContainer>
|
|
||||||
{areLogsExpandable && (
|
|
||||||
<ShowMoreButton type="link" onClick={() => setShowExpandedLogs(!showExpandedLogs)}>
|
|
||||||
{showExpandedLogs ? 'Hide' : 'Show More'}
|
|
||||||
</ShowMoreButton>
|
|
||||||
)}
|
|
||||||
</LogsSection>
|
|
||||||
{recipe && (
|
|
||||||
<RecipeSection>
|
|
||||||
<SectionHeader level={5}>Recipe</SectionHeader>
|
|
||||||
<SectionSubHeader>
|
|
||||||
<SubHeaderParagraph type="secondary">
|
|
||||||
The configurations used for this sync with the data source.
|
|
||||||
</SubHeaderParagraph>
|
|
||||||
</SectionSubHeader>
|
|
||||||
<DetailsContainer
|
|
||||||
areDetailsExpandable={isRecipeExpandable}
|
|
||||||
showExpandedDetails={showExpandedRecipe}
|
|
||||||
>
|
|
||||||
<Typography.Paragraph ellipsis>
|
|
||||||
<pre>{`${recipe}${!showExpandedRecipe && isRecipeExpandable ? '...' : ''}`}</pre>
|
|
||||||
</Typography.Paragraph>
|
|
||||||
</DetailsContainer>
|
|
||||||
{isRecipeExpandable && (
|
|
||||||
<ShowMoreButton type="link" onClick={() => setShowExpandedRecipe((v) => !v)}>
|
|
||||||
{showExpandedRecipe ? 'Hide' : 'Show More'}
|
|
||||||
</ShowMoreButton>
|
|
||||||
)}
|
|
||||||
</RecipeSection>
|
|
||||||
)}
|
|
||||||
</Section>
|
|
||||||
</Modal>
|
</Modal>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|||||||
@ -0,0 +1,49 @@
|
|||||||
|
import { DownloadOutlined } from '@ant-design/icons';
|
||||||
|
import { Button, Typography } from 'antd';
|
||||||
|
import React from 'react';
|
||||||
|
import styled from 'styled-components';
|
||||||
|
|
||||||
|
import { SectionHeader } from '@app/ingestV2/executions/components/BaseTab';
|
||||||
|
import { downloadFile } from '@app/search/utils/csvUtils';
|
||||||
|
|
||||||
|
import { GetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
||||||
|
|
||||||
|
const SectionSubHeader = styled.div`
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const SubHeaderParagraph = styled(Typography.Paragraph)`
|
||||||
|
margin-bottom: 0px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const LogsSection = styled.div`
|
||||||
|
padding-top: 16px;
|
||||||
|
padding-left: 30px;
|
||||||
|
padding-right: 30px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
export const LogsTab = ({ urn, data }: { urn: string; data: GetIngestionExecutionRequestQuery | undefined }) => {
|
||||||
|
const output = data?.executionRequest?.result?.report || 'No output found.';
|
||||||
|
|
||||||
|
const downloadLogs = () => {
|
||||||
|
downloadFile(output, `exec-${urn}.log`);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<LogsSection>
|
||||||
|
<SectionHeader level={5}>Logs</SectionHeader>
|
||||||
|
<SectionSubHeader>
|
||||||
|
<SubHeaderParagraph type="secondary">View logs that were collected during the sync.</SubHeaderParagraph>
|
||||||
|
<Button type="text" onClick={downloadLogs}>
|
||||||
|
<DownloadOutlined />
|
||||||
|
Download
|
||||||
|
</Button>
|
||||||
|
</SectionSubHeader>
|
||||||
|
<Typography.Paragraph ellipsis>
|
||||||
|
<pre>{output}</pre>
|
||||||
|
</Typography.Paragraph>
|
||||||
|
</LogsSection>
|
||||||
|
);
|
||||||
|
};
|
||||||
@ -0,0 +1,47 @@
|
|||||||
|
import { Typography } from 'antd';
|
||||||
|
import React from 'react';
|
||||||
|
import styled from 'styled-components';
|
||||||
|
import YAML from 'yamljs';
|
||||||
|
|
||||||
|
import { SectionBase, SectionHeader } from '@app/ingestV2/executions/components/BaseTab';
|
||||||
|
import colors from '@src/alchemy-components/theme/foundations/colors';
|
||||||
|
|
||||||
|
import { GetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
||||||
|
|
||||||
|
const SectionSubHeader = styled.div`
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const SubHeaderParagraph = styled(Typography.Paragraph)`
|
||||||
|
margin-bottom: 0px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const RecipeSection = styled(SectionBase)`
|
||||||
|
border-top: 1px solid ${colors.gray[1400]};
|
||||||
|
`;
|
||||||
|
|
||||||
|
export const RecipeTab = ({ data }: { data: GetIngestionExecutionRequestQuery | undefined }) => {
|
||||||
|
const recipeJson = data?.executionRequest?.input?.arguments?.find((arg) => arg.key === 'recipe')?.value;
|
||||||
|
let recipeYaml: string;
|
||||||
|
try {
|
||||||
|
recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
|
||||||
|
} catch (e) {
|
||||||
|
recipeYaml = '';
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<RecipeSection>
|
||||||
|
<SectionHeader level={5}>Recipe</SectionHeader>
|
||||||
|
<SectionSubHeader>
|
||||||
|
<SubHeaderParagraph type="secondary">
|
||||||
|
The configurations used for this sync with the data source.
|
||||||
|
</SubHeaderParagraph>
|
||||||
|
</SectionSubHeader>
|
||||||
|
<Typography.Paragraph ellipsis>
|
||||||
|
<pre>{recipeYaml || 'No recipe found.'}</pre>
|
||||||
|
</Typography.Paragraph>
|
||||||
|
</RecipeSection>
|
||||||
|
);
|
||||||
|
};
|
||||||
@ -0,0 +1,187 @@
|
|||||||
|
import { DownloadOutlined } from '@ant-design/icons';
|
||||||
|
import { Button, Typography } from 'antd';
|
||||||
|
import React, { useState } from 'react';
|
||||||
|
import styled from 'styled-components';
|
||||||
|
import YAML from 'yamljs';
|
||||||
|
|
||||||
|
import { SectionBase, SectionHeader } from '@app/ingestV2/executions/components/BaseTab';
|
||||||
|
import { StructuredReport, hasSomethingToShow } from '@app/ingestV2/executions/components/reporting/StructuredReport';
|
||||||
|
import { EXECUTION_REQUEST_STATUS_SUCCESS } from '@app/ingestV2/executions/constants';
|
||||||
|
import { TabType } from '@app/ingestV2/executions/types';
|
||||||
|
import { getExecutionRequestSummaryText } from '@app/ingestV2/executions/utils';
|
||||||
|
import IngestedAssets from '@app/ingestV2/source/IngestedAssets';
|
||||||
|
import { getStructuredReport } from '@app/ingestV2/source/utils';
|
||||||
|
import { downloadFile } from '@app/search/utils/csvUtils';
|
||||||
|
import colors from '@src/alchemy-components/theme/foundations/colors';
|
||||||
|
|
||||||
|
import { GetIngestionExecutionRequestQuery } from '@graphql/ingestion.generated';
|
||||||
|
import { ExecutionRequestResult } from '@types';
|
||||||
|
|
||||||
|
const Section = styled.div`
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
padding-bottom: 12px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const SectionSubHeader = styled.div`
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const ButtonGroup = styled.div`
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
align-items: center;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const SubHeaderParagraph = styled(Typography.Paragraph)`
|
||||||
|
margin-bottom: 0px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const StatusSection = styled.div`
|
||||||
|
border-bottom: 1px solid ${colors.gray[1400]};
|
||||||
|
padding: 16px;
|
||||||
|
padding-left: 30px;
|
||||||
|
padding-right: 30px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const IngestedAssetsSection = styled.div<{ isFirstSection?: boolean }>`
|
||||||
|
border-bottom: 1px solid ${colors.gray[1400]};
|
||||||
|
${({ isFirstSection }) => !isFirstSection && `border-top: 1px solid ${colors.gray[1400]};`}
|
||||||
|
padding: 16px;
|
||||||
|
padding-left: 30px;
|
||||||
|
padding-right: 30px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const RecipeSection = styled(SectionBase)``;
|
||||||
|
|
||||||
|
const LogsSection = styled(SectionBase)``;
|
||||||
|
|
||||||
|
const ShowMoreButton = styled(Button)`
|
||||||
|
padding: 0px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const DetailsContainer = styled.div`
|
||||||
|
margin-top: 12px;
|
||||||
|
|
||||||
|
pre {
|
||||||
|
background-color: ${colors.gray[1500]};
|
||||||
|
border: 1px solid ${colors.gray[1400]};
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 16px;
|
||||||
|
margin: 0;
|
||||||
|
color: ${colors.gray[1700]};
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
export const SummaryTab = ({
|
||||||
|
urn,
|
||||||
|
status,
|
||||||
|
result,
|
||||||
|
data,
|
||||||
|
onTabChange,
|
||||||
|
}: {
|
||||||
|
urn: string;
|
||||||
|
status: string | undefined;
|
||||||
|
result: Partial<ExecutionRequestResult>;
|
||||||
|
data: GetIngestionExecutionRequestQuery | undefined;
|
||||||
|
onTabChange: (tab: TabType) => void;
|
||||||
|
}) => {
|
||||||
|
const [showExpandedLogs] = useState(false);
|
||||||
|
const [showExpandedRecipe] = useState(false);
|
||||||
|
|
||||||
|
const output = data?.executionRequest?.result?.report || 'No output found.';
|
||||||
|
|
||||||
|
const downloadLogs = () => {
|
||||||
|
downloadFile(output, `exec-${urn}.log`);
|
||||||
|
};
|
||||||
|
|
||||||
|
const logs = (showExpandedLogs && output) || output?.split('\n')?.slice(0, 14)?.join('\n');
|
||||||
|
const structuredReport = result && getStructuredReport(result);
|
||||||
|
const resultSummaryText =
|
||||||
|
(status && status !== EXECUTION_REQUEST_STATUS_SUCCESS && (
|
||||||
|
<Typography.Text type="secondary">{getExecutionRequestSummaryText(status)}</Typography.Text>
|
||||||
|
)) ||
|
||||||
|
undefined;
|
||||||
|
const recipeJson = data?.executionRequest?.input?.arguments?.find((arg) => arg.key === 'recipe')?.value;
|
||||||
|
let recipeYaml: string;
|
||||||
|
try {
|
||||||
|
recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
|
||||||
|
} catch (e) {
|
||||||
|
recipeYaml = '';
|
||||||
|
}
|
||||||
|
const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n')?.slice(0, 14)?.join('\n');
|
||||||
|
const areLogsExpandable = output?.split(/\r\n|\r|\n/)?.length > 14;
|
||||||
|
const isRecipeExpandable = recipeYaml?.split(/\r\n|\r|\n/)?.length > 14;
|
||||||
|
|
||||||
|
const downloadRecipe = () => {
|
||||||
|
downloadFile(recipeYaml, `recipe-${urn}.yaml`);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Section>
|
||||||
|
{(resultSummaryText || (structuredReport && hasSomethingToShow(structuredReport))) && (
|
||||||
|
<StatusSection>
|
||||||
|
<SubHeaderParagraph>{resultSummaryText}</SubHeaderParagraph>
|
||||||
|
{structuredReport && structuredReport ? <StructuredReport report={structuredReport} /> : null}
|
||||||
|
</StatusSection>
|
||||||
|
)}
|
||||||
|
<IngestedAssetsSection
|
||||||
|
isFirstSection={!resultSummaryText && !(structuredReport && hasSomethingToShow(structuredReport))}
|
||||||
|
>
|
||||||
|
{data?.executionRequest?.id && (
|
||||||
|
<IngestedAssets executionResult={result} id={data?.executionRequest?.id} />
|
||||||
|
)}
|
||||||
|
</IngestedAssetsSection>
|
||||||
|
<LogsSection>
|
||||||
|
<SectionHeader level={5}>Logs</SectionHeader>
|
||||||
|
<SectionSubHeader>
|
||||||
|
<SubHeaderParagraph type="secondary">
|
||||||
|
View logs that were collected during the sync.
|
||||||
|
</SubHeaderParagraph>
|
||||||
|
<ButtonGroup>
|
||||||
|
{areLogsExpandable && (
|
||||||
|
<ShowMoreButton type="text" onClick={() => onTabChange(TabType.Logs)}>
|
||||||
|
View More
|
||||||
|
</ShowMoreButton>
|
||||||
|
)}
|
||||||
|
<Button type="text" onClick={downloadLogs}>
|
||||||
|
<DownloadOutlined />
|
||||||
|
</Button>
|
||||||
|
</ButtonGroup>
|
||||||
|
</SectionSubHeader>
|
||||||
|
<DetailsContainer>
|
||||||
|
<Typography.Paragraph ellipsis>
|
||||||
|
<pre>{`${logs}${areLogsExpandable ? '...' : ''}`}</pre>
|
||||||
|
</Typography.Paragraph>
|
||||||
|
</DetailsContainer>
|
||||||
|
</LogsSection>
|
||||||
|
{recipe && (
|
||||||
|
<RecipeSection>
|
||||||
|
<SectionHeader level={5}>Recipe</SectionHeader>
|
||||||
|
<SectionSubHeader>
|
||||||
|
<SubHeaderParagraph type="secondary">
|
||||||
|
The configurations used for this sync with the data source.
|
||||||
|
</SubHeaderParagraph>
|
||||||
|
<ButtonGroup>
|
||||||
|
{isRecipeExpandable && (
|
||||||
|
<ShowMoreButton type="text" onClick={() => onTabChange(TabType.Recipe)}>
|
||||||
|
View More
|
||||||
|
</ShowMoreButton>
|
||||||
|
)}
|
||||||
|
<Button type="text" onClick={downloadRecipe}>
|
||||||
|
<DownloadOutlined />
|
||||||
|
</Button>
|
||||||
|
</ButtonGroup>
|
||||||
|
</SectionSubHeader>
|
||||||
|
<DetailsContainer>
|
||||||
|
<Typography.Paragraph ellipsis>
|
||||||
|
<pre>{`${recipe}${!showExpandedRecipe && isRecipeExpandable ? '...' : ''}`}</pre>
|
||||||
|
</Typography.Paragraph>
|
||||||
|
</DetailsContainer>
|
||||||
|
</RecipeSection>
|
||||||
|
)}
|
||||||
|
</Section>
|
||||||
|
);
|
||||||
|
};
|
||||||
@ -23,6 +23,13 @@ interface Props {
|
|||||||
report: StructuredReportType;
|
report: StructuredReportType;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function hasSomethingToShow(report: StructuredReportType): boolean {
|
||||||
|
const warnings = report.items.filter((item) => item.level === StructuredReportItemLevel.WARN);
|
||||||
|
const errors = report.items.filter((item) => item.level === StructuredReportItemLevel.ERROR);
|
||||||
|
const infos = report.items.filter((item) => item.level === StructuredReportItemLevel.INFO);
|
||||||
|
return warnings.length > 0 || errors.length > 0 || infos.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
export function StructuredReport({ report }: Props) {
|
export function StructuredReport({ report }: Props) {
|
||||||
if (!report.items.length) {
|
if (!report.items.length) {
|
||||||
return null;
|
return null;
|
||||||
@ -34,7 +41,12 @@ export function StructuredReport({ report }: Props) {
|
|||||||
return (
|
return (
|
||||||
<Container>
|
<Container>
|
||||||
{errors.length ? (
|
{errors.length ? (
|
||||||
<StructuredReportItemList items={errors} color={ERROR_COLOR} icon={CloseCircleOutlined} />
|
<StructuredReportItemList
|
||||||
|
items={errors}
|
||||||
|
color={ERROR_COLOR}
|
||||||
|
icon={CloseCircleOutlined}
|
||||||
|
defaultActiveKey="0"
|
||||||
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
{warnings.length ? (
|
{warnings.length ? (
|
||||||
<StructuredReportItemList items={warnings} color={WARNING_COLOR} icon={ExclamationCircleOutlined} />
|
<StructuredReportItemList items={warnings} color={WARNING_COLOR} icon={ExclamationCircleOutlined} />
|
||||||
|
|||||||
@ -55,12 +55,13 @@ interface Props {
|
|||||||
item: StructuredReportLogEntry;
|
item: StructuredReportLogEntry;
|
||||||
color: string;
|
color: string;
|
||||||
icon?: React.ComponentType<any>;
|
icon?: React.ComponentType<any>;
|
||||||
|
defaultActiveKey?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function StructuredReportItem({ item, color, icon }: Props) {
|
export function StructuredReportItem({ item, color, icon, defaultActiveKey }: Props) {
|
||||||
const Icon = icon;
|
const Icon = icon;
|
||||||
return (
|
return (
|
||||||
<StyledCollapse color={color}>
|
<StyledCollapse color={color} defaultActiveKey={defaultActiveKey}>
|
||||||
<Collapse.Panel
|
<Collapse.Panel
|
||||||
header={
|
header={
|
||||||
<Item>
|
<Item>
|
||||||
|
|||||||
@ -16,9 +16,10 @@ interface Props {
|
|||||||
color: string;
|
color: string;
|
||||||
icon?: React.ComponentType<any>;
|
icon?: React.ComponentType<any>;
|
||||||
pageSize?: number;
|
pageSize?: number;
|
||||||
|
defaultActiveKey?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function StructuredReportItemList({ items, color, icon, pageSize = 3 }: Props) {
|
export function StructuredReportItemList({ items, color, icon, pageSize = 3, defaultActiveKey }: Props) {
|
||||||
const [visibleCount, setVisibleCount] = useState(pageSize);
|
const [visibleCount, setVisibleCount] = useState(pageSize);
|
||||||
const visibleItems = items.slice(0, visibleCount);
|
const visibleItems = items.slice(0, visibleCount);
|
||||||
const totalCount = items.length;
|
const totalCount = items.length;
|
||||||
@ -31,6 +32,7 @@ export function StructuredReportItemList({ items, color, icon, pageSize = 3 }: P
|
|||||||
item={item}
|
item={item}
|
||||||
color={color}
|
color={color}
|
||||||
icon={icon}
|
icon={icon}
|
||||||
|
defaultActiveKey={defaultActiveKey}
|
||||||
key={`${item.message}-${item.context}`}
|
key={`${item.message}-${item.context}`}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
|
|||||||
@ -20,3 +20,9 @@ export interface ExecutionCancelInfo {
|
|||||||
executionUrn: string;
|
executionUrn: string;
|
||||||
sourceUrn: string;
|
sourceUrn: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const enum TabType {
|
||||||
|
Summary = 'Summary',
|
||||||
|
Logs = 'Logs',
|
||||||
|
Recipe = 'Recipe',
|
||||||
|
}
|
||||||
|
|||||||
@ -1,59 +1,129 @@
|
|||||||
import { Button, Typography } from 'antd';
|
import { Button } from 'antd';
|
||||||
import React, { useState } from 'react';
|
import React, { useMemo, useState } from 'react';
|
||||||
import styled from 'styled-components';
|
import styled from 'styled-components';
|
||||||
|
|
||||||
import { EmbeddedListSearchModal } from '@app/entity/shared/components/styled/search/EmbeddedListSearchModal';
|
import { EmbeddedListSearchModal } from '@app/entity/shared/components/styled/search/EmbeddedListSearchModal';
|
||||||
import { ANTD_GRAY } from '@app/entity/shared/constants';
|
|
||||||
import {
|
import {
|
||||||
extractEntityTypeCountsFromFacets,
|
extractEntityTypeCountsFromFacets,
|
||||||
getEntitiesIngestedByType,
|
getEntitiesIngestedByType,
|
||||||
|
getIngestionContents,
|
||||||
|
getOtherIngestionContents,
|
||||||
getTotalEntitiesIngested,
|
getTotalEntitiesIngested,
|
||||||
} from '@app/ingestV2/source/utils';
|
} from '@app/ingestV2/source/utils';
|
||||||
import { UnionType } from '@app/search/utils/constants';
|
import { UnionType } from '@app/search/utils/constants';
|
||||||
import { Message } from '@app/shared/Message';
|
import { Message } from '@app/shared/Message';
|
||||||
import { formatNumber } from '@app/shared/formatNumber';
|
import { formatNumber } from '@app/shared/formatNumber';
|
||||||
|
import { capitalizeFirstLetterOnly } from '@app/shared/textUtil';
|
||||||
import { useEntityRegistry } from '@app/useEntityRegistry';
|
import { useEntityRegistry } from '@app/useEntityRegistry';
|
||||||
|
import { Heading, Pill, Text } from '@src/alchemy-components';
|
||||||
|
import colors from '@src/alchemy-components/theme/foundations/colors';
|
||||||
import { ExecutionRequestResult, Maybe } from '@src/types.generated';
|
import { ExecutionRequestResult, Maybe } from '@src/types.generated';
|
||||||
|
|
||||||
import { useGetSearchResultsForMultipleQuery } from '@graphql/search.generated';
|
import { useGetSearchResultsForMultipleQuery } from '@graphql/search.generated';
|
||||||
|
|
||||||
const HeaderContainer = styled.div`
|
// Base flex container with common spacing
|
||||||
|
const FlexContainer = styled.div`
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: space-between;
|
gap: 16px;
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const TitleContainer = styled.div``;
|
// Base card styling
|
||||||
|
const BaseCard = styled.div`
|
||||||
const TotalContainer = styled.div`
|
|
||||||
display: flex;
|
display: flex;
|
||||||
|
padding: 12px;
|
||||||
|
background-color: white;
|
||||||
|
border: 1px solid ${colors.gray[1400]};
|
||||||
|
border-radius: 12px;
|
||||||
|
box-shadow: 0px 4px 8px 0px rgba(33, 23, 95, 0.04);
|
||||||
|
min-height: 60px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const MainContainer = styled(FlexContainer)`
|
||||||
|
align-items: stretch;
|
||||||
|
margin-top: 16px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const CardContainer = styled(BaseCard)`
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
justify-content: right;
|
justify-content: center;
|
||||||
align-items: end;
|
align-items: flex-start;
|
||||||
|
flex: 1 0 0;
|
||||||
|
min-height: 80px;
|
||||||
|
max-height: 80px;
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const TotalText = styled(Typography.Text)`
|
const TotalContainer = styled(BaseCard)`
|
||||||
font-size: 16px;
|
flex-direction: row;
|
||||||
color: ${ANTD_GRAY[8]};
|
|
||||||
`;
|
|
||||||
|
|
||||||
const EntityCountsContainer = styled.div`
|
|
||||||
display: flex;
|
|
||||||
justify-content: left;
|
|
||||||
align-items: center;
|
align-items: center;
|
||||||
max-width: 100%;
|
justify-content: space-between;
|
||||||
flex-wrap: wrap;
|
flex: 1 0 0;
|
||||||
|
min-height: 80px;
|
||||||
|
max-height: 80px;
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const EntityCount = styled.div`
|
const TotalInfo = styled.div`
|
||||||
margin-right: 40px;
|
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
align-items: flex-start;
|
align-items: flex-start;
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const ViewAllButton = styled(Button)`
|
const TypesSection = styled.div`
|
||||||
padding: 0px;
|
flex: 1;
|
||||||
margin-top: 4px;
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: stretch;
|
||||||
|
width: 100%;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const IngestionBoxesContainer = styled(FlexContainer)`
|
||||||
|
width: 100%;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const EntityCountsContainer = styled(FlexContainer)`
|
||||||
|
flex: 1;
|
||||||
|
width: 100%;
|
||||||
|
align-items: stretch;
|
||||||
|
justify-content: flex-start;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const TypesHeaderContainer = styled.div`
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 4px;
|
||||||
|
margin-top: 16px;
|
||||||
|
position: relative;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const TypesHeader = styled(Text)`
|
||||||
|
position: absolute;
|
||||||
|
top: 0;
|
||||||
|
left: calc(50% + 33px);
|
||||||
|
margin-bottom: 0;
|
||||||
|
z-index: 1;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const VerticalDivider = styled.div`
|
||||||
|
width: 2px;
|
||||||
|
background-color: ${colors.gray[1400]};
|
||||||
|
height: 80px;
|
||||||
|
align-self: center;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const IngestionContentsContainer = styled.div`
|
||||||
|
margin-top: 20px;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const IngestionBoxTopRow = styled.div`
|
||||||
|
display: flex;
|
||||||
|
flex-direction: row;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: flex-start;
|
||||||
|
width: 100%;
|
||||||
|
`;
|
||||||
|
|
||||||
|
const IngestionRowCount = styled(Text)`
|
||||||
|
margin-right: 10px;
|
||||||
`;
|
`;
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
@ -64,6 +134,42 @@ type Props = {
|
|||||||
const ENTITY_FACET_NAME = 'entity';
|
const ENTITY_FACET_NAME = 'entity';
|
||||||
const TYPE_NAMES_FACET_NAME = 'typeNames';
|
const TYPE_NAMES_FACET_NAME = 'typeNames';
|
||||||
|
|
||||||
|
type IngestionContentItem = {
|
||||||
|
title?: string;
|
||||||
|
type?: string;
|
||||||
|
count: number;
|
||||||
|
percent: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type RenderIngestionContentsProps = {
|
||||||
|
items: IngestionContentItem[];
|
||||||
|
getKey: (item: IngestionContentItem) => string;
|
||||||
|
getLabel: (item: IngestionContentItem) => string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const IngestionContents: React.FC<RenderIngestionContentsProps> = ({ items, getKey, getLabel }) => (
|
||||||
|
<IngestionBoxesContainer>
|
||||||
|
{items.map((item) => (
|
||||||
|
<CardContainer key={getKey(item)}>
|
||||||
|
<IngestionBoxTopRow>
|
||||||
|
<IngestionRowCount size="xl" weight="bold" color="gray" colorLevel={800}>
|
||||||
|
{formatNumber(item.count)}
|
||||||
|
</IngestionRowCount>
|
||||||
|
<Pill
|
||||||
|
size="sm"
|
||||||
|
variant="filled"
|
||||||
|
color="gray"
|
||||||
|
label={item.count === 0 ? 'Missing' : `${item.percent} of Total`}
|
||||||
|
/>
|
||||||
|
</IngestionBoxTopRow>
|
||||||
|
<Text size="md" color="gray" colorLevel={600}>
|
||||||
|
{getLabel(item)}
|
||||||
|
</Text>
|
||||||
|
</CardContainer>
|
||||||
|
))}
|
||||||
|
</IngestionBoxesContainer>
|
||||||
|
);
|
||||||
|
|
||||||
export default function IngestedAssets({ id, executionResult }: Props) {
|
export default function IngestedAssets({ id, executionResult }: Props) {
|
||||||
const entityRegistry = useEntityRegistry();
|
const entityRegistry = useEntityRegistry();
|
||||||
|
|
||||||
@ -113,44 +219,99 @@ export default function IngestedAssets({ id, executionResult }: Props) {
|
|||||||
// The total number of assets ingested
|
// The total number of assets ingested
|
||||||
const total = totalEntitiesIngested ?? data?.searchAcrossEntities?.total ?? 0;
|
const total = totalEntitiesIngested ?? data?.searchAcrossEntities?.total ?? 0;
|
||||||
|
|
||||||
|
const ingestionContents = useMemo(
|
||||||
|
() => executionResult && getIngestionContents(executionResult),
|
||||||
|
[executionResult],
|
||||||
|
);
|
||||||
|
const otherIngestionContents = useMemo(
|
||||||
|
() => executionResult && getOtherIngestionContents(executionResult),
|
||||||
|
[executionResult],
|
||||||
|
);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
{error && <Message type="error" content="" />}
|
{error && <Message type="error" content="" />}
|
||||||
<HeaderContainer>
|
<Heading type="h4" size="lg" weight="bold">
|
||||||
<TitleContainer>
|
Assets
|
||||||
<Typography.Title level={5}>Ingested Assets</Typography.Title>
|
</Heading>
|
||||||
{(loading && <Typography.Text type="secondary">Loading...</Typography.Text>) || (
|
{loading && (
|
||||||
|
<Text color="gray" colorLevel={600}>
|
||||||
|
Loading...
|
||||||
|
</Text>
|
||||||
|
)}
|
||||||
|
{!loading && total === 0 && <Text>No assets were ingested.</Text>}
|
||||||
|
{!loading && total > 0 && (
|
||||||
<>
|
<>
|
||||||
{(total > 0 && (
|
<TypesHeaderContainer>
|
||||||
<Typography.Paragraph type="secondary">
|
<Text color="gray" colorLevel={600}>
|
||||||
The following asset types were ingested during this run.
|
Types and counts for this ingestion run.
|
||||||
</Typography.Paragraph>
|
</Text>
|
||||||
)) || <Typography.Text>No assets were ingested.</Typography.Text>}
|
<TypesHeader size="sm" color="gray" colorLevel={600} weight="bold">
|
||||||
</>
|
Types
|
||||||
)}
|
</TypesHeader>
|
||||||
</TitleContainer>
|
</TypesHeaderContainer>
|
||||||
{!loading && (
|
<MainContainer>
|
||||||
<TotalContainer>
|
<TotalContainer>
|
||||||
<Typography.Text type="secondary">Total</Typography.Text>
|
<TotalInfo>
|
||||||
<TotalText style={{ fontSize: 16, color: ANTD_GRAY[8] }}>
|
<Text size="xl" weight="bold" color="gray" colorLevel={800}>
|
||||||
<b>{formatNumber(total)}</b> assets
|
{formatNumber(total)}
|
||||||
</TotalText>
|
</Text>
|
||||||
|
<Text size="md" color="gray" colorLevel={600} style={{ marginTop: 2 }}>
|
||||||
|
Total Assets Ingested
|
||||||
|
</Text>
|
||||||
|
</TotalInfo>
|
||||||
|
<Button type="link" onClick={() => setShowAssetSearch(true)}>
|
||||||
|
View All
|
||||||
|
</Button>
|
||||||
</TotalContainer>
|
</TotalContainer>
|
||||||
)}
|
<VerticalDivider />
|
||||||
</HeaderContainer>
|
<TypesSection>
|
||||||
<EntityCountsContainer>
|
<EntityCountsContainer>
|
||||||
{countsByEntityType.map((entityCount) => (
|
{countsByEntityType.map((entityCount) => (
|
||||||
<EntityCount>
|
<CardContainer key={entityCount.displayName}>
|
||||||
<Typography.Text style={{ paddingLeft: 2, fontSize: 18, color: ANTD_GRAY[8] }}>
|
<Text size="xl" weight="bold" color="gray" colorLevel={800}>
|
||||||
<b>{formatNumber(entityCount.count)}</b>
|
{formatNumber(entityCount.count)}
|
||||||
</Typography.Text>
|
</Text>
|
||||||
<Typography.Text type="secondary">{entityCount.displayName}</Typography.Text>
|
<Text size="md" color="gray" colorLevel={600}>
|
||||||
</EntityCount>
|
{capitalizeFirstLetterOnly(entityCount.displayName)}
|
||||||
|
</Text>
|
||||||
|
</CardContainer>
|
||||||
))}
|
))}
|
||||||
</EntityCountsContainer>
|
</EntityCountsContainer>
|
||||||
<ViewAllButton type="link" onClick={() => setShowAssetSearch(true)}>
|
</TypesSection>
|
||||||
View All
|
</MainContainer>
|
||||||
</ViewAllButton>
|
{ingestionContents && (
|
||||||
|
<IngestionContentsContainer>
|
||||||
|
<Heading type="h5" size="lg" weight="bold">
|
||||||
|
Coverage
|
||||||
|
</Heading>
|
||||||
|
<Text color="gray" colorLevel={600}>
|
||||||
|
Additional metadata collected during this ingestion run.
|
||||||
|
</Text>
|
||||||
|
<Text weight="semiBold" size="md">
|
||||||
|
Lineage
|
||||||
|
</Text>
|
||||||
|
<IngestionContents
|
||||||
|
items={ingestionContents}
|
||||||
|
getKey={(item) => item.title || ''}
|
||||||
|
getLabel={(item) => item.title || ''}
|
||||||
|
/>
|
||||||
|
{otherIngestionContents && (
|
||||||
|
<>
|
||||||
|
<Text weight="semiBold" size="md">
|
||||||
|
Statistics
|
||||||
|
</Text>
|
||||||
|
<IngestionContents
|
||||||
|
items={otherIngestionContents}
|
||||||
|
getKey={(item) => item.type || ''}
|
||||||
|
getLabel={(item) => item.type || ''}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</IngestionContentsContainer>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
{showAssetSearch && (
|
{showAssetSearch && (
|
||||||
<EmbeddedListSearchModal
|
<EmbeddedListSearchModal
|
||||||
title="View Ingested Assets"
|
title="View Ingested Assets"
|
||||||
|
|||||||
@ -15,6 +15,8 @@ import {
|
|||||||
capitalizeMonthsAndDays,
|
capitalizeMonthsAndDays,
|
||||||
formatTimezone,
|
formatTimezone,
|
||||||
getEntitiesIngestedByType,
|
getEntitiesIngestedByType,
|
||||||
|
getIngestionContents,
|
||||||
|
getOtherIngestionContents,
|
||||||
getSortInput,
|
getSortInput,
|
||||||
getSourceStatus,
|
getSourceStatus,
|
||||||
getTotalEntitiesIngested,
|
getTotalEntitiesIngested,
|
||||||
@ -412,6 +414,430 @@ describe('getSourceStatus', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('getIngestionContents', () => {
|
||||||
|
test('returns null when structured report is not available', () => {
|
||||||
|
const result = getIngestionContents({} as Partial<ExecutionRequestResult>);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('returns null when aspects_by_subtypes is empty', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('processes dataset subtypes with lineage information correctly', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
container: {
|
||||||
|
containerProperties: 156,
|
||||||
|
container: 117,
|
||||||
|
},
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 10,
|
||||||
|
upstreamLineage: 5,
|
||||||
|
datasetProfile: 10,
|
||||||
|
},
|
||||||
|
View: {
|
||||||
|
status: 20,
|
||||||
|
upstreamLineage: 10,
|
||||||
|
datasetProfile: 20,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
title: 'Table',
|
||||||
|
count: 10,
|
||||||
|
percent: '50%',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'View',
|
||||||
|
count: 20,
|
||||||
|
percent: '50%',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('filters out subtypes with 0% lineage', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 10,
|
||||||
|
upstreamLineage: 0,
|
||||||
|
},
|
||||||
|
View: {
|
||||||
|
status: 20,
|
||||||
|
upstreamLineage: 5,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
title: 'View',
|
||||||
|
count: 20,
|
||||||
|
percent: '25%',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('filters out subtypes with status count of 0', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 0,
|
||||||
|
upstreamLineage: 5,
|
||||||
|
},
|
||||||
|
View: {
|
||||||
|
status: 20,
|
||||||
|
upstreamLineage: 10,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
title: 'View',
|
||||||
|
count: 20,
|
||||||
|
percent: '50%',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('handles missing upstreamLineage property', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 10,
|
||||||
|
// upstreamLineage is missing
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('handles missing status property', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
// status is missing
|
||||||
|
upstreamLineage: 5,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('calculates percentage correctly and rounds to nearest integer', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 7,
|
||||||
|
upstreamLineage: 2, // 2/7 = 28.57...% rounds to 29%
|
||||||
|
},
|
||||||
|
View: {
|
||||||
|
status: 3,
|
||||||
|
upstreamLineage: 1, // 1/3 = 33.33...% rounds to 33%
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
title: 'Table',
|
||||||
|
count: 7,
|
||||||
|
percent: '29%',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'View',
|
||||||
|
count: 3,
|
||||||
|
percent: '33%',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getOtherIngestionContents', () => {
|
||||||
|
test('returns null when structured report is not available', () => {
|
||||||
|
const result = getOtherIngestionContents({} as Partial<ExecutionRequestResult>);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('returns null when aspects_by_subtypes is empty', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('processes multiple dataset subtypes and aggregates profiling and usage entries', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 10,
|
||||||
|
datasetProfile: 5,
|
||||||
|
datasetUsageStatistics: 3,
|
||||||
|
},
|
||||||
|
View: {
|
||||||
|
status: 20,
|
||||||
|
datasetProfile: 8,
|
||||||
|
datasetUsageStatistics: 12,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
type: 'Profiling',
|
||||||
|
count: 13, // 5 + 8
|
||||||
|
percent: '43%', // (13 / 30) * 100 = 43.33...% rounds to 43%
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'Usage',
|
||||||
|
count: 15, // 3 + 12
|
||||||
|
percent: '50%', // (15 / 30) * 100 = 50%
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('filters out subtypes with zero profiling and usage counts', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 10,
|
||||||
|
datasetProfile: 0,
|
||||||
|
datasetUsageStatistics: 0,
|
||||||
|
},
|
||||||
|
View: {
|
||||||
|
status: 20,
|
||||||
|
datasetProfile: 8,
|
||||||
|
datasetUsageStatistics: 12,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
type: 'Profiling',
|
||||||
|
count: 8,
|
||||||
|
percent: '27%', // (8 / 30) * 100 = 26.66...% rounds to 27%
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'Usage',
|
||||||
|
count: 12,
|
||||||
|
percent: '40%', // (12 / 30) * 100 = 40%
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('filters out subtypes with zero status count', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 0,
|
||||||
|
datasetProfile: 5,
|
||||||
|
datasetUsageStatistics: 3,
|
||||||
|
},
|
||||||
|
View: {
|
||||||
|
status: 20,
|
||||||
|
datasetProfile: 8,
|
||||||
|
datasetUsageStatistics: 12,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
type: 'Profiling',
|
||||||
|
count: 8,
|
||||||
|
percent: '40%', // (8 / 20) * 100 = 40%
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'Usage',
|
||||||
|
count: 12,
|
||||||
|
percent: '60%', // (12 / 20) * 100 = 60%
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('handles missing datasetProfile and datasetUsageStatistics properties', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 10,
|
||||||
|
// datasetProfile and datasetUsageStatistics are missing
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
count: 0,
|
||||||
|
percent: '0%',
|
||||||
|
type: 'Usage',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ignores non-dataset entity types', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
container: {
|
||||||
|
Container: {
|
||||||
|
status: 10,
|
||||||
|
datasetProfile: 5,
|
||||||
|
datasetUsageStatistics: 3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 20,
|
||||||
|
datasetProfile: 8,
|
||||||
|
datasetUsageStatistics: 12,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
type: 'Profiling',
|
||||||
|
count: 8,
|
||||||
|
percent: '40%', // (8 / 20) * 100 = 40%
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'Usage',
|
||||||
|
count: 12,
|
||||||
|
percent: '60%', // (12 / 20) * 100 = 60%
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('calculates percentage correctly and rounds to nearest integer', () => {
|
||||||
|
const structuredReport = {
|
||||||
|
source: {
|
||||||
|
report: {
|
||||||
|
aspects_by_subtypes: {
|
||||||
|
dataset: {
|
||||||
|
Table: {
|
||||||
|
status: 7,
|
||||||
|
datasetProfile: 2, // 2/7 = 28.57...% rounds to 29%
|
||||||
|
datasetUsageStatistics: 1, // 1/7 = 14.28...% rounds to 14%
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = getOtherIngestionContents(mockExecutionRequestResult(structuredReport));
|
||||||
|
expect(result).toEqual([
|
||||||
|
{
|
||||||
|
type: 'Profiling',
|
||||||
|
count: 2,
|
||||||
|
percent: '29%',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'Usage',
|
||||||
|
count: 1,
|
||||||
|
percent: '14%',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('buildOwnerEntities', () => {
|
describe('buildOwnerEntities', () => {
|
||||||
const entityUrn = 'urn:li:entity:123';
|
const entityUrn = 'urn:li:entity:123';
|
||||||
const ownerUrn = 'urn:li:user:123';
|
const ownerUrn = 'urn:li:user:123';
|
||||||
|
|||||||
@ -303,6 +303,118 @@ export const getTotalEntitiesIngested = (result: Partial<ExecutionRequestResult>
|
|||||||
return entityTypeCounts.reduce((total, entityType) => total + entityType.count, 0);
|
return entityTypeCounts.reduce((total, entityType) => total + entityType.count, 0);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const getOtherIngestionContents = (executionResult: Partial<ExecutionRequestResult>) => {
|
||||||
|
const structuredReportObject = extractStructuredReportPOJO(executionResult);
|
||||||
|
if (!structuredReportObject) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const aspectsBySubtypes = structuredReportObject?.source?.report?.aspects_by_subtypes;
|
||||||
|
|
||||||
|
if (!aspectsBySubtypes || Object.keys(aspectsBySubtypes).length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
let totalStatusCount = 0;
|
||||||
|
let totalDatasetProfileCount = 0;
|
||||||
|
let totalDatasetUsageStatisticsCount = 0;
|
||||||
|
|
||||||
|
Object.entries(aspectsBySubtypes).forEach(([entityType, subtypes]) => {
|
||||||
|
if (entityType !== 'dataset') {
|
||||||
|
// temporary for now - we have not decided on the design for non dataset entity types
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Object.entries(subtypes as Record<string, any>).forEach(([_, aspects]) => {
|
||||||
|
const statusCount = (aspects as any)?.status || 0;
|
||||||
|
if (statusCount === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const dataSetProfileCount = (aspects as any)?.datasetProfile || 0;
|
||||||
|
const dataSetUsageStatisticsCount = (aspects as any)?.datasetUsageStatistics || 0;
|
||||||
|
|
||||||
|
totalStatusCount += statusCount;
|
||||||
|
totalDatasetProfileCount += dataSetProfileCount;
|
||||||
|
totalDatasetUsageStatisticsCount += dataSetUsageStatisticsCount;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
if (totalStatusCount === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result: Array<{ type: string; count: number; percent: string }> = [];
|
||||||
|
|
||||||
|
if (totalDatasetProfileCount > 0) {
|
||||||
|
const datasetProfilePercent = `${((totalDatasetProfileCount / totalStatusCount) * 100).toFixed(0)}%`;
|
||||||
|
result.push({
|
||||||
|
type: 'Profiling',
|
||||||
|
count: totalDatasetProfileCount,
|
||||||
|
percent: datasetProfilePercent,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (totalDatasetUsageStatisticsCount > 0) {
|
||||||
|
const datasetUsageStatisticsPercent = `${((totalDatasetUsageStatisticsCount / totalStatusCount) * 100).toFixed(0)}%`;
|
||||||
|
result.push({
|
||||||
|
type: 'Usage',
|
||||||
|
count: totalDatasetUsageStatisticsCount,
|
||||||
|
percent: datasetUsageStatisticsPercent,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
result.push({
|
||||||
|
type: 'Usage',
|
||||||
|
count: 0,
|
||||||
|
percent: '0%',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getIngestionContents = (executionResult: Partial<ExecutionRequestResult>) => {
|
||||||
|
const structuredReportObject = extractStructuredReportPOJO(executionResult);
|
||||||
|
if (!structuredReportObject) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const aspectsBySubtypes = structuredReportObject.source.report.aspects_by_subtypes;
|
||||||
|
|
||||||
|
if (!aspectsBySubtypes || Object.keys(aspectsBySubtypes).length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result: Array<{ title: string; count: number; percent: string }> = [];
|
||||||
|
Object.entries(aspectsBySubtypes).forEach(([entityType, subtypes]) => {
|
||||||
|
if (entityType !== 'dataset') {
|
||||||
|
// temporary for now - we have not decided on the design for non dataset entity types
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Object.entries(subtypes as Record<string, any>).forEach(([subtype, aspects]) => {
|
||||||
|
const statusCount = (aspects as any)?.status || 0;
|
||||||
|
const upstreamLineage = (aspects as any)?.upstreamLineage || 0;
|
||||||
|
if (statusCount === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const percent = `${((upstreamLineage / statusCount) * 100).toFixed(0)}%`;
|
||||||
|
if (percent === '0%') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
result.push({
|
||||||
|
title: subtype,
|
||||||
|
count: statusCount,
|
||||||
|
percent,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
if (result.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
export const getIngestionSourceStatus = (result?: Partial<ExecutionRequestResult> | null) => {
|
export const getIngestionSourceStatus = (result?: Partial<ExecutionRequestResult> | null) => {
|
||||||
if (!result) {
|
if (!result) {
|
||||||
return undefined;
|
return undefined;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user