yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-27 02:16:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Minor: lineage data quality failure tracing support (#18580)

Browse Source
This commit is contained in:
Shailesh Parmar 2024-11-11 15:21:29 +05:30 committed by GitHub
parent a6d97b67a8
commit eeb27c3cbf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 192 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/DataQuality.interface.ts
View File

@ -1,4 +1,5 @@
import { DateRangeObject } from 'Models';
import { LinkProps } from 'react-router-dom';
import { TestCaseStatus } from '../../generated/tests/testCase';
import { TestCaseResolutionStatusTypes } from '../../generated/tests/testCaseResolutionStatus';
import { TestPlatform } from '../../generated/tests/testDefinition';
@ -76,7 +77,7 @@ export interface DataStatisticWidgetProps {
icon: SvgComponent;
dataLabel: string;
countValue: number;
redirectPath: string;
redirectPath: LinkProps['to'];
linkLabel: string;
isLoading?: boolean;
}

10
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx
View File

@ -19,6 +19,7 @@ import { EdgeProps, getBezierPath } from 'reactflow';
import { ReactComponent as FunctionIcon } from '../../../assets/svg/ic-function.svg';
import { ReactComponent as IconTimesCircle } from '../../../assets/svg/ic-times-circle.svg';
import { ReactComponent as PipelineIcon } from '../../../assets/svg/pipeline-grey.svg';
import { RED_3 } from '../../../constants/Color.constants';
import { FOREIGN_OBJECT_SIZE } from '../../../constants/Lineage.constants';
import { useLineageProvider } from '../../../context/LineageProvider/LineageProvider';
import { EntityType } from '../../../enums/entity.enum';
@ -76,6 +77,7 @@ export const CustomEdge = ({
isPipelineRootNode,
...rest
} = data;
const offset = 4;
const { fromEntity, toEntity, pipeline, pipelineEntityType } =
@ -150,10 +152,16 @@ export const CustomEdge = ({
opacity = tracedNodes.length === 0 || isStrokeNeeded ? 1 : 0.25;
}
let stroke = isStrokeNeeded ? theme.primaryColor : undefined;
if (edge?.isDqTestFailure) {
stroke = RED_3;
}
return {
...style,
...{
stroke: isStrokeNeeded ? theme.primaryColor : undefined,
stroke,
opacity,
},
};

2
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx
View File

@ -49,6 +49,7 @@ const CustomNodeV1 = (props: NodeProps) => {
} = useLineageProvider();
const { label, isNewNode, node = {}, isRootNode } = data;
const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type;
const isSelected = selectedNode === node;
const { id, lineage, fullyQualifiedName } = node;
@ -257,6 +258,7 @@ const CustomNodeV1 = (props: NodeProps) => {
className={classNames(
'lineage-node p-0',
isSelected ? 'custom-node-header-active' : 'custom-node-header-normal',
{ 'data-quality-failed-custom-node-header': node?.isDqTestFailure },
{ 'custom-node-header-tracing': isTraced }
)}
data-testid={`lineage-node-${fullyQualifiedName}`}>

44
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less
View File

@ -154,6 +154,41 @@
}
}
.react-flow__node-default.selectable.selected,
.react-flow__node-default.selectable:focus,
.react-flow__node-default.selectable:focus-visible,
.react-flow__node-input.selectable.selected,
.react-flow__node-input.selectable:focus,
.react-flow__node-input.selectable:focus-visible,
.react-flow__node-output.selectable.selected,
.react-flow__node-output.selectable:focus,
.react-flow__node-output.selectable:focus-visible,
.react-flow__node-group.selectable.selected,
.react-flow__node-group.selectable:focus,
.react-flow__node-group.selectable:focus-visible,
.react-flow__node-load-more.selectable.selected,
.react-flow__node-load-more.selectable:focus,
.react-flow__node-load-more.selectable:focus-visible {
.data-quality-failed-custom-node-header {
&.lineage-node {
border-color: @red-3 !important;
}
.lineage-node-handle {
border-color: @red-3;
svg {
color: @red-3;
}
}
.label-container {
background: fade(@red-3, 10%);
}
.column-container {
background: fade(@red-3, 10%);
border-top: 1px solid @border-color;
}
}
}
.custom-node-header-active {
.label-container {
background: @primary-1;
@ -163,6 +198,15 @@
border-top: 1px solid @border-color;
}
}
.data-quality-failed-custom-node-header.custom-node-header-active {
.label-container {
background: fade(@red-3, 10%);
}
.column-container {
background: fade(@red-3, 10%);
border-top: 1px solid @red-3;
}
}
.react-flow__handle.connectable {
background-color: @text-color;

19
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less
View File

@ -44,6 +44,25 @@
}
}
}
.data-quality-failed-custom-node-header {
&.custom-node-header-normal {
border: 1px solid @red-3;
}
.react-flow__handle {
border-color: @red-3;
}
.lineage-node-handle {
border: 1px solid @red-3;
svg {
color: @red-3;
}
}
&.custom-node-header-active {
border-color: @red-3;
}
}
.custom-node-header-active {
border-color: @primary-color;
}

1
openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts
View File

@ -52,6 +52,7 @@ export interface EdgeDetails {
columns?: ColumnLineage[];
description?: string;
pipelineEntityType?: EntityType.PIPELINE | EntityType.STORED_PROCEDURE;
doc_id?: string;
}
export type LineageSourceType = Omit<SourceType, 'service'> & {

44
openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx
View File

@ -15,7 +15,10 @@ import React, { useEffect } from 'react';
import { Edge } from 'reactflow';
import { EdgeTypeEnum } from '../../components/Entity/EntityLineage/EntityLineage.interface';
import { EntityType } from '../../enums/entity.enum';
import { getLineageDataByFQN } from '../../rest/lineageAPI';
import {
getDataQualityLineage,
getLineageDataByFQN,
} from '../../rest/lineageAPI';
import LineageProvider, { useLineageProvider } from './LineageProvider';
const mockLocation = {
@ -128,9 +131,16 @@ jest.mock(
});
}
);
let mockIsAlertSupported = false;
jest.mock('../../utils/TableClassBase', () => ({
getAlertEnableStatus: jest
.fn()
.mockImplementation(() => mockIsAlertSupported),
}));
jest.mock('../../rest/lineageAPI', () => ({
getLineageDataByFQN: jest.fn(),
getDataQualityLineage: jest.fn(),
}));
describe('LineageProvider', () => {
@ -148,6 +158,38 @@ describe('LineageProvider', () => {
});
expect(getLineageDataByFQN).toHaveBeenCalled();
expect(getDataQualityLineage).not.toHaveBeenCalled();
});
it('getDataQualityLineage should be called if alert is supported', async () => {
mockIsAlertSupported = true;
(getLineageDataByFQN as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
nodes: [],
edges: [],
})
);
await act(async () => {
render(
<LineageProvider>
<DummyChildrenComponent />
</LineageProvider>
);
});
expect(getLineageDataByFQN).toHaveBeenCalledWith(
'table1',
'table',
{ downstreamDepth: 1, nodesPerLayer: 50, upstreamDepth: 1 },
''
);
expect(getDataQualityLineage).toHaveBeenCalledWith(
'table1',
{ downstreamDepth: 1, nodesPerLayer: 50, upstreamDepth: 1 },
''
);
mockIsAlertSupported = false;
});
it('should call loadChildNodesHandler', async () => {

75
openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx
View File

@ -14,8 +14,9 @@ import Icon from '@ant-design/icons/lib/components/Icon';
import { Button, Modal } from 'antd';
import { AxiosError } from 'axios';
import classNames from 'classnames';
import { isEqual, isUndefined, uniqueId, uniqWith } from 'lodash';
import { isEqual, isUndefined, uniq, uniqueId, uniqWith } from 'lodash';
import { LoadingState } from 'Models';
import QueryString from 'qs';
import React, {
createContext,
DragEvent,
@ -76,8 +77,13 @@ import {
LineageDetails,
} from '../../generated/type/entityLineage';
import { useApplicationStore } from '../../hooks/useApplicationStore';
import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation';
import { useFqn } from '../../hooks/useFqn';
import { getLineageDataByFQN, updateLineageEdge } from '../../rest/lineageAPI';
import {
getDataQualityLineage,
getLineageDataByFQN,
updateLineageEdge,
} from '../../rest/lineageAPI';
import {
addLineageHandler,
centerNodePosition,
@ -104,6 +110,7 @@ import {
removeLineageHandler,
} from '../../utils/EntityLineageUtils';
import { getEntityReferenceFromEntity } from '../../utils/EntityUtils';
import tableClassBase from '../../utils/TableClassBase';
import { showErrorToast } from '../../utils/ToastUtils';
import { useTourProvider } from '../TourProvider/TourProvider';
import {
@ -117,6 +124,7 @@ export const LineageContext = createContext({} as LineageContextType);
const LineageProvider = ({ children }: LineageProviderProps) => {
const { t } = useTranslation();
const { fqn: decodedFqn } = useFqn();
const location = useCustomLocation();
const { isTourOpen, isTourPage } = useTourProvider();
const { appPreferences } = useApplicationStore();
const defaultLineageConfig = appPreferences?.lineageConfig as LineageSettings;
@ -180,6 +188,15 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
const [paginationData, setPaginationData] = useState({});
const { showModal } = useEntityExportModalProvider();
const lineageLayer = useMemo(() => {
const param = location.search;
const searchData = QueryString.parse(
param.startsWith('?') ? param.substring(1) : param
);
return searchData.layers as LineageLayer[] | undefined;
}, [location.search]);
const initLineageChildMaps = useCallback(
(
lineageData: EntityLineageResponse,
@ -220,25 +237,63 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
config,
queryFilter
);
const dqLineageResp =
entityType === EntityType.TABLE &&
tableClassBase.getAlertEnableStatus()
? await getDataQualityLineage(fqn, config, queryFilter)
: { nodes: [], edges: [] };
if (res) {
const { nodes = [], entity } = res;
const { nodes = [], entity, edges } = res;
const allNodes = uniqWith(
[...nodes, entity].filter(Boolean),
isEqual
);
).map((node) => {
return {
...node,
isDqTestFailure:
dqLineageResp.nodes?.some((dqNode) => dqNode.id === node.id) ??
false,
};
});
const updatedEntity = {
...entity,
isDqTestFailure:
dqLineageResp.nodes?.some((dqNode) => dqNode.id === entity.id) ??
false,
};
const updatedEdges = edges?.map((edge) => {
return {
...edge,
isDqTestFailure:
dqLineageResp.edges?.some(
(dqEdge) => dqEdge?.doc_id === edge?.doc_id
) ?? false,
};
});
if (
entityType !== EntityType.PIPELINE &&
entityType !== EntityType.STORED_PROCEDURE
) {
const { map: childMapObj } = getChildMap(
{ ...res, nodes: allNodes },
{
...res,
nodes: allNodes,
edges: updatedEdges,
entity: updatedEntity,
},
decodedFqn
);
setChildMap(childMapObj);
const { nodes: newNodes, edges: newEdges } = getPaginatedChildMap(
{
...res,
entity: updatedEntity,
edges: updatedEdges,
nodes: allNodes,
},
childMapObj,
@ -248,12 +303,14 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
setEntityLineage({
...res,
entity: updatedEntity,
nodes: newNodes,
edges: [...(res.edges ?? []), ...newEdges],
edges: [...(updatedEdges ?? []), ...newEdges],
});
} else {
setEntityLineage({
...res,
entity: updatedEntity,
nodes: allNodes,
});
}
@ -1308,6 +1365,12 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
}
}, [isTourOpen, isTourPage]);
useEffect(() => {
if (lineageLayer) {
setActiveLayer((pre) => uniq([...lineageLayer, ...pre]));
}
}, [lineageLayer]);
return (
<LineageContext.Provider value={activityFeedContextValues}>
<div

4
openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts
View File

@ -48,7 +48,8 @@ export const getLineageDataByFQN = async (
export const getDataQualityLineage = async (
fqn: string,
config?: Partial<LineageConfig>
config?: Partial<LineageConfig>,
queryFilter?: string
) => {
const { upstreamDepth = 1 } = config ?? {};
const response = await APIClient.get<EntityLineageResponse>(
@ -58,6 +59,7 @@ export const getDataQualityLineage = async (
fqn,
upstreamDepth,
includeDeleted: false,
query_filter: queryFilter,
},
}
);