From 35ba0738d70e2481af49ffadb84908c85a1283d3 Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Tue, 29 Apr 2025 19:53:54 +0530 Subject: [PATCH] enhancement: improve DQ failure lineage tracing (#21022) * enhancement: improve DQ failure lineage tracing * Enhancement: Update lineage component to conditionally render controls and improve platform view handling in lineage provider. Added translations for platform-type lineage in multiple languages. * Refactor LineageProvider to conditionally update history based on platform lineage and set default platform view in PlatformLineage component. * minor fix --- .../EntityLineage/CustomEdge.component.tsx | 18 ++++--- .../components/Lineage/Lineage.component.tsx | 4 +- .../LineageProvider.interface.tsx | 1 + .../LineageProvider/LineageProvider.tsx | 51 ++++++++++++++++--- .../ui/src/locale/languages/de-de.json | 1 + .../ui/src/locale/languages/en-us.json | 1 + .../ui/src/locale/languages/es-es.json | 1 + .../ui/src/locale/languages/fr-fr.json | 1 + .../ui/src/locale/languages/gl-es.json | 1 + .../ui/src/locale/languages/he-he.json | 1 + .../ui/src/locale/languages/ja-jp.json | 1 + .../ui/src/locale/languages/ko-kr.json | 1 + .../ui/src/locale/languages/mr-in.json | 1 + .../ui/src/locale/languages/nl-nl.json | 1 + .../ui/src/locale/languages/pr-pr.json | 1 + .../ui/src/locale/languages/pt-br.json | 1 + .../ui/src/locale/languages/pt-pt.json | 1 + .../ui/src/locale/languages/ru-ru.json | 1 + .../ui/src/locale/languages/th-th.json | 1 + .../ui/src/locale/languages/zh-cn.json | 1 + .../pages/PlatformLineage/PlatformLineage.tsx | 38 +++++++++----- .../ui/src/utils/EntityLineageUtils.tsx | 37 ++++++++++++++ 22 files changed, 136 insertions(+), 29 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx index 2ec1a5efbee..81e86958a79 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx @@ -95,6 +95,7 @@ export const CustomEdge = ({ onAddPipelineClick, onColumnEdgeRemove, dataQualityLineage, + dqHighlightedEdges, } = useLineageProvider(); const { theme } = useApplicationStore(); @@ -124,14 +125,15 @@ export const CustomEdge = ({ // Compute if should show DQ tracing const showDqTracing = useMemo(() => { - return ( - (activeLayer.includes(LineageLayer.DataObservability) && - dataQualityLineage?.edges?.some( - (dqEdge) => dqEdge?.docId === edge?.docId - )) ?? - false - ); - }, [activeLayer, dataQualityLineage?.edges, edge?.docId]); + if ( + !activeLayer.includes(LineageLayer.DataObservability) || + !dataQualityLineage?.nodes + ) { + return false; + } + + return dqHighlightedEdges?.has(id); + }, [activeLayer, dataQualityLineage?.nodes, id, dqHighlightedEdges]); // Determine if column is highlighted based on traced columns const isColumnHighlighted = useMemo(() => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx index 91ae5cd264f..966e8efbffc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx @@ -164,7 +164,9 @@ const Lineage = ({ ref={reactFlowWrapper}> {entityLineage && ( <> - + {isPlatformLineage ? null : ( + + )} void; onUpdateLayerView: (layers: LineageLayer[]) => void; redraw: () => Promise; + dqHighlightedEdges?: Set; } diff --git a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx index 3bbddfb9343..fbec4fc728d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx @@ -26,6 +26,7 @@ import React, { useState, } from 'react'; import { useTranslation } from 'react-i18next'; +import { useHistory } from 'react-router-dom'; import { Connection, Edge, @@ -96,6 +97,7 @@ import { createNewEdge, createNodes, decodeLineageHandles, + getAllDownstreamEdges, getAllTracedColumnEdge, getClassifiedEdge, getConnectedNodesEdges, @@ -135,6 +137,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const { t } = useTranslation(); const { fqn: decodedFqn } = useFqn(); const location = useCustomLocation(); + const history = useHistory(); const { isTourOpen, isTourPage } = useTourProvider(); const { appPreferences } = useApplicationStore(); const defaultLineageConfig = appPreferences?.lineageConfig as LineageSettings; @@ -197,6 +200,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const backspacePressed = useKeyPress('Backspace'); const { showModal } = useEntityExportModalProvider(); const [isPlatformLineage, setIsPlatformLineage] = useState(false); + const [dqHighlightedEdges, setDqHighlightedEdges] = useState>(); const lineageLayer = useMemo(() => { const param = location.search; @@ -426,12 +430,29 @@ const LineageProvider = ({ children }: LineageProviderProps) => { [queryFilter, decodedFqn] ); - const onPlatformViewChange = useCallback((view: LineagePlatformView) => { - setPlatformView(view); - if (view !== LineagePlatformView.None) { - setActiveLayer([]); - } - }, []); + const onPlatformViewChange = useCallback( + (view: LineagePlatformView) => { + setPlatformView(view); + if (view !== LineagePlatformView.None) { + setActiveLayer([]); + } + + if (isPlatformLineage) { + const searchData = QueryString.parse( + location.search.startsWith('?') + ? location.search.substring(1) + : location.search + ); + history.push({ + search: QueryString.stringify({ + ...searchData, + platformView: view !== LineagePlatformView.None ? view : undefined, + }), + }); + } + }, + [isPlatformLineage, location.search] + ); const exportLineageData = useCallback( async (_: string) => { @@ -596,7 +617,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { setEntityType(entityType); setIsPlatformLineage(isPlatformLineage ?? false); if (isPlatformLineage && !entity) { - setPlatformView(LineagePlatformView.Service); + onPlatformViewChange(LineagePlatformView.Service); } }, [] @@ -1527,6 +1548,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { dataQualityLineage, redraw, onPlatformViewChange, + dqHighlightedEdges, }; }, [ dataQualityLineage, @@ -1575,6 +1597,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { onExportClick, redraw, onPlatformViewChange, + dqHighlightedEdges, ]); useEffect(() => { @@ -1599,6 +1622,20 @@ const LineageProvider = ({ children }: LineageProviderProps) => { } }, [activeLayer, decodedFqn, lineageConfig]); + useEffect(() => { + if ( + dataQualityLineage?.nodes && + !isUndefined(edges) && + isUndefined(dqHighlightedEdges) + ) { + const edgesToHighlight = dataQualityLineage.nodes + .flatMap((dqNode) => getAllDownstreamEdges(dqNode.id, edges ?? [])) + .map((edge) => edge.id); + const edgesToHighlightSet = new Set(edgesToHighlight); + setDqHighlightedEdges(edgesToHighlightSet); + } + }, [dataQualityLineage, edges, dqHighlightedEdges]); + return (
{ const { t } = useTranslation(); + const location = useCustomLocation(); const history = useHistory(); + const queryParams = new URLSearchParams(location.search); + const platformView = + queryParams.get('platformView') ?? LineagePlatformView.Service; const { entityType } = useParams<{ entityType: EntityType }>(); const { fqn: decodedFqn } = useFqn(); const [selectedEntity, setSelectedEntity] = useState(); @@ -57,6 +63,16 @@ const PlatformLineage = () => { ); const [permissions, setPermissions] = useState(); + const handleEntitySelect = useCallback( + (value: EntityReference) => { + history.push( + `/lineage/${(value as SourceType).entityType}/${ + value.fullyQualifiedName + }` + ); + }, + [history] + ); const debouncedSearch = useCallback( debounce(async (value: string) => { try { @@ -94,17 +110,6 @@ const PlatformLineage = () => { [] ); - const handleEntitySelect = useCallback( - (value: EntityReference) => { - history.push( - `/lineage/${(value as SourceType).entityType}/${ - value.fullyQualifiedName - }` - ); - }, - [history] - ); - const init = useCallback(async () => { if (!decodedFqn || !entityType) { setDefaultValue(undefined); @@ -169,7 +174,14 @@ const PlatformLineage = () => { - +
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index 546b4321ab4..73119a8eba3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -1791,3 +1791,40 @@ export const getEntityTypeFromPlatformView = ( return 'service'; } }; + +/** + * Recursively finds all downstream edges from a given node in a graph. + * This function traverses the graph depth-first, collecting all edges that flow downstream + * from the specified node while avoiding cycles by tracking visited nodes. + * + * @param {string} nodeId - The ID of the starting node + * @param {Edge[]} edges - Array of all edges in the graph + * @param {Set} [visitedNodes=new Set()] - Set of already visited node IDs to prevent cycles + * @returns {Edge[]} Array of all downstream edges from the starting node + */ +export const getAllDownstreamEdges = ( + nodeId: string, + edges: Edge[], + visitedNodes: Set = new Set() +): Edge[] => { + // If we've already visited this node, return empty array to avoid cycles + if (visitedNodes.has(nodeId)) { + return []; + } + + visitedNodes.add(nodeId); + + // Get direct downstream edges + const directDownstreamEdges = edges.filter((edge) => edge.source === nodeId); + + // Get target nodes from direct downstream edges + const targetNodes = directDownstreamEdges.map((edge) => edge.target); + + // Recursively get downstream edges for each target node + const nestedDownstreamEdges = targetNodes.flatMap((targetNodeId) => + getAllDownstreamEdges(targetNodeId, edges, visitedNodes) + ); + + // Combine direct and nested downstream edges + return [...directDownstreamEdges, ...nestedDownstreamEdges]; +};