Minor: lineage data quality failure tracing support (#18580)

openmetadata-ui/src/main/resources/ui/src/components/DataQuality/DataQuality.interface.ts

@@ -1,4 +1,5 @@
 import { DateRangeObject } from 'Models';
+import { LinkProps } from 'react-router-dom';
 import { TestCaseStatus } from '../../generated/tests/testCase';
 import { TestCaseResolutionStatusTypes } from '../../generated/tests/testCaseResolutionStatus';
 import { TestPlatform } from '../../generated/tests/testDefinition';
@@ -76,7 +77,7 @@ export interface DataStatisticWidgetProps {
   icon: SvgComponent;
   dataLabel: string;
   countValue: number;
-  redirectPath: string;
+  redirectPath: LinkProps['to'];
   linkLabel: string;
   isLoading?: boolean;
 }

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx

@@ -19,6 +19,7 @@ import { EdgeProps, getBezierPath } from 'reactflow';
 import { ReactComponent as FunctionIcon } from '../../../assets/svg/ic-function.svg';
 import { ReactComponent as IconTimesCircle } from '../../../assets/svg/ic-times-circle.svg';
 import { ReactComponent as PipelineIcon } from '../../../assets/svg/pipeline-grey.svg';
+import { RED_3 } from '../../../constants/Color.constants';
 import { FOREIGN_OBJECT_SIZE } from '../../../constants/Lineage.constants';
 import { useLineageProvider } from '../../../context/LineageProvider/LineageProvider';
 import { EntityType } from '../../../enums/entity.enum';
@@ -76,6 +77,7 @@ export const CustomEdge = ({
     isPipelineRootNode,
     ...rest
   } = data;
+
   const offset = 4;
 
   const { fromEntity, toEntity, pipeline, pipelineEntityType } =
@@ -150,10 +152,16 @@ export const CustomEdge = ({
       opacity = tracedNodes.length === 0 || isStrokeNeeded ? 1 : 0.25;
     }
 
+    let stroke = isStrokeNeeded ? theme.primaryColor : undefined;
+
+    if (edge?.isDqTestFailure) {
+      stroke = RED_3;
+    }
+
     return {
       ...style,
       ...{
-        stroke: isStrokeNeeded ? theme.primaryColor : undefined,
+        stroke,
         opacity,
       },
     };

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx

@@ -49,6 +49,7 @@ const CustomNodeV1 = (props: NodeProps) => {
   } = useLineageProvider();
 
   const { label, isNewNode, node = {}, isRootNode } = data;
+
   const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type;
   const isSelected = selectedNode === node;
   const { id, lineage, fullyQualifiedName } = node;
@@ -257,6 +258,7 @@ const CustomNodeV1 = (props: NodeProps) => {
       className={classNames(
         'lineage-node p-0',
         isSelected ? 'custom-node-header-active' : 'custom-node-header-normal',
+        { 'data-quality-failed-custom-node-header': node?.isDqTestFailure },
         { 'custom-node-header-tracing': isTraced }
       )}
       data-testid={`lineage-node-${fullyQualifiedName}`}>

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less

@@ -154,6 +154,41 @@
   }
 }
 
+.react-flow__node-default.selectable.selected,
+.react-flow__node-default.selectable:focus,
+.react-flow__node-default.selectable:focus-visible,
+.react-flow__node-input.selectable.selected,
+.react-flow__node-input.selectable:focus,
+.react-flow__node-input.selectable:focus-visible,
+.react-flow__node-output.selectable.selected,
+.react-flow__node-output.selectable:focus,
+.react-flow__node-output.selectable:focus-visible,
+.react-flow__node-group.selectable.selected,
+.react-flow__node-group.selectable:focus,
+.react-flow__node-group.selectable:focus-visible,
+.react-flow__node-load-more.selectable.selected,
+.react-flow__node-load-more.selectable:focus,
+.react-flow__node-load-more.selectable:focus-visible {
+  .data-quality-failed-custom-node-header {
+    &.lineage-node {
+      border-color: @red-3 !important;
+    }
+    .lineage-node-handle {
+      border-color: @red-3;
+      svg {
+        color: @red-3;
+      }
+    }
+    .label-container {
+      background: fade(@red-3, 10%);
+    }
+    .column-container {
+      background: fade(@red-3, 10%);
+      border-top: 1px solid @border-color;
+    }
+  }
+}
+
 .custom-node-header-active {
   .label-container {
     background: @primary-1;
@@ -163,6 +198,15 @@
     border-top: 1px solid @border-color;
   }
 }
+.data-quality-failed-custom-node-header.custom-node-header-active {
+  .label-container {
+    background: fade(@red-3, 10%);
+  }
+  .column-container {
+    background: fade(@red-3, 10%);
+    border-top: 1px solid @red-3;
+  }
+}
 
 .react-flow__handle.connectable {
   background-color: @text-color;

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less

@@ -44,6 +44,25 @@
     }
   }
 }
+
+.data-quality-failed-custom-node-header {
+  &.custom-node-header-normal {
+    border: 1px solid @red-3;
+  }
+  .react-flow__handle {
+    border-color: @red-3;
+  }
+  .lineage-node-handle {
+    border: 1px solid @red-3;
+    svg {
+      color: @red-3;
+    }
+  }
+
+  &.custom-node-header-active {
+    border-color: @red-3;
+  }
+}
 .custom-node-header-active {
   border-color: @primary-color;
 }

openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts

@@ -52,6 +52,7 @@ export interface EdgeDetails {
   columns?: ColumnLineage[];
   description?: string;
   pipelineEntityType?: EntityType.PIPELINE | EntityType.STORED_PROCEDURE;
+  doc_id?: string;
 }
 
 export type LineageSourceType = Omit<SourceType, 'service'> & {

openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx

@@ -15,7 +15,10 @@ import React, { useEffect } from 'react';
 import { Edge } from 'reactflow';
 import { EdgeTypeEnum } from '../../components/Entity/EntityLineage/EntityLineage.interface';
 import { EntityType } from '../../enums/entity.enum';
-import { getLineageDataByFQN } from '../../rest/lineageAPI';
+import {
+  getDataQualityLineage,
+  getLineageDataByFQN,
+} from '../../rest/lineageAPI';
 import LineageProvider, { useLineageProvider } from './LineageProvider';
 
 const mockLocation = {
@@ -128,9 +131,16 @@ jest.mock(
     });
   }
 );
+let mockIsAlertSupported = false;
+jest.mock('../../utils/TableClassBase', () => ({
+  getAlertEnableStatus: jest
+    .fn()
+    .mockImplementation(() => mockIsAlertSupported),
+}));
 
 jest.mock('../../rest/lineageAPI', () => ({
   getLineageDataByFQN: jest.fn(),
+  getDataQualityLineage: jest.fn(),
 }));
 
 describe('LineageProvider', () => {
@@ -148,6 +158,38 @@ describe('LineageProvider', () => {
     });
 
     expect(getLineageDataByFQN).toHaveBeenCalled();
+    expect(getDataQualityLineage).not.toHaveBeenCalled();
+  });
+
+  it('getDataQualityLineage should be called if alert is supported', async () => {
+    mockIsAlertSupported = true;
+    (getLineageDataByFQN as jest.Mock).mockImplementationOnce(() =>
+      Promise.resolve({
+        nodes: [],
+        edges: [],
+      })
+    );
+    await act(async () => {
+      render(
+        <LineageProvider>
+          <DummyChildrenComponent />
+        </LineageProvider>
+      );
+    });
+
+    expect(getLineageDataByFQN).toHaveBeenCalledWith(
+      'table1',
+      'table',
+      { downstreamDepth: 1, nodesPerLayer: 50, upstreamDepth: 1 },
+      ''
+    );
+    expect(getDataQualityLineage).toHaveBeenCalledWith(
+      'table1',
+      { downstreamDepth: 1, nodesPerLayer: 50, upstreamDepth: 1 },
+      ''
+    );
+
+    mockIsAlertSupported = false;
   });
 
   it('should call loadChildNodesHandler', async () => {

openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx

@@ -14,8 +14,9 @@ import Icon from '@ant-design/icons/lib/components/Icon';
 import { Button, Modal } from 'antd';
 import { AxiosError } from 'axios';
 import classNames from 'classnames';
-import { isEqual, isUndefined, uniqueId, uniqWith } from 'lodash';
+import { isEqual, isUndefined, uniq, uniqueId, uniqWith } from 'lodash';
 import { LoadingState } from 'Models';
+import QueryString from 'qs';
 import React, {
   createContext,
   DragEvent,
@@ -76,8 +77,13 @@ import {
   LineageDetails,
 } from '../../generated/type/entityLineage';
 import { useApplicationStore } from '../../hooks/useApplicationStore';
+import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation';
 import { useFqn } from '../../hooks/useFqn';
-import { getLineageDataByFQN, updateLineageEdge } from '../../rest/lineageAPI';
+import {
+  getDataQualityLineage,
+  getLineageDataByFQN,
+  updateLineageEdge,
+} from '../../rest/lineageAPI';
 import {
   addLineageHandler,
   centerNodePosition,
@@ -104,6 +110,7 @@ import {
   removeLineageHandler,
 } from '../../utils/EntityLineageUtils';
 import { getEntityReferenceFromEntity } from '../../utils/EntityUtils';
+import tableClassBase from '../../utils/TableClassBase';
 import { showErrorToast } from '../../utils/ToastUtils';
 import { useTourProvider } from '../TourProvider/TourProvider';
 import {
@@ -117,6 +124,7 @@ export const LineageContext = createContext({} as LineageContextType);
 const LineageProvider = ({ children }: LineageProviderProps) => {
   const { t } = useTranslation();
   const { fqn: decodedFqn } = useFqn();
+  const location = useCustomLocation();
   const { isTourOpen, isTourPage } = useTourProvider();
   const { appPreferences } = useApplicationStore();
   const defaultLineageConfig = appPreferences?.lineageConfig as LineageSettings;
@@ -180,6 +188,15 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
   const [paginationData, setPaginationData] = useState({});
   const { showModal } = useEntityExportModalProvider();
 
+  const lineageLayer = useMemo(() => {
+    const param = location.search;
+    const searchData = QueryString.parse(
+      param.startsWith('?') ? param.substring(1) : param
+    );
+
+    return searchData.layers as LineageLayer[] | undefined;
+  }, [location.search]);
+
   const initLineageChildMaps = useCallback(
     (
       lineageData: EntityLineageResponse,
@@ -220,25 +237,63 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
           config,
           queryFilter
         );
+
+        const dqLineageResp =
+          entityType === EntityType.TABLE &&
+          tableClassBase.getAlertEnableStatus()
+            ? await getDataQualityLineage(fqn, config, queryFilter)
+            : { nodes: [], edges: [] };
+
         if (res) {
-          const { nodes = [], entity } = res;
+          const { nodes = [], entity, edges } = res;
           const allNodes = uniqWith(
             [...nodes, entity].filter(Boolean),
             isEqual
-          );
+          ).map((node) => {
+            return {
+              ...node,
+              isDqTestFailure:
+                dqLineageResp.nodes?.some((dqNode) => dqNode.id === node.id) ??
+                false,
+            };
+          });
+
+          const updatedEntity = {
+            ...entity,
+            isDqTestFailure:
+              dqLineageResp.nodes?.some((dqNode) => dqNode.id === entity.id) ??
+              false,
+          };
+
+          const updatedEdges = edges?.map((edge) => {
+            return {
+              ...edge,
+              isDqTestFailure:
+                dqLineageResp.edges?.some(
+                  (dqEdge) => dqEdge?.doc_id === edge?.doc_id
+                ) ?? false,
+            };
+          });
 
           if (
             entityType !== EntityType.PIPELINE &&
             entityType !== EntityType.STORED_PROCEDURE
           ) {
             const { map: childMapObj } = getChildMap(
-              { ...res, nodes: allNodes },
+              {
+                ...res,
+                nodes: allNodes,
+                edges: updatedEdges,
+                entity: updatedEntity,
+              },
               decodedFqn
             );
             setChildMap(childMapObj);
             const { nodes: newNodes, edges: newEdges } = getPaginatedChildMap(
               {
                 ...res,
+                entity: updatedEntity,
+                edges: updatedEdges,
                 nodes: allNodes,
               },
               childMapObj,
@@ -248,12 +303,14 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
 
             setEntityLineage({
               ...res,
+              entity: updatedEntity,
               nodes: newNodes,
-              edges: [...(res.edges ?? []), ...newEdges],
+              edges: [...(updatedEdges ?? []), ...newEdges],
             });
           } else {
             setEntityLineage({
               ...res,
+              entity: updatedEntity,
               nodes: allNodes,
             });
           }
@@ -1308,6 +1365,12 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
     }
   }, [isTourOpen, isTourPage]);
 
+  useEffect(() => {
+    if (lineageLayer) {
+      setActiveLayer((pre) => uniq([...lineageLayer, ...pre]));
+    }
+  }, [lineageLayer]);
+
   return (
     <LineageContext.Provider value={activityFeedContextValues}>
       <div

openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts

@@ -48,7 +48,8 @@ export const getLineageDataByFQN = async (
 
 export const getDataQualityLineage = async (
   fqn: string,
-  config?: Partial<LineageConfig>
+  config?: Partial<LineageConfig>,
+  queryFilter?: string
 ) => {
   const { upstreamDepth = 1 } = config ?? {};
   const response = await APIClient.get<EntityLineageResponse>(
@@ -58,6 +59,7 @@ export const getDataQualityLineage = async (
         fqn,
         upstreamDepth,
         includeDeleted: false,
+        query_filter: queryFilter,
       },
     }
   );