Minor: address dq feedback and move lineage tracing in observability layer (#18751)

* Minor: address dq feedback and move lineage tracing in observability layer * fixed: failing unit test
2025-11-02 11:39:12 +00:00 · 2024-11-23 17:35:52 +05:30 · 2024-11-23 17:35:52 +05:30 · 9edf3791c0
commit 9edf3791c0
parent 04659132b7
8 changed files with 81 additions and 47 deletions
--- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/no-dimension-icon.svg
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/no-dimension-icon.svg
@ -0,0 +1,10 @@
+<svg viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_13077_320457)">
+<path d="M7.49998 10.8335H2.49998C1.58331 10.8335 0.833313 11.5835 0.833313 12.5002V17.5002C0.833313 18.4168 1.58331 19.1668 2.49998 19.1668H7.49998C8.41665 19.1668 9.16665 18.4168 9.16665 17.5002V12.5002C9.16665 11.5835 8.41665 10.8335 7.49998 10.8335ZM7.49998 17.5002H2.49998V12.5002H7.49998V17.5002ZM7.49998 0.833496H2.49998C1.58331 0.833496 0.833313 1.5835 0.833313 2.50016V7.50016C0.833313 8.41683 1.58331 9.16683 2.49998 9.16683H7.49998C8.41665 9.16683 9.16665 8.41683 9.16665 7.50016V2.50016C9.16665 1.5835 8.41665 0.833496 7.49998 0.833496ZM7.49998 7.50016H2.49998V2.50016H7.49998V7.50016ZM18.3333 10.8335H11.6666C11.1666 10.8335 10.8333 11.1668 10.8333 11.6668C10.8333 12.1668 11.1666 12.5002 11.6666 12.5002H18.3333C18.8333 12.5002 19.1666 12.1668 19.1666 11.6668C19.1666 11.1668 18.8333 10.8335 18.3333 10.8335ZM18.3333 17.5002H11.6666C11.1666 17.5002 10.8333 17.8335 10.8333 18.3335C10.8333 18.8335 11.1666 19.1668 11.6666 19.1668H18.3333C18.8333 19.1668 19.1666 18.8335 19.1666 18.3335C19.1666 17.8335 18.8333 17.5002 18.3333 17.5002ZM17.5 0.833496H12.5C11.5833 0.833496 10.8333 1.5835 10.8333 2.50016V7.50016C10.8333 8.41683 11.5833 9.16683 12.5 9.16683H17.5C18.4166 9.16683 19.1666 8.41683 19.1666 7.50016V2.50016C19.1666 1.5835 18.4166 0.833496 17.5 0.833496ZM17.5 7.50016H12.5V2.50016H17.5V7.50016ZM18.3333 14.1668H15C14.5 14.1668 14.1666 14.5002 14.1666 15.0002C14.1666 15.5002 14.5 15.8335 15 15.8335H18.3333C18.8333 15.8335 19.1666 15.5002 19.1666 15.0002C19.1666 14.5002 18.8333 14.1668 18.3333 14.1668Z" fill="#0950C5"/>
+</g>
+<defs>
+<clipPath id="clip0_13077_320457">
+<rect width="20" height="20" fill="white"/>
+</clipPath>
+</defs>
+</svg>
--- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/ChartWidgets/StatusCardWidget/status-card-widget.less
+++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/ChartWidgets/StatusCardWidget/status-card-widget.less
@ -15,7 +15,9 @@
 .status-count-container {
  border: @global-border;
  height: 30px;
-  width: 30px;
+  width: 100%;
+  min-width: 50px;
+  padding: 0 12px;
  display: flex;
  align-items: center;
  justify-content: center;
--- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx
@ -90,10 +90,21 @@ export const CustomEdge = ({
    activeLayer,
    onAddPipelineClick,
    onColumnEdgeRemove,
+    dataQualityLineage,
  } = useLineageProvider();

  const { theme } = useApplicationStore();

+  const showDqTracing = useMemo(() => {
+    return (
+      (activeLayer.includes(LineageLayer.DataObservability) &&
+        dataQualityLineage?.edges?.some(
+          (dqEdge) => dqEdge?.doc_id === edge?.doc_id
+        )) ??
+      false
+    );
+  }, [activeLayer, dataQualityLineage, edge]);
+
  const isColumnHighlighted = useMemo(() => {
    if (!isColumnLineage) {
      return false;
@ -154,7 +165,7 @@ export const CustomEdge = ({

    let stroke = isStrokeNeeded ? theme.primaryColor : undefined;

-    if (edge?.isDqTestFailure) {
+    if (showDqTracing) {
      stroke = RED_3;
    }

@ -172,6 +183,7 @@ export const CustomEdge = ({
    isColumnHighlighted,
    isColumnLineage,
    tracedColumns,
+    showDqTracing,
  ]);

  const isPipelineEdgeAllowed = (
--- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx
@ -25,6 +25,7 @@ import {
 import { ReactComponent as IconTimesCircle } from '../../../assets/svg/ic-times-circle.svg';
 import { useLineageProvider } from '../../../context/LineageProvider/LineageProvider';
 import { EntityLineageNodeType } from '../../../enums/entity.enum';
+import { LineageLayer } from '../../../generated/configuration/lineageSettings';
 import { checkUpstreamDownstream } from '../../../utils/EntityLineageUtils';
 import './custom-node.less';
 import { getCollapseHandle, getExpandHandle } from './CustomNode.utils';
@ -46,10 +47,20 @@ const CustomNodeV1 = (props: NodeProps) => {
    onNodeCollapse,
    removeNodeHandler,
    loadChildNodesHandler,
+    activeLayer,
+    dataQualityLineage,
  } = useLineageProvider();

  const { label, isNewNode, node = {}, isRootNode } = data;

+  const showDqTracing = useMemo(() => {
+    return (
+      (activeLayer.includes(LineageLayer.DataObservability) &&
+        dataQualityLineage?.nodes?.some((dqNode) => dqNode.id === node?.id)) ??
+      false
+    );
+  }, [activeLayer, dataQualityLineage, node]);
+
  const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type;
  const isSelected = selectedNode === node;
  const { id, lineage, fullyQualifiedName } = node;
@ -258,7 +269,9 @@ const CustomNodeV1 = (props: NodeProps) => {
      className={classNames(
        'lineage-node p-0',
        isSelected ? 'custom-node-header-active' : 'custom-node-header-normal',
-        { 'data-quality-failed-custom-node-header': node?.isDqTestFailure },
+        {
+          'data-quality-failed-custom-node-header': showDqTracing,
+        },
        { 'custom-node-header-tracing': isTraced }
      )}
      data-testid={`lineage-node-${fullyQualifiedName}`}>
--- a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx
@ -47,6 +47,7 @@ export type UpstreamDownstreamData = {

 export interface LineageContextType {
  reactFlowInstance?: ReactFlowInstance;
+  dataQualityLineage?: EntityLineageResponse;
  nodes: Node[];
  edges: Edge[];
  tracedNodes: string[];
--- a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx
@ -11,6 +11,7 @@
 *  limitations under the License.
 */
 import { act, fireEvent, render, screen } from '@testing-library/react';
+import QueryString from 'qs';
 import React, { useEffect } from 'react';
 import { Edge } from 'reactflow';
 import { EdgeTypeEnum } from '../../components/Entity/EntityLineage/EntityLineage.interface';
@ -162,6 +163,9 @@ describe('LineageProvider', () => {
  });

  it('getDataQualityLineage should be called if alert is supported', async () => {
+    mockLocation.search = QueryString.stringify({
+      layers: ['DataObservability'],
+    });
    mockIsAlertSupported = true;
    (getLineageDataByFQN as jest.Mock).mockImplementationOnce(() =>
      Promise.resolve({
@ -190,6 +194,7 @@ describe('LineageProvider', () => {
    );

    mockIsAlertSupported = false;
+    mockLocation.search = '';
  });

  it('should call loadChildNodesHandler', async () => {
--- a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx
@ -147,6 +147,8 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
    edges: [],
    entity: {} as EntityReference,
  });
+  const [dataQualityLineage, setDataQualityLineage] =
+    useState<EntityLineageResponse>();
  const [updatedEntityLineage, setUpdatedEntityLineage] =
    useState<EntityLineageResponse | null>(null);
  const [showDeleteModal, setShowDeleteModal] = useState<boolean>(false);
@ -221,6 +223,25 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
    [entityLineage]
  );

+  const fetchDataQualityLineage = async (
+    fqn: string,
+    config?: LineageConfig
+  ) => {
+    if (isTourOpen || !tableClassBase.getAlertEnableStatus()) {
+      return;
+    }
+    try {
+      const dqLineageResp = await getDataQualityLineage(
+        fqn,
+        config,
+        queryFilter
+      );
+      setDataQualityLineage(dqLineageResp);
+    } catch (error) {
+      setDataQualityLineage(undefined);
+    }
+  };
+
  const fetchLineageData = useCallback(
    async (fqn: string, entityType: string, config?: LineageConfig) => {
      if (isTourOpen) {
@ -237,61 +258,25 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
          config,
          queryFilter
        );
-
-        const dqLineageResp = tableClassBase.getAlertEnableStatus()
-          ? await getDataQualityLineage(fqn, config, queryFilter)
-          : { nodes: [], edges: [] };
-
        if (res) {
-          const { nodes = [], entity, edges } = res;
+          const { nodes = [], entity } = res;
          const allNodes = uniqWith(
            [...nodes, entity].filter(Boolean),
            isEqual
-          ).map((node) => {
-            return {
-              ...node,
-              isDqTestFailure:
-                dqLineageResp.nodes?.some((dqNode) => dqNode.id === node.id) ??
-                false,
-            };
-          });
-
-          const updatedEntity = {
-            ...entity,
-            isDqTestFailure:
-              dqLineageResp.nodes?.some((dqNode) => dqNode.id === entity.id) ??
-              false,
-          };
-
-          const updatedEdges = edges?.map((edge) => {
-            return {
-              ...edge,
-              isDqTestFailure:
-                dqLineageResp.edges?.some(
-                  (dqEdge) => dqEdge?.doc_id === edge?.doc_id
-                ) ?? false,
-            };
-          });
+          );

          if (
            entityType !== EntityType.PIPELINE &&
            entityType !== EntityType.STORED_PROCEDURE
          ) {
            const { map: childMapObj } = getChildMap(
-              {
-                ...res,
-                nodes: allNodes,
-                edges: updatedEdges,
-                entity: updatedEntity,
-              },
+              { ...res, nodes: allNodes },
              decodedFqn
            );
            setChildMap(childMapObj);
            const { nodes: newNodes, edges: newEdges } = getPaginatedChildMap(
              {
                ...res,
-                entity: updatedEntity,
-                edges: updatedEdges,
                nodes: allNodes,
              },
              childMapObj,
@ -301,14 +286,12 @@ const LineageProvider = ({ children }: LineageProviderProps) => {

            setEntityLineage({
              ...res,
-              entity: updatedEntity,
              nodes: newNodes,
-              edges: [...(updatedEdges ?? []), ...newEdges],
+              edges: [...(res.edges ?? []), ...newEdges],
            });
          } else {
            setEntityLineage({
              ...res,
-              entity: updatedEntity,
              nodes: allNodes,
            });
          }
@ -1299,8 +1282,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
      onAddPipelineClick,
      onUpdateLayerView,
      onExportClick,
+      dataQualityLineage,
    };
  }, [
+    dataQualityLineage,
    isDrawerOpen,
    loading,
    isEditMode,
@ -1363,6 +1348,12 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
    }
  }, [lineageLayer]);

+  useEffect(() => {
+    if (activeLayer.includes(LineageLayer.DataObservability)) {
+      fetchDataQualityLineage(decodedFqn, lineageConfig);
+    }
+  }, [activeLayer, decodedFqn, lineageConfig]);
+
  return (
    <LineageContext.Provider value={activityFeedContextValues}>
      <div
--- a/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts
@ -11,7 +11,6 @@
 *  limitations under the License.
 */
 import { cloneDeep, isArray, isUndefined, omit, omitBy } from 'lodash';
-import { ReactComponent as TestCaseIcon } from '../../assets/svg/all-activity-v2.svg';
 import { ReactComponent as AccuracyIcon } from '../../assets/svg/ic-accuracy.svg';
 import { ReactComponent as CompletenessIcon } from '../../assets/svg/ic-completeness.svg';
 import { ReactComponent as ConsistencyIcon } from '../../assets/svg/ic-consistency.svg';
@ -19,6 +18,7 @@ import { ReactComponent as IntegrityIcon } from '../../assets/svg/ic-integrity.s
 import { ReactComponent as SqlIcon } from '../../assets/svg/ic-sql.svg';
 import { ReactComponent as UniquenessIcon } from '../../assets/svg/ic-uniqueness.svg';
 import { ReactComponent as ValidityIcon } from '../../assets/svg/ic-validity.svg';
+import { ReactComponent as NoDimensionIcon } from '../../assets/svg/no-dimension-icon.svg';
 import { StatusData } from '../../components/DataQuality/ChartWidgets/StatusCardWidget/StatusCardWidget.interface';
 import { TestCaseSearchParams } from '../../components/DataQuality/DataQuality.interface';
 import {
@ -235,6 +235,6 @@ export const getDimensionIcon = (dimension: DataQualityDimensions) => {
    case DataQualityDimensions.Validity:
      return ValidityIcon;
    default:
-      return TestCaseIcon;
+      return NoDimensionIcon;
  }
 };