diff --git a/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/CustomControlElements.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/CustomControlElements.component.tsx new file mode 100644 index 00000000000..9879a80a45a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/CustomControlElements.component.tsx @@ -0,0 +1,64 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import classNames from 'classnames'; +import React, { useCallback } from 'react'; +import { NO_PERMISSION_FOR_ACTION } from '../../constants/HelperTextUtil'; +import { LoadingStatus } from '../../utils/EntityLineageUtils'; +import SVGIcons from '../../utils/SvgUtils'; +import CustomControls, { ControlButton } from './CustomControls.component'; +import { CustomControlElementsProps } from './EntityLineage.interface'; + +const CustomControlElements = ({ + deleted, + isEditMode, + hasEditAccess, + onClick, + loading, + status, +}: CustomControlElementsProps) => { + const getLoadingStatus = useCallback(() => { + const editIcon = ( + + ); + + return LoadingStatus(editIcon, loading, status); + }, [loading, status, isEditMode]); + + return ( + + {!deleted && ( + + {getLoadingStatus()} + + )} + + ); +}; + +export default CustomControlElements; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/EntityLineage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/EntityLineage.component.tsx index 6747a9ac72b..d1a86478ffc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/EntityLineage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/EntityLineage.component.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Modal } from 'antd'; import { AxiosError } from 'axios'; import classNames from 'classnames'; import { @@ -41,7 +41,6 @@ import ReactFlow, { Edge, getConnectedEdges, isNode, - MarkerType, Node, ReactFlowInstance, ReactFlowProvider, @@ -49,26 +48,37 @@ import ReactFlow, { useNodesState, } from 'reactflow'; import { getTableDetails } from '../../axiosAPIs/tableAPI'; -import { NO_PERMISSION_FOR_ACTION } from '../../constants/HelperTextUtil'; import { ELEMENT_DELETE_STATE } from '../../constants/Lineage.constants'; import { AddLineage, ColumnLineage, } from '../../generated/api/lineage/addLineage'; import { Column } from '../../generated/entity/data/table'; -import { EntityLineage } from '../../generated/type/entityLineage'; +import { + EntityLineage, + LineageDetails, +} from '../../generated/type/entityLineage'; import { EntityReference } from '../../generated/type/entityReference'; import { withLoader } from '../../hoc/withLoader'; import { + createNewEdge, dragHandle, + findUpstreamDownStreamEdge, getColumnType, getDataLabel, getDeletedLineagePlaceholder, + getEdgeType, getLayoutedElements, getLineageData, getModalBodyText, getNodeRemoveButton, + getRemovedNodeData, + getSelectedEdgeArr, getUniqueFlowElements, + getUpdatedEdge, + getUpdatedUpstreamDownStreamEdgeArr, + getUpStreamDownStreamColumnLineageArr, + LoadingStatus, onLoad, onNodeContextMenu, onNodeMouseEnter, @@ -80,14 +90,14 @@ import { getEntityIcon } from '../../utils/TableUtils'; import { showErrorToast } from '../../utils/ToastUtils'; import EntityInfoDrawer from '../EntityInfoDrawer/EntityInfoDrawer.component'; import Loader from '../Loader/Loader'; -import ConfirmationModal from '../Modals/ConfirmationModal/ConfirmationModal'; -import CustomControls, { ControlButton } from './CustomControls.component'; +import CustomControlElements from './CustomControlElements.component'; import { CustomEdge } from './CustomEdge.component'; import CustomNode from './CustomNode.component'; import { CustomEdgeData, - CustomeElement, + CustomElement, EdgeData, + EdgeTypeEnum, ElementLoadingState, EntityLineageProp, ModifiedColumn, @@ -118,7 +128,6 @@ const EntityLineageComponent: FunctionComponent = ({ ); const expandButton = useRef(null); const [isEditMode, setEditMode] = useState(false); - const tableColumnsRef = useRef<{ [key: string]: Column[] }>( {} as { [key: string]: Column[] } ); @@ -127,13 +136,10 @@ const EntityLineageComponent: FunctionComponent = ({ {} as EntityReference ); const [confirmDelete, setConfirmDelete] = useState(false); - - const [showdeleteModal, setShowDeleteModal] = useState(false); - + const [showDeleteModal, setShowDeleteModal] = useState(false); const [selectedEdge, setSelectedEdge] = useState( {} as SelectedEdge ); - const [loading, setLoading] = useState(false); const [status, setStatus] = useState('initial'); const [deletionState, setDeletionState] = useState<{ @@ -162,30 +168,8 @@ const EntityLineageComponent: FunctionComponent = ({ }), [] ); - const customEdges = useMemo(() => ({ buttonedge: CustomEdge }), []); - /** - * take node as input and check if node is main entity or not - * @param node - * @returns class `leaf-node core` for main node and `leaf-node` for leaf node - */ - const getNodeClass = (node: Node) => { - return `${ - node.id.includes(updatedLineageData.entity?.id) && !isEditMode - ? 'leaf-node core' - : 'leaf-node' - }`; - }; - - /** - * take entity as input and set it as selected entity - * @param entity - */ - const selectedEntityHandler = (entity: EntityReference) => { - setSelectedEntity(entity); - }; - /** * take state and value to set selected node * @param state @@ -196,6 +180,11 @@ const EntityLineageComponent: FunctionComponent = ({ setSelectedNode(value); }; + const resetSelectedData = () => { + setNewAddedNode({} as Node); + setSelectedEntity({} as EntityReference); + }; + /** * * @param node @@ -238,7 +227,7 @@ const EntityLineageComponent: FunctionComponent = ({ /** * * @param data selected edge - * @param confirmDelete confirmation state for deleting seslected edge + * @param confirmDelete confirmation state for deleting selected edge */ const removeEdgeHandler = (data: SelectedEdge, confirmDelete: boolean) => { if (confirmDelete) { @@ -255,31 +244,16 @@ const EntityLineageComponent: FunctionComponent = ({ (e) => e.source !== data.source.id && e.target !== data.target.id ); }); - - /** - * Get new downstreamEdges - */ - const newDownStreamEdges = updatedLineageData.downstreamEdges?.filter( - (dn) => - !updatedLineageData.downstreamEdges?.find( - () => - edgeData.fromId === dn.fromEntity && edgeData.toId === dn.toEntity - ) + const newDownStreamEdges = getSelectedEdgeArr( + updatedLineageData.downstreamEdges || [], + edgeData + ); + const newUpStreamEdges = getSelectedEdgeArr( + updatedLineageData.upstreamEdges || [], + edgeData ); - /** - * Get new upstreamEdges - */ - const newUpStreamEdges = updatedLineageData.upstreamEdges?.filter( - (up) => - !updatedLineageData.upstreamEdges?.find( - () => - edgeData.fromId === up.fromEntity && edgeData.toId === up.toEntity - ) - ); - - setNewAddedNode({} as Node); - setSelectedEntity({} as EntityReference); + resetSelectedData(); setUpdatedLineageData({ ...updatedLineageData, downstreamEdges: newDownStreamEdges, @@ -291,14 +265,14 @@ const EntityLineageComponent: FunctionComponent = ({ const removeColumnEdge = (data: SelectedEdge, confirmDelete: boolean) => { if (confirmDelete) { - const upStreamEdge = updatedLineageData.upstreamEdges?.find( - (up) => - up.fromEntity === data.source.id && up.toEntity === data.target.id + const upStreamEdge = findUpstreamDownStreamEdge( + updatedLineageData.upstreamEdges, + data ); - const downStreamEdge = updatedLineageData.downstreamEdges?.find( - (down) => - down.fromEntity === data.source.id && down.toEntity === data.target.id + const downStreamEdge = findUpstreamDownStreamEdge( + updatedLineageData.downstreamEdges, + data ); const selectedEdge: AddLineage = { @@ -313,92 +287,39 @@ const EntityLineageComponent: FunctionComponent = ({ }, }, }; + let lineageDetails: LineageDetails | undefined; - if (!isUndefined(upStreamEdge)) { - const upColumnsLineage: ColumnLineage[] = - upStreamEdge.lineageDetails?.columnsLineage?.reduce((col, curr) => { - if (curr.toColumn === data.data?.targetHandle) { - const newCol = { - ...curr, - fromColumns: - curr.fromColumns?.filter( - (c) => c !== data.data?.sourceHandle - ) || [], - }; - if (newCol.fromColumns?.length) { - return [...col, newCol]; - } else { - return col; - } - } - - return [...col, curr]; - }, [] as ColumnLineage[]) || []; - selectedEdge.edge.lineageDetails = { - sqlQuery: upStreamEdge.lineageDetails?.sqlQuery || '', - columnsLineage: upColumnsLineage, - }; - + if (!isUndefined(upStreamEdge) && upStreamEdge.lineageDetails) { + lineageDetails = getUpStreamDownStreamColumnLineageArr( + upStreamEdge.lineageDetails, + data + ); setUpdatedLineageData({ ...updatedLineageData, - upstreamEdges: updatedLineageData.upstreamEdges?.map((up) => { - if ( - up.fromEntity === data.source.id && - up.toEntity === data.target.id - ) { - return { - ...up, - lineageDetails: selectedEdge.edge.lineageDetails, - }; - } - - return up; - }), + upstreamEdges: getUpdatedUpstreamDownStreamEdgeArr( + updatedLineageData.upstreamEdges || [], + data, + lineageDetails + ), }); - } - - if (!isUndefined(downStreamEdge)) { - const downColumnsLineage: ColumnLineage[] = - downStreamEdge.lineageDetails?.columnsLineage?.reduce((col, curr) => { - if (curr.toColumn === data.data?.targetHandle) { - const newCol: ColumnLineage = { - ...curr, - fromColumns: - curr.fromColumns?.filter( - (c) => c !== data.data?.sourceHandle - ) || [], - }; - if (newCol.fromColumns?.length) { - return [...col, newCol]; - } else { - return col; - } - } - - return [...col, curr]; - }, [] as ColumnLineage[]) || []; - selectedEdge.edge.lineageDetails = { - sqlQuery: downStreamEdge.lineageDetails?.sqlQuery || '', - columnsLineage: downColumnsLineage, - }; - + } else if ( + !isUndefined(downStreamEdge) && + downStreamEdge.lineageDetails + ) { + lineageDetails = getUpStreamDownStreamColumnLineageArr( + downStreamEdge.lineageDetails, + data + ); setUpdatedLineageData({ ...updatedLineageData, - downstreamEdges: updatedLineageData.downstreamEdges?.map((down) => { - if ( - down.fromEntity === data.source.id && - down.toEntity === data.target.id - ) { - return { - ...down, - lineageDetails: selectedEdge.edge.lineageDetails, - }; - } - - return down; - }), + downstreamEdges: getUpdatedUpstreamDownStreamEdgeArr( + updatedLineageData.downstreamEdges || [], + data, + lineageDetails + ), }); } + selectedEdge.edge.lineageDetails = lineageDetails; setEdges((pre) => { return pre.filter( (e) => @@ -409,8 +330,7 @@ const EntityLineageComponent: FunctionComponent = ({ ); }); addLineageHandler(selectedEdge); - setNewAddedNode({} as Node); - setSelectedEntity({} as EntityReference); + resetSelectedData(); setConfirmDelete(false); } }; @@ -431,42 +351,53 @@ const EntityLineageComponent: FunctionComponent = ({ ...(updatedLineageData.nodes || []), updatedLineageData.entity, ]; - let targetNode = allNode.find((n) => data.target?.includes(n.id)); - let sourceNode = allNode.find((n) => data.source?.includes(n.id)); - - if (isUndefined(targetNode)) { - targetNode = isEmpty(selectedEntity) - ? updatedLineageData.entity - : selectedEntity; - } - if (isUndefined(sourceNode)) { - sourceNode = isEmpty(selectedEntity) - ? updatedLineageData.entity - : selectedEntity; - } - - return { id: data.id, source: sourceNode, target: targetNode, data }; + return { + ...getRemovedNodeData( + allNode, + data, + updatedLineageData.entity, + selectedEntity + ), + data, + }; }); }; - /** - * Reset State between view and edit mode toggle - */ - const resetViewEditState = () => { - setConfirmDelete(false); - }; + const removeNodeHandler = useCallback( + (node: Node) => { + // Get edges connected to selected node + const edgesToRemove = getConnectedEdges([node], edges); + + edgesToRemove.forEach((edge) => { + removeEdgeHandler( + getRemovedNodeData( + updatedLineageData.nodes || [], + edge, + updatedLineageData.entity, + selectedEntity + ), + true + ); + }); + + setNodes( + (previousNodes) => + getUniqueFlowElements( + previousNodes.filter((previousNode) => previousNode.id !== node.id) + ) as Node[] + ); + setNewAddedNode({} as Node); + }, + [nodes, updatedLineageData] + ); const setElementsHandle = (data: EntityLineage) => { - let uniqueElements: CustomeElement = { - node: [], - edge: [], - }; - const currentData = { - nodes: [...(nodes || [])], - edges: [...(edges || [])], - }; if (!isEmpty(data)) { + const currentData = { + nodes: [...(nodes || [])], + edges: [...(edges || [])], + }; const graphElements = getLineageData( data, selectNodeHandler, @@ -477,13 +408,12 @@ const EntityLineageComponent: FunctionComponent = ({ isEditMode, 'buttonedge', onEdgeClick, - // eslint-disable-next-line @typescript-eslint/no-use-before-define removeNodeHandler, tableColumnsRef.current, currentData - ) as CustomeElement; + ) as CustomElement; - uniqueElements = { + const uniqueElements: CustomElement = { node: getUniqueFlowElements(graphElements.node) as Node[], edge: getUniqueFlowElements(graphElements.edge) as Edge[], }; @@ -491,7 +421,7 @@ const EntityLineageComponent: FunctionComponent = ({ setNodes(node); setEdges(edge); - resetViewEditState(); + setConfirmDelete(false); } }; @@ -502,20 +432,38 @@ const EntityLineageComponent: FunctionComponent = ({ const closeDrawer = (value: boolean) => { setIsDrawerOpen(value); setNodes((prevElements) => { - return prevElements.map((el) => { - if (el.id === selectedNode.id) { + return prevElements.map((prevElement) => { + if (prevElement.id === selectedNode.id) { + const className = + prevElement.id.includes(updatedLineageData.entity?.id) && + !isEditMode + ? 'leaf-node core' + : 'leaf-node'; + return { - ...el, - className: getNodeClass(el), + ...prevElement, + className, }; } else { - return el; + return prevElement; } }); }); setSelectedNode({} as SelectedNode); }; + const getSourceOrTargetNode = (queryStr: string) => { + return queryStr.includes(updatedLineageData.entity?.id) + ? updatedLineageData.entity + : selectedEntity; + }; + + const getUpdatedNodes = (entityLineage: EntityLineage) => { + return !isEmpty(selectedEntity) + ? [...(entityLineage.nodes || []), selectedEntity] + : entityLineage.nodes; + }; + /** * take edge or connection to add new element in the graph * @param params @@ -527,293 +475,183 @@ const EntityLineageComponent: FunctionComponent = ({ if (target === source) return; const columnConnection = !isNil(sourceHandle) && !isNil(targetHandle); - const normalConnection = isNil(sourceHandle) && isNil(targetHandle); - const mainEntity = updatedLineageData.entity; - if (columnConnection || normalConnection) { - setStatus('waiting'); - setLoading(true); - let edgeType: 'upstream' | 'downstream' | '' = ''; + setStatus('waiting'); + setLoading(true); - const nodes = [ - ...(updatedLineageData.nodes as EntityReference[]), - updatedLineageData.entity, - ]; + const edgeType = getEdgeType(updatedLineageData, params); + const nodes = [ + ...(updatedLineageData.nodes as EntityReference[]), + updatedLineageData.entity, + ]; - const sourceDownstreamNode = updatedLineageData.downstreamEdges?.find( - (d) => - (source?.includes(d.fromEntity) || source?.includes(d.toEntity)) && - source !== mainEntity.id - ); + let targetNode = nodes?.find((n) => target?.includes(n.id)); - const sourceUpStreamNode = updatedLineageData.upstreamEdges?.find( - (u) => - (source?.includes(u.fromEntity) || source?.includes(u.toEntity)) && - source !== mainEntity.id - ); + let sourceNode = nodes?.find((n) => source?.includes(n.id)); - const targetDownStreamNode = updatedLineageData.downstreamEdges?.find( - (d) => - (target?.includes(d.toEntity) || target?.includes(d.fromEntity)) && - target !== mainEntity.id - ); + if (isUndefined(targetNode) && sourceNode?.id !== selectedEntity?.id) { + targetNode = getSourceOrTargetNode(target || ''); + } + if (isUndefined(sourceNode) && targetNode?.id !== selectedEntity?.id) { + sourceNode = getSourceOrTargetNode(source || ''); + } - const targetUpStreamNode = updatedLineageData.upstreamEdges?.find( - (u) => - (target?.includes(u.toEntity) || target?.includes(u.fromEntity)) && - target !== mainEntity.id - ); - - const isUpstream = - (!isNil(sourceUpStreamNode) && !isNil(targetDownStreamNode)) || - !isNil(sourceUpStreamNode) || - !isNil(targetUpStreamNode) || - target?.includes(mainEntity.id); - - const isDownstream = - (!isNil(sourceDownstreamNode) && !isNil(targetUpStreamNode)) || - !isNil(sourceDownstreamNode) || - !isNil(targetDownStreamNode) || - source?.includes(mainEntity.id); - - if (isUpstream) { - edgeType = 'upstream'; - } else if (isDownstream) { - edgeType = 'downstream'; - } - - let targetNode = nodes?.find((n) => target?.includes(n.id)); - - let sourceNode = nodes?.find((n) => source?.includes(n.id)); - - if (isUndefined(targetNode) && sourceNode?.id !== selectedEntity?.id) { - targetNode = target?.includes(updatedLineageData.entity?.id) - ? updatedLineageData.entity - : selectedEntity; - } - if (isUndefined(sourceNode) && targetNode?.id !== selectedEntity?.id) { - sourceNode = source?.includes(updatedLineageData.entity?.id) - ? updatedLineageData.entity - : selectedEntity; - } - - if (!isUndefined(sourceNode) && !isUndefined(targetNode)) { - const newEdge: AddLineage = { - edge: { - fromEntity: { - id: sourceNode.id, - type: sourceNode.type, - }, - toEntity: { - id: targetNode.id, - type: targetNode.type, - }, + if (!isUndefined(sourceNode) && !isUndefined(targetNode)) { + const newEdge: AddLineage = { + edge: { + fromEntity: { + id: sourceNode.id, + type: sourceNode.type, }, - }; + toEntity: { + id: targetNode.id, + type: targetNode.type, + }, + }, + }; - if (columnConnection) { - const allEdge = [ - ...(updatedLineageData.downstreamEdges || []), - ...(updatedLineageData.upstreamEdges || []), - ]; - const currentEdge = allEdge.find( - (e) => e.fromEntity === source && e.toEntity === target - )?.lineageDetails; + if (columnConnection) { + const allEdge = [ + ...(updatedLineageData.downstreamEdges || []), + ...(updatedLineageData.upstreamEdges || []), + ]; + const currentEdge = allEdge.find( + (edge) => edge.fromEntity === source && edge.toEntity === target + )?.lineageDetails; - if (isUndefined(currentEdge)) { - newEdge.edge.lineageDetails = { - sqlQuery: '', - columnsLineage: [ - { - fromColumns: [sourceHandle || ''], - toColumn: targetHandle || '', - }, - ], - }; - } else { - const updatedColumnsLineage: ColumnLineage[] = - currentEdge.columnsLineage?.map((l) => { - if (l.toColumn === targetHandle) { - return { - ...l, - fromColumns: [ - ...(l.fromColumns || []), - sourceHandle || '', - ], - }; - } - - return l; - }) || []; - if ( - !updatedColumnsLineage.find((l) => l.toColumn === targetHandle) - ) { - updatedColumnsLineage.push({ + if (isUndefined(currentEdge)) { + newEdge.edge.lineageDetails = { + sqlQuery: '', + columnsLineage: [ + { fromColumns: [sourceHandle || ''], toColumn: targetHandle || '', - }); - } - newEdge.edge.lineageDetails = { - sqlQuery: currentEdge.sqlQuery || '', - columnsLineage: updatedColumnsLineage, - }; + }, + ], + }; + } else { + const updatedColumnsLineage: ColumnLineage[] = + currentEdge.columnsLineage?.map((lineage) => { + if (lineage.toColumn === targetHandle) { + return { + ...lineage, + fromColumns: [ + ...(lineage.fromColumns || []), + sourceHandle || '', + ], + }; + } + + return lineage; + }) || []; + if ( + !updatedColumnsLineage.find( + (lineage) => lineage.toColumn === targetHandle + ) + ) { + updatedColumnsLineage.push({ + fromColumns: [sourceHandle || ''], + toColumn: targetHandle || '', + }); } - - setEdges((els) => { - const newEdgeData = { - id: `column-${sourceHandle}-${targetHandle}-edge-${params.source}-${params.target}`, - source: source || '', - target: target || '', - sourceHandle: sourceHandle, - targetHandle: targetHandle, - type: isEditMode ? 'buttonedge' : 'custom', - markerEnd: { - type: MarkerType.ArrowClosed, - }, - data: { - id: `column-${sourceHandle}-${targetHandle}-edge-${params.source}-${params.target}`, - source: params.source, - target: params.target, - sourceHandle: sourceHandle, - targetHandle: targetHandle, - sourceType: sourceNode?.type, - targetType: targetNode?.type, - isColumnLineage: true, - onEdgeClick, - }, - }; - - return getUniqueFlowElements(addEdge(newEdgeData, els)) as Edge[]; - }); + newEdge.edge.lineageDetails = { + sqlQuery: currentEdge.sqlQuery || '', + columnsLineage: updatedColumnsLineage, + }; } - setEdges((els) => { - const newEdgeData = { - id: `edge-${params.source}-${params.target}`, - source: `${params.source}`, - target: `${params.target}`, - type: isEditMode ? 'buttonedge' : 'custom', - style: { strokeWidth: '2px' }, - markerEnd: { - type: MarkerType.ArrowClosed, - }, - data: { - id: `edge-${params.source}-${params.target}`, - source: params.source, - target: params.target, - sourceType: sourceNode?.type, - targetType: targetNode?.type, - isColumnLineage: false, - onEdgeClick, - }, - }; + setEdges((previousEdges) => { + const newEdgeData = createNewEdge( + params, + isEditMode, + sourceNode?.type || '', + targetNode?.type || '', + true, + onEdgeClick + ); - return getUniqueFlowElements(addEdge(newEdgeData, els)) as Edge[]; + return getUniqueFlowElements( + addEdge(newEdgeData, previousEdges) + ) as Edge[]; }); - - const updatedDownStreamEdges = (pre: EntityLineage) => { - if (edgeType !== 'downstream') { - return pre.downstreamEdges; - } - - const isExist = pre.downstreamEdges?.find( - (e) => e.fromEntity === source && e.toEntity === target - ); - - if (!isUndefined(isExist)) { - const updatedEdge: EntityLineage['downstreamEdges'] = []; - pre.downstreamEdges?.forEach((e) => { - if (e.fromEntity === source && e.toEntity === target) { - updatedEdge.push({ - ...e, - lineageDetails: newEdge.edge.lineageDetails, - }); - } else { - updatedEdge.push(e); - } - }); - - return updatedEdge; - } - - return [ - ...(pre.downstreamEdges || []), - { - fromEntity: sourceNode?.id as string, - toEntity: targetNode?.id as string, - lineageDetails: newEdge.edge.lineageDetails, - }, - ]; - }; - - const updatedUpStreamEdges = (pre: EntityLineage) => { - if (edgeType !== 'upstream') { - return pre.upstreamEdges; - } - - const isExist = pre.upstreamEdges?.find( - (e) => e.fromEntity === source && e.toEntity === target - ); - - if (!isUndefined(isExist)) { - const updatedEdge: EntityLineage['upstreamEdges'] = []; - pre.upstreamEdges?.forEach((e) => { - if (e.fromEntity === source && e.toEntity === target) { - updatedEdge.push({ - ...e, - lineageDetails: newEdge.edge.lineageDetails, - }); - } else { - updatedEdge.push(e); - } - }); - - return updatedEdge; - } - - return [ - ...(pre.upstreamEdges || []), - { - fromEntity: sourceNode?.id as string, - toEntity: targetNode?.id as string, - lineageDetails: newEdge.edge.lineageDetails, - }, - ]; - }; - - const getUpdatedNodes = (pre: EntityLineage) => { - return !isEmpty(selectedEntity) - ? [...(pre.nodes || []), selectedEntity] - : pre.nodes; - }; - - setTimeout(() => { - addLineageHandler(newEdge) - .then(() => { - setStatus('success'); - setLoading(false); - setUpdatedLineageData((pre) => { - const newData = { - ...pre, - nodes: getUpdatedNodes(pre), - downstreamEdges: updatedDownStreamEdges(pre), - upstreamEdges: updatedUpStreamEdges(pre), - }; - - return newData; - }); - setTimeout(() => { - setStatus('initial'); - }, 100); - setNewAddedNode({} as Node); - setSelectedEntity({} as EntityReference); - }) - .catch(() => { - setStatus('initial'); - setLoading(false); - }); - }, 500); } + + setEdges((previousEdges) => { + const newEdgeData = createNewEdge( + params, + isEditMode, + sourceNode?.type || '', + targetNode?.type || '', + false, + onEdgeClick + ); + + return getUniqueFlowElements( + addEdge(newEdgeData, previousEdges) + ) as Edge[]; + }); + + const updatedStreamEdges = ( + pre: EntityLineage['downstreamEdges'], + type: EdgeTypeEnum + ) => { + if (edgeType !== type) { + return pre; + } + + const isExist = pre?.find( + (e) => e.fromEntity === source && e.toEntity === target + ); + + if (!isUndefined(isExist)) { + return getUpdatedEdge( + pre || [], + params, + newEdge.edge.lineageDetails + ); + } + + return [ + ...(pre || []), + { + fromEntity: sourceNode?.id as string, + toEntity: targetNode?.id as string, + lineageDetails: newEdge.edge.lineageDetails, + }, + ]; + }; + + setTimeout(() => { + addLineageHandler(newEdge) + .then(() => { + setStatus('success'); + setLoading(false); + setUpdatedLineageData((pre) => { + const newData = { + ...pre, + nodes: getUpdatedNodes(pre), + downstreamEdges: updatedStreamEdges( + pre.downstreamEdges, + EdgeTypeEnum.DOWN_STREAM + ), + upstreamEdges: updatedStreamEdges( + pre.upstreamEdges, + EdgeTypeEnum.UP_STREAM + ), + }; + + return newData; + }); + setTimeout(() => { + setStatus('initial'); + }, 100); + resetSelectedData(); + }) + .catch(() => { + setStatus('initial'); + setLoading(false); + }); + }, 500); } }, [selectedNode, updatedLineageData, selectedEntity] @@ -942,57 +780,6 @@ const EntityLineageComponent: FunctionComponent = ({ } }; - /** - * take node and remove it from the graph - * @param node - */ - const removeNodeHandler = useCallback( - (node: Node) => { - // Get all edges for the flow - // const edges = elements.filter((element) => isEdge(element)); - - // Get edges connected to selected node - const edgesToRemove = getConnectedEdges([node], edges as Edge[]); - - edgesToRemove.forEach((edge) => { - let targetNode = updatedLineageData.nodes?.find((n) => - edge.target?.includes(n.id) - ); - - let sourceNode = updatedLineageData.nodes?.find((n) => - edge.source?.includes(n.id) - ); - - if (isUndefined(targetNode)) { - targetNode = isEmpty(selectedEntity) - ? updatedLineageData.entity - : selectedEntity; - } - if (isUndefined(sourceNode)) { - sourceNode = isEmpty(selectedEntity) - ? updatedLineageData.entity - : selectedEntity; - } - - removeEdgeHandler( - { - id: edge.id, - source: sourceNode, - target: targetNode, - }, - true - ); - }); - - setNodes( - (es) => - getUniqueFlowElements(es.filter((n) => n.id !== node.id)) as Node[] - ); - setNewAddedNode({} as Node); - }, - [nodes, updatedLineageData] - ); - /** * handle node drag event * @param event @@ -1040,7 +827,7 @@ const EntityLineageComponent: FunctionComponent = ({ /> @@ -1079,8 +866,7 @@ const EntityLineageComponent: FunctionComponent = ({ ) .filter((es) => es.id !== newAddedNode.id) ); - setNewAddedNode({} as Node); - setSelectedEntity({} as EntityReference); + resetSelectedData(); } else { setNodes((es) => { return es.map((el) => { @@ -1131,116 +917,10 @@ const EntityLineageComponent: FunctionComponent = ({ }, 500); }; - /** - * - * @returns Custom control elements - */ - const getCustomControlElements = () => { - return ( - - {!deleted && ( - { - setEditMode((pre) => !pre && !deleted); - setSelectedNode({} as SelectedNode); - setIsDrawerOpen(false); - setNewAddedNode({} as Node); - }}> - {loading ? ( - - ) : status === 'success' ? ( - - ) : ( - - )} - - )} - - ); - }; - - /** - * - * @returns Grid background if editmode is enabled otherwise null - */ - const getGraphBackGround = () => { - if (!isEditMode) { - return null; - } else { - return ; - } - }; - - /** - * - * @returns Side drawer if node is selected and view mode is enabled otherwise null - */ - const getEntityDrawer = () => { - if (isEmpty(selectedNode) || isEditMode) { - return null; - } else { - return ( - - ); - } - }; - - const getConfirmationModal = () => { - if (!showdeleteModal) { - return null; - } else { - return ( - - Cancel - - } - confirmText={ - deletionState.loading ? ( - - ) : deletionState.status === 'success' ? ( - - ) : ( - 'Confirm' - ) - } - header="Remove lineage edge" - onCancel={() => { - setShowDeleteModal(false); - }} - onConfirm={onRemove} - /> - ); - } + const handleCustomControlClick = () => { + setEditMode((pre) => !pre && !deleted); + resetSelectedData(); + setIsDrawerOpen(false); }; /** @@ -1318,9 +998,11 @@ const EntityLineageComponent: FunctionComponent = ({ return ; } - return deleted ? ( - getDeletedLineagePlaceholder() - ) : ( + if (deleted) { + return getDeletedLineagePlaceholder(); + } + + return (
@@ -1355,14 +1037,45 @@ const EntityLineageComponent: FunctionComponent = ({ onNodeMouseLeave={onNodeMouseLeave} onNodeMouseMove={onNodeMouseMove} onNodesChange={onNodesChange}> - {getCustomControlElements()} - {getGraphBackGround()} + + {isEditMode && ( + + )}
- {getEntityDrawer()} + {(!isEmpty(selectedNode) || !isEditMode) && ( + + )} - {getConfirmationModal()} + {showDeleteModal && ( + { + setShowDeleteModal(false); + }} + onOk={onRemove}> + {getModalBodyText(selectedEdge)} + + )} ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/EntityLineage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/EntityLineage.interface.ts index 051624ad99f..3d3e9aba430 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/EntityLineage.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/EntityLineage.interface.ts @@ -83,8 +83,23 @@ export interface SelectedEdge { export type ElementLoadingState = Exclude; -export type CustomeElement = { node: Node[]; edge: FlowEdge[] }; -export type CustomeFlow = Node | FlowEdge; +export type CustomElement = { node: Node[]; edge: FlowEdge[] }; +export type CustomFlow = Node | FlowEdge; export type ModifiedColumn = Column & { type: string; }; + +export interface CustomControlElementsProps { + deleted: boolean | undefined; + isEditMode: boolean; + hasEditAccess: boolean | undefined; + onClick: () => void; + loading: boolean; + status: LoadingState; +} + +export enum EdgeTypeEnum { + UP_STREAM = 'upstream', + DOWN_STREAM = 'downstream', + NO_STREAM = '', +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/Entitylineage.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/Entitylineage.component.test.tsx index 5f8495dc649..390e6b1dfb8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/Entitylineage.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/Entitylineage.component.test.tsx @@ -150,6 +150,7 @@ jest.mock('../../utils/EntityLineageUtils', () => ({

Lineage data is not available for deleted entities.

), getHeaderLabel: jest.fn().mockReturnValue(

Header label

), + LoadingStatus: jest.fn().mockReturnValue(

Confirm

), getLayoutedElements: jest.fn().mockImplementation(() => mockFlowData), getLineageData: jest.fn().mockImplementation(() => mockFlowData), getModalBodyText: jest.fn(), diff --git a/openmetadata-ui/src/main/resources/ui/src/mocks/Lineage.mock.ts b/openmetadata-ui/src/main/resources/ui/src/mocks/Lineage.mock.ts new file mode 100644 index 00000000000..3064f2591b5 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/mocks/Lineage.mock.ts @@ -0,0 +1,420 @@ +/* eslint-disable max-len */ +export const MOCK_LINEAGE_DATA = { + entity: { + id: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + type: 'table', + name: 'fact_session', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.fact_session', + description: + 'This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/f80de28c-ecce-46fb-88c7-152cc111f9ec', + }, + nodes: [ + { + id: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + type: 'table', + name: 'dim_customer', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_customer', + description: + 'The dimension table contains data about your customers. The customers table contains one row per customer. It includes historical metrics (such as the total amount that each customer has spent in your store) as well as forward-looking metrics (such as the predicted number of days between future orders and the expected order value in the next 30 days). This table also includes columns that segment customers into various categories (such as new, returning, promising, at risk, dormant, and loyal), which you can use to target marketing activities.', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/5f2eee5d-1c08-4756-af31-dabce7cb26fd', + }, + { + id: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + type: 'table', + name: 'storage_service_entity', + fullyQualifiedName: + 'mysql.default.openmetadata_db.storage_service_entity', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/92d7cb90-cc49-497a-9b01-18f4c6a61951', + }, + { + id: '2d30f754-05de-4372-af27-f221997bfe9a', + type: 'table', + name: 'dim_address', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address', + description: + 'This dimension table contains the billing and shipping addresses of customers. You can join this table with the sales table to generate lists of the billing and shipping addresses. Customers can enter their addresses more than once, so the same address can appear in more than one row in this table. This table contains one row per customer address.', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/2d30f754-05de-4372-af27-f221997bfe9a', + }, + { + id: 'b5d520fd-a4a5-4173-85d5-f804ddab452a', + type: 'table', + name: 'dashboard_service_entity', + fullyQualifiedName: + 'mysql.default.openmetadata_db.dashboard_service_entity', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/b5d520fd-a4a5-4173-85d5-f804ddab452a', + }, + { + id: 'bf99a241-76e9-4947-86a7-c9bf3c326974', + type: 'table', + name: 'dim.product', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify."dim.product"', + description: + 'This dimension table contains information about each of the products in your store. This table contains one row per product. This table reflects the current state of products in your Shopify admin.', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/bf99a241-76e9-4947-86a7-c9bf3c326974', + }, + { + id: 'd4aab894-5877-44f1-840c-b08a2dc664a4', + type: 'pipeline', + name: 'dim_address_etl', + fullyQualifiedName: 'sample_airflow.dim_address_etl', + description: 'dim_address ETL pipeline', + displayName: 'dim_address etl', + deleted: false, + href: 'http://localhost:8585/api/v1/pipelines/d4aab894-5877-44f1-840c-b08a2dc664a4', + }, + { + id: '5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + type: 'pipeline', + name: 'presto_etl', + fullyQualifiedName: 'sample_airflow.presto_etl', + description: 'Presto ETL pipeline', + displayName: 'Presto ETL', + deleted: false, + href: 'http://localhost:8585/api/v1/pipelines/5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + }, + ], + upstreamEdges: [ + { + fromEntity: '2d30f754-05de-4372-af27-f221997bfe9a', + toEntity: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + lineageDetails: { + sqlQuery: '', + columnsLineage: [ + { + fromColumns: [ + 'sample_data.ecommerce_db.shopify.dim_address.address_id', + ], + toColumn: + 'sample_data.ecommerce_db.shopify.dim_customer.total_order_value', + }, + ], + }, + }, + { + fromEntity: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + toEntity: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + lineageDetails: { + sqlQuery: '', + columnsLineage: [ + { + fromColumns: [ + 'sample_data.ecommerce_db.shopify.dim_customer.customer_id', + ], + toColumn: + 'sample_data.ecommerce_db.shopify.fact_session.derived_session_token', + }, + ], + }, + }, + { + fromEntity: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + toEntity: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + }, + { + fromEntity: 'b5d520fd-a4a5-4173-85d5-f804ddab452a', + toEntity: '2d30f754-05de-4372-af27-f221997bfe9a', + lineageDetails: { + sqlQuery: '', + columnsLineage: [ + { + fromColumns: [ + 'mysql.default.openmetadata_db.dashboard_service_entity.id', + ], + toColumn: 'sample_data.ecommerce_db.shopify.dim_address.shop_id', + }, + ], + }, + }, + { + fromEntity: 'bf99a241-76e9-4947-86a7-c9bf3c326974', + toEntity: '2d30f754-05de-4372-af27-f221997bfe9a', + lineageDetails: { + sqlQuery: '', + columnsLineage: [ + { + fromColumns: [ + 'sample_data.ecommerce_db.shopify."dim.product".shop_id', + ], + toColumn: 'sample_data.ecommerce_db.shopify.dim_address.first_name', + }, + ], + }, + }, + { + fromEntity: 'd4aab894-5877-44f1-840c-b08a2dc664a4', + toEntity: '2d30f754-05de-4372-af27-f221997bfe9a', + }, + { + fromEntity: '5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + toEntity: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + }, + ], + downstreamEdges: [ + { + toEntity: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + fromEntity: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + }, + ], +}; +export const SELECTED_EDGE = { + id: 'column-sample_data.ecommerce_db.shopify.dim_customer.customer_id-sample_data.ecommerce_db.shopify.fact_session.derived_session_token-edge-5f2eee5d-1c08-4756-af31-dabce7cb26fd-f80de28c-ecce-46fb-88c7-152cc111f9ec', + source: { + id: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + type: 'table', + name: 'dim_customer', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_customer', + description: + 'The dimension table contains data about your customers. The customers table contains one row per customer. It includes historical metrics (such as the total amount that each customer has spent in your store) as well as forward-looking metrics (such as the predicted number of days between future orders and the expected order value in the next 30 days). This table also includes columns that segment customers into various categories (such as new, returning, promising, at risk, dormant, and loyal), which you can use to target marketing activities.', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/5f2eee5d-1c08-4756-af31-dabce7cb26fd', + }, + target: { + id: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + type: 'table', + name: 'fact_session', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.fact_session', + description: + 'This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/f80de28c-ecce-46fb-88c7-152cc111f9ec', + }, + data: { + id: 'column-sample_data.ecommerce_db.shopify.dim_customer.customer_id-sample_data.ecommerce_db.shopify.fact_session.derived_session_token-edge-5f2eee5d-1c08-4756-af31-dabce7cb26fd-f80de28c-ecce-46fb-88c7-152cc111f9ec', + source: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + target: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + targetHandle: + 'sample_data.ecommerce_db.shopify.fact_session.derived_session_token', + sourceHandle: 'sample_data.ecommerce_db.shopify.dim_customer.customer_id', + isColumnLineage: true, + }, +}; +export const UP_STREAM_EDGE = { + fromEntity: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + toEntity: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + lineageDetails: { + sqlQuery: '', + columnsLineage: [ + { + fromColumns: [ + 'sample_data.ecommerce_db.shopify.dim_customer.customer_id', + ], + toColumn: + 'sample_data.ecommerce_db.shopify.fact_session.derived_session_token', + }, + ], + }, +}; + +export const COLUMN_LINEAGE_DETAILS = { + sqlQuery: '', + columnsLineage: [ + { + fromColumns: ['sample_data.ecommerce_db.shopify.dim_address.address_id'], + toColumn: + 'sample_data.ecommerce_db.shopify.dim_customer.total_order_value', + }, + ], +}; + +export const UPDATED_LINEAGE_EDGE = [ + { + fromEntity: '2d30f754-05de-4372-af27-f221997bfe9a', + lineageDetails: { + columnsLineage: [ + { + fromColumns: [ + 'sample_data.ecommerce_db.shopify.dim_address.address_id', + ], + toColumn: + 'sample_data.ecommerce_db.shopify.dim_customer.total_order_value', + }, + ], + sqlQuery: '', + }, + toEntity: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + }, + { + fromEntity: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + lineageDetails: { + columnsLineage: [ + { + fromColumns: [ + 'sample_data.ecommerce_db.shopify.dim_address.address_id', + ], + toColumn: + 'sample_data.ecommerce_db.shopify.dim_customer.total_order_value', + }, + ], + sqlQuery: '', + }, + toEntity: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + }, + { + fromEntity: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + toEntity: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + }, + { + fromEntity: 'b5d520fd-a4a5-4173-85d5-f804ddab452a', + lineageDetails: { + columnsLineage: [ + { + fromColumns: [ + 'mysql.default.openmetadata_db.dashboard_service_entity.id', + ], + toColumn: 'sample_data.ecommerce_db.shopify.dim_address.shop_id', + }, + ], + sqlQuery: '', + }, + toEntity: '2d30f754-05de-4372-af27-f221997bfe9a', + }, + { + fromEntity: 'bf99a241-76e9-4947-86a7-c9bf3c326974', + lineageDetails: { + columnsLineage: [ + { + fromColumns: [ + 'sample_data.ecommerce_db.shopify."dim.product".shop_id', + ], + toColumn: 'sample_data.ecommerce_db.shopify.dim_address.first_name', + }, + ], + sqlQuery: '', + }, + toEntity: '2d30f754-05de-4372-af27-f221997bfe9a', + }, + { + fromEntity: 'd4aab894-5877-44f1-840c-b08a2dc664a4', + toEntity: '2d30f754-05de-4372-af27-f221997bfe9a', + }, + { + fromEntity: '5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + toEntity: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + }, +]; + +export const EDGE_TO_BE_REMOVED = { + id: 'edge-5a51ea54-8304-4fa8-a7b2-1f083ff1580c-92d7cb90-cc49-497a-9b01-18f4c6a61951', + source: '5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + target: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + type: 'buttonedge', + style: { + strokeWidth: '2px', + }, + markerEnd: { + type: 'arrowclosed', + }, + data: { + id: 'edge-5a51ea54-8304-4fa8-a7b2-1f083ff1580c-92d7cb90-cc49-497a-9b01-18f4c6a61951', + source: '5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + target: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + sourceType: 'pipeline', + targetType: 'table', + isColumnLineage: false, + }, +}; + +export const MOCK_REMOVED_NODE = { + id: 'edge-5a51ea54-8304-4fa8-a7b2-1f083ff1580c-92d7cb90-cc49-497a-9b01-18f4c6a61951', + source: { + id: '5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + type: 'pipeline', + name: 'presto_etl', + fullyQualifiedName: 'sample_airflow.presto_etl', + description: 'Presto ETL pipeline', + displayName: 'Presto ETL', + deleted: false, + href: 'http://localhost:8585/api/v1/pipelines/5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + }, + target: { + id: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + type: 'table', + name: 'storage_service_entity', + fullyQualifiedName: 'mysql.default.openmetadata_db.storage_service_entity', + deleted: false, + href: 'http://localhost:8585/api/v1/tables/92d7cb90-cc49-497a-9b01-18f4c6a61951', + }, +}; + +export const MOCK_PARAMS_FOR_UP_STREAM = { + source: '5a51ea54-8304-4fa8-a7b2-1f083ff1580c', + sourceHandle: null, + target: '92d7cb90-cc49-497a-9b01-18f4c6a61951', + targetHandle: null, +}; + +export const MOCK_PARAMS_FOR_DOWN_STREAM = { + source: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + sourceHandle: null, + target: 'b32555fc-f38b-4e4b-9dbf-f156fa8ba3c9', + targetHandle: null, +}; + +export const UPDATED_COLUMN_LINEAGE = { + sqlQuery: '', + columnsLineage: [ + { + fromColumns: [ + 'sample_data.ecommerce_db.shopify.dim_customer.customer_id', + ], + toColumn: + 'sample_data.ecommerce_db.shopify.fact_session.derived_session_token', + }, + { + fromColumns: ['sample_data.ecommerce_db.shopify.dim_customer.shop_id'], + toColumn: 'sample_data.ecommerce_db.shopify.fact_session.shop_id', + }, + ], +}; + +export const UPDATED_EDGE_PARAM = { + source: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + sourceHandle: 'sample_data.ecommerce_db.shopify.dim_customer.shop_id', + target: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + targetHandle: 'sample_data.ecommerce_db.shopify.fact_session.shop_id', +}; + +export const MOCK_COLUMN_LINEAGE_EDGE = { + data: { + id: 'column-sample_data.ecommerce_db.shopify.dim_customer.shop_id-sample_data.ecommerce_db.shopify.fact_session.shop_id-edge-5f2eee5d-1c08-4756-af31-dabce7cb26fd-f80de28c-ecce-46fb-88c7-152cc111f9ec', + isColumnLineage: true, + source: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + sourceHandle: 'sample_data.ecommerce_db.shopify.dim_customer.shop_id', + sourceType: 'table', + target: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + targetHandle: 'sample_data.ecommerce_db.shopify.fact_session.shop_id', + targetType: 'table', + }, + id: 'column-sample_data.ecommerce_db.shopify.dim_customer.shop_id-sample_data.ecommerce_db.shopify.fact_session.shop_id-edge-5f2eee5d-1c08-4756-af31-dabce7cb26fd-f80de28c-ecce-46fb-88c7-152cc111f9ec', + markerEnd: { type: 'arrowclosed' }, + source: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + sourceHandle: 'sample_data.ecommerce_db.shopify.dim_customer.shop_id', + style: undefined, + target: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + targetHandle: 'sample_data.ecommerce_db.shopify.fact_session.shop_id', + type: 'buttonedge', +}; + +export const MOCK_NORMAL_LINEAGE_EDGE = { + id: 'edge-5f2eee5d-1c08-4756-af31-dabce7cb26fd-f80de28c-ecce-46fb-88c7-152cc111f9ec', + source: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + target: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + type: 'buttonedge', + style: { strokeWidth: '2px' }, + markerEnd: { type: 'arrowclosed' }, + data: { + id: 'edge-5f2eee5d-1c08-4756-af31-dabce7cb26fd-f80de28c-ecce-46fb-88c7-152cc111f9ec', + source: '5f2eee5d-1c08-4756-af31-dabce7cb26fd', + target: 'f80de28c-ecce-46fb-88c7-152cc111f9ec', + sourceType: 'table', + targetType: 'table', + isColumnLineage: false, + }, +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/styles/app.less b/openmetadata-ui/src/main/resources/ui/src/styles/app.less index ef8bc84003d..6b6bc0a9146 100644 --- a/openmetadata-ui/src/main/resources/ui/src/styles/app.less +++ b/openmetadata-ui/src/main/resources/ui/src/styles/app.less @@ -15,6 +15,8 @@ @primary: #7147e8; @primary-light: rgb(244, 240, 253); +@primary-hover-light: rgba(219, 209, 249); +@white: #fff; //font weight .font-medium { @@ -43,6 +45,9 @@ .error-text { color: #ff4c3b; } +.text-white { + color: @white; +} // text alignment @@ -60,6 +65,9 @@ .w-8 { width: 32px; } +.w-9 { + width: 36px; +} .w-16 { width: 64px; } @@ -110,6 +118,9 @@ .h-7 { height: 28px; } +.h-9 { + height: 36px; +} .h-min-100 { min-height: 100vh; } @@ -180,9 +191,19 @@ border-color: @primary; } +.rounded-full { + border-radius: 9999px; +} + .bg-primary-lite { background: @primary-light; } +.bg-primary { + background: @primary; +} +.bg-primary-hover-lite { + background-color: @primary-hover-light; +} .activeCategory { border-left: 2px solid @primary; diff --git a/openmetadata-ui/src/main/resources/ui/src/styles/position.less b/openmetadata-ui/src/main/resources/ui/src/styles/position.less index 628c27038da..841a694f8a5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/styles/position.less +++ b/openmetadata-ui/src/main/resources/ui/src/styles/position.less @@ -36,3 +36,18 @@ .flex-1 { flex: 1; } + +//top +.top-1 { + top: 4px; +} + +//right +.right-3 { + right: 12px; +} + +//bottom +.bottom-full { + bottom: 100%; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/styles/spacing.less b/openmetadata-ui/src/main/resources/ui/src/styles/spacing.less index 0d0fd4f9e83..02aeb2d1182 100644 --- a/openmetadata-ui/src/main/resources/ui/src/styles/spacing.less +++ b/openmetadata-ui/src/main/resources/ui/src/styles/spacing.less @@ -251,6 +251,10 @@ .p-lg { padding: @padding-lg; } +.p-x-xss { + padding-right: @padding-xss; + padding-left: @padding-xss; +} .p-x-xs { padding-right: @padding-xs; padding-left: @padding-xs; diff --git a/openmetadata-ui/src/main/resources/ui/src/styles/x-master.css b/openmetadata-ui/src/main/resources/ui/src/styles/x-master.css index d125b1d673a..a3ef2e6954c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/styles/x-master.css +++ b/openmetadata-ui/src/main/resources/ui/src/styles/x-master.css @@ -69,10 +69,6 @@ pre { color: #c45296 !important; } -.bg-primary { - color: #ffffff; - background-color: #2eaadc !important; -} .bg-success { color: #ffffff; background-color: #28a745 !important; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx new file mode 100644 index 00000000000..a3840bac576 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx @@ -0,0 +1,170 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Edge } from 'reactflow'; +import { + EdgeTypeEnum, + SelectedEdge, +} from '../components/EntityLineage/EntityLineage.interface'; +import { LineageDetails } from '../generated/api/lineage/addLineage'; +import { EntityReference } from '../generated/type/entityReference'; +import { + COLUMN_LINEAGE_DETAILS, + EDGE_TO_BE_REMOVED, + MOCK_COLUMN_LINEAGE_EDGE, + MOCK_LINEAGE_DATA, + MOCK_NORMAL_LINEAGE_EDGE, + MOCK_PARAMS_FOR_DOWN_STREAM, + MOCK_PARAMS_FOR_UP_STREAM, + MOCK_REMOVED_NODE, + SELECTED_EDGE, + UPDATED_COLUMN_LINEAGE, + UPDATED_EDGE_PARAM, + UPDATED_LINEAGE_EDGE, + UP_STREAM_EDGE, +} from '../mocks/Lineage.mock'; +import { + createNewEdge, + findUpstreamDownStreamEdge, + getEdgeType, + getRemovedNodeData, + getUpdatedEdge, + getUpdatedUpstreamDownStreamEdgeArr, + getUpStreamDownStreamColumnLineageArr, +} from './EntityLineageUtils'; + +describe('Test EntityLineageUtils utility', () => { + it('findUpstreamDownStreamEdge function should work properly', () => { + const upstreamData = findUpstreamDownStreamEdge( + MOCK_LINEAGE_DATA.upstreamEdges, + SELECTED_EDGE as SelectedEdge + ); + const nodata = findUpstreamDownStreamEdge( + undefined, + SELECTED_EDGE as SelectedEdge + ); + + expect(upstreamData).toStrictEqual(UP_STREAM_EDGE); + expect(nodata).toStrictEqual(undefined); + }); + + it('getUpStreamDownStreamColumnLineageArr function should work properly', () => { + const columnLineageData = getUpStreamDownStreamColumnLineageArr( + MOCK_LINEAGE_DATA.upstreamEdges[0].lineageDetails as LineageDetails, + SELECTED_EDGE as SelectedEdge + ); + const nodata = getUpStreamDownStreamColumnLineageArr( + MOCK_LINEAGE_DATA.upstreamEdges[1].lineageDetails as LineageDetails, + SELECTED_EDGE as SelectedEdge + ); + + expect(columnLineageData).toStrictEqual(COLUMN_LINEAGE_DETAILS); + expect(nodata).toStrictEqual({ sqlQuery: '', columnsLineage: [] }); + }); + + it('getUpdatedUpstreamDownStreamEdgeArr function should work properly', () => { + const columnLineageData = getUpdatedUpstreamDownStreamEdgeArr( + MOCK_LINEAGE_DATA.upstreamEdges, + SELECTED_EDGE as SelectedEdge, + COLUMN_LINEAGE_DETAILS + ); + const nodata = getUpdatedUpstreamDownStreamEdgeArr( + [], + SELECTED_EDGE as SelectedEdge, + COLUMN_LINEAGE_DETAILS + ); + + expect(columnLineageData).toStrictEqual(UPDATED_LINEAGE_EDGE); + expect(nodata).toStrictEqual([]); + }); + + it('getRemovedNodeData function should work properly', () => { + const data = getRemovedNodeData( + MOCK_LINEAGE_DATA.nodes, + EDGE_TO_BE_REMOVED as Edge, + MOCK_LINEAGE_DATA.entity, + MOCK_LINEAGE_DATA.nodes[0] + ); + const nodata = getRemovedNodeData( + [], + SELECTED_EDGE.data, + MOCK_LINEAGE_DATA.entity, + {} as EntityReference + ); + + expect(data).toStrictEqual(MOCK_REMOVED_NODE); + expect(nodata).toStrictEqual({ + id: SELECTED_EDGE.data.id, + source: MOCK_LINEAGE_DATA.entity, + target: MOCK_LINEAGE_DATA.entity, + }); + }); + + it('getEdgeType function should work properly', () => { + const upStreamData = getEdgeType( + MOCK_LINEAGE_DATA, + MOCK_PARAMS_FOR_UP_STREAM + ); + const downStreamData = getEdgeType( + MOCK_LINEAGE_DATA, + MOCK_PARAMS_FOR_DOWN_STREAM + ); + + expect(upStreamData).toStrictEqual(EdgeTypeEnum.UP_STREAM); + expect(downStreamData).toStrictEqual(EdgeTypeEnum.DOWN_STREAM); + }); + + it('getUpdatedEdge function should work properly', () => { + const node = MOCK_LINEAGE_DATA.upstreamEdges[1]; + const data = getUpdatedEdge( + [node], + UPDATED_EDGE_PARAM, + UPDATED_COLUMN_LINEAGE + ); + + expect(data).toStrictEqual([ + { + ...node, + lineageDetails: UPDATED_COLUMN_LINEAGE, + }, + ]); + }); + + it('createNewEdge function should work properly', () => { + const columnLineageEdge = createNewEdge( + UPDATED_EDGE_PARAM, + true, + 'table', + 'table', + true, + jest.fn + ); + const normalLineageEdge = createNewEdge( + UPDATED_EDGE_PARAM, + true, + 'table', + 'table', + false, + jest.fn + ); + + const updatedColLineageEdge = MOCK_COLUMN_LINEAGE_EDGE as Edge; + updatedColLineageEdge.data.onEdgeClick = jest.fn; + + const updatedNormalLineageEdge = MOCK_NORMAL_LINEAGE_EDGE as Edge; + updatedNormalLineageEdge.data.onEdgeClick = jest.fn; + + expect(columnLineageEdge).toMatchObject(updatedColLineageEdge); + expect(normalLineageEdge).toMatchObject(updatedNormalLineageEdge); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index 4480620f384..a9e2f7848ee 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -17,15 +17,24 @@ import { } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import dagre from 'dagre'; -import { isUndefined } from 'lodash'; -import { LeafNodes, LineagePos, LoadingNodeState } from 'Models'; +import { isEmpty, isNil, isUndefined } from 'lodash'; +import { LeafNodes, LineagePos, LoadingNodeState, LoadingState } from 'Models'; import React, { Fragment, MouseEvent as ReactMouseEvent } from 'react'; import { Link } from 'react-router-dom'; -import { Edge, MarkerType, Node, Position, ReactFlowInstance } from 'reactflow'; +import { + Connection, + Edge, + MarkerType, + Node, + Position, + ReactFlowInstance, +} from 'reactflow'; import { CustomEdgeData, - CustomeElement, - CustomeFlow, + CustomElement, + CustomFlow, + EdgeData, + EdgeTypeEnum, ModifiedColumn, SelectedEdge, SelectedNode, @@ -44,7 +53,12 @@ import { FqnPart, } from '../enums/entity.enum'; import { Column } from '../generated/entity/data/table'; -import { EntityLineage } from '../generated/type/entityLineage'; +import { + ColumnLineage, + Edge as EntityLineageEdge, + EntityLineage, + LineageDetails, +} from '../generated/type/entityLineage'; import { EntityReference } from '../generated/type/entityReference'; import { getPartialNameFromFQN, @@ -420,7 +434,7 @@ const dagreGraph = new dagre.graphlib.Graph(); dagreGraph.setDefaultEdgeLabel(() => ({})); export const getLayoutedElements = ( - elements: CustomeElement, + elements: CustomElement, direction = EntityLineageDirection.LEFT_RIGHT ) => { const { node, edge } = elements; @@ -487,9 +501,9 @@ export const getModalBodyText = (selectedEdge: SelectedEdge) => { } and ${target.displayName ? target.displayName : targetEntity}"?`; }; -export const getUniqueFlowElements = (elements: CustomeFlow[]) => { +export const getUniqueFlowElements = (elements: CustomFlow[]) => { const flag: { [x: string]: boolean } = {}; - const uniqueElements: CustomeFlow[] = []; + const uniqueElements: CustomFlow[] = []; elements.forEach((elem) => { if (!flag[elem.id]) { @@ -515,3 +529,292 @@ export const getNodeRemoveButton = (onClick: () => void) => { ); }; + +export const getSelectedEdgeArr = ( + edgeArr: EntityLineageEdge[], + edgeData: EdgeData +) => { + return edgeArr.filter( + (edge) => + !edgeArr.find( + () => + edgeData.fromId === edge.fromEntity && edgeData.toId === edge.toEntity + ) + ); +}; + +/** + * Finds the upstream/downstream edge based on selected edge + * @param edgeArr edge[] + * @param data selected edge + * @returns edge + */ + +export const findUpstreamDownStreamEdge = ( + edgeArr: EntityLineageEdge[] | undefined, + data: SelectedEdge +) => { + return edgeArr?.find( + (edge) => + edge.fromEntity === data.source.id && edge.toEntity === data.target.id + ); +}; + +/** + * Get upstream/downstream column lineage array + * @param lineageDetails LineageDetails + * @param data SelectedEdge + * @returns Updated LineageDetails + */ + +export const getUpStreamDownStreamColumnLineageArr = ( + lineageDetails: LineageDetails, + data: SelectedEdge +) => { + const columnsLineage = lineageDetails.columnsLineage.reduce((col, curr) => { + if (curr.toColumn === data.data?.targetHandle) { + const newCol = { + ...curr, + fromColumns: + curr.fromColumns?.filter( + (column) => column !== data.data?.sourceHandle + ) || [], + }; + if (newCol.fromColumns?.length) { + return [...col, newCol]; + } else { + return col; + } + } + + return [...col, curr]; + }, [] as ColumnLineage[]); + + return { + sqlQuery: lineageDetails.sqlQuery || '', + columnsLineage: columnsLineage, + }; +}; + +/** + * Get updated EntityLineageEdge Array based on selected data + * @param edge EntityLineageEdge[] + * @param data SelectedEdge + * @param lineageDetails updated LineageDetails + * @returns updated EntityLineageEdge[] + */ +export const getUpdatedUpstreamDownStreamEdgeArr = ( + edge: EntityLineageEdge[], + data: SelectedEdge, + lineageDetails: LineageDetails +) => { + return edge.map((down) => { + if ( + down.fromEntity === data.source.id && + down.toEntity === data.target.id + ) { + return { + ...down, + lineageDetails: lineageDetails, + }; + } + + return down; + }); +}; + +/** + * Get array of the removed node + * @param nodes All the node + * @param edge selected edge + * @param entity main entity + * @param selectedEntity selected entity + * @returns details of removed node + */ +export const getRemovedNodeData = ( + nodes: EntityReference[], + edge: Edge, + entity: EntityReference, + selectedEntity: EntityReference +) => { + let targetNode = nodes.find((node) => edge.target?.includes(node.id)); + let sourceNode = nodes.find((node) => edge.source?.includes(node.id)); + const selectedNode = isEmpty(selectedEntity) ? entity : selectedEntity; + + if (isUndefined(targetNode)) { + targetNode = selectedNode; + } + if (isUndefined(sourceNode)) { + sourceNode = selectedNode; + } + + return { + id: edge.id, + source: sourceNode, + target: targetNode, + }; +}; + +/** + * Get source/target edge based on query string + * @param edge upstream/downstream edge array + * @param queryStr source/target string + * @param id main entity id + * @returns source/target edge + */ +const getSourceTargetNode = ( + edge: EntityLineageEdge[], + queryStr: string | null, + id: string +) => { + return edge.find( + (d) => + (queryStr?.includes(d.fromEntity) || queryStr?.includes(d.toEntity)) && + queryStr !== id + ); +}; + +export const getEdgeType = ( + updatedLineageData: EntityLineage, + params: Edge | Connection +) => { + const { entity } = updatedLineageData; + const { target, source } = params; + const sourceDownstreamNode = getSourceTargetNode( + updatedLineageData.downstreamEdges || [], + source, + entity.id + ); + + const sourceUpStreamNode = getSourceTargetNode( + updatedLineageData.upstreamEdges || [], + source, + entity.id + ); + + const targetDownStreamNode = getSourceTargetNode( + updatedLineageData.downstreamEdges || [], + target, + entity.id + ); + + const targetUpStreamNode = getSourceTargetNode( + updatedLineageData.upstreamEdges || [], + target, + entity.id + ); + + const isUpstream = + (!isNil(sourceUpStreamNode) && !isNil(targetDownStreamNode)) || + !isNil(sourceUpStreamNode) || + !isNil(targetUpStreamNode) || + target?.includes(entity.id); + + const isDownstream = + (!isNil(sourceDownstreamNode) && !isNil(targetUpStreamNode)) || + !isNil(sourceDownstreamNode) || + !isNil(targetDownStreamNode) || + source?.includes(entity.id); + + if (isUpstream) { + return EdgeTypeEnum.UP_STREAM; + } else if (isDownstream) { + return EdgeTypeEnum.DOWN_STREAM; + } + + return EdgeTypeEnum.NO_STREAM; +}; + +/** + * Get updated Edge with lineageDetails + * @param edges Array of Edge + * @param params new connected edge + * @param lineageDetails updated lineage details + * @returns updated edge array + */ +export const getUpdatedEdge = ( + edges: EntityLineageEdge[], + params: Edge | Connection, + lineageDetails: LineageDetails | undefined +) => { + const updatedEdge: EntityLineageEdge[] = []; + const { target, source } = params; + edges.forEach((edge) => { + if (edge.fromEntity === source && edge.toEntity === target) { + updatedEdge.push({ + ...edge, + lineageDetails: lineageDetails, + }); + } else { + updatedEdge.push(edge); + } + }); + + return updatedEdge; +}; + +// create new edge +export const createNewEdge = ( + params: Edge | Connection, + isEditMode: boolean, + sourceNodeType: string, + targetNodeType: string, + isColumnLineage: boolean, + onEdgeClick: ( + evt: React.MouseEvent, + data: CustomEdgeData + ) => void +) => { + const { target, source, sourceHandle, targetHandle } = params; + let data: Edge = { + id: `edge-${source}-${target}`, + source: `${source}`, + target: `${target}`, + type: isEditMode ? 'buttonedge' : 'custom', + style: { strokeWidth: '2px' }, + markerEnd: { + type: MarkerType.ArrowClosed, + }, + data: { + id: `edge-${source}-${target}`, + source: source, + target: target, + sourceType: sourceNodeType, + targetType: targetNodeType, + isColumnLineage: isColumnLineage, + onEdgeClick, + }, + }; + + if (isColumnLineage) { + data = { + ...data, + id: `column-${sourceHandle}-${targetHandle}-edge-${source}-${target}`, + sourceHandle: sourceHandle, + targetHandle: targetHandle, + style: undefined, + data: { + ...data.data, + id: `column-${sourceHandle}-${targetHandle}-edge-${source}-${target}`, + sourceHandle: sourceHandle, + targetHandle: targetHandle, + }, + }; + } + + return data; +}; + +export const LoadingStatus = ( + defaultState: string | JSX.Element, + loading: boolean, + status: LoadingState +) => { + if (loading) { + return ; + } else if (status === 'success') { + return ; + } else { + return defaultState; + } +};