From 7a517cd26d49a8bf6ee5639b6e182ae2ba525c13 Mon Sep 17 00:00:00 2001 From: darth-coder00 <86726556+darth-coder00@users.noreply.github.com> Date: Thu, 16 Jun 2022 03:30:58 +0530 Subject: [PATCH] Fix #4850: Airflow Dags in the Open Metadata lineage not showing the tasks flow (#5323) --- .../PipelineDetails.component.tsx | 160 +----------------- .../PipelineDetails/PipelineDetails.test.tsx | 15 +- .../TasksDAGView/TasksDAGView.test.tsx | 86 ++++++++++ .../components/TasksDAGView/TasksDAGView.tsx | 100 +++++++++++ .../resources/ui/src/utils/CommonUtils.tsx | 4 + 5 files changed, 213 insertions(+), 152 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.tsx diff --git a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx index 9844c8789b4..be86a54c7cf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.component.tsx @@ -11,12 +11,9 @@ * limitations under the License. */ -import classNames from 'classnames'; import { compare } from 'fast-json-patch'; -import { isNil, isUndefined } from 'lodash'; -import { EntityFieldThreads, EntityTags, ExtraInfo } from 'Models'; -import React, { Fragment, RefObject, useEffect, useState } from 'react'; -import { Link } from 'react-router-dom'; +import { EntityTags, ExtraInfo } from 'Models'; +import React, { RefObject, useEffect, useState } from 'react'; import AppState from '../../AppState'; import { FQN_SEPARATOR_CHAR } from '../../constants/char.constants'; import { getTeamAndUserDetailsPath } from '../../constants/constants'; @@ -24,7 +21,6 @@ import { observerOptions } from '../../constants/Mydata.constants'; import { EntityType } from '../../enums/entity.enum'; import { OwnerType } from '../../enums/user.enum'; import { Pipeline, Task } from '../../generated/entity/data/pipeline'; -import { Operation } from '../../generated/entity/policies/accessControl/rule'; import { EntityReference } from '../../generated/type/entityReference'; import { Paging } from '../../generated/type/paging'; import { LabelType, State } from '../../generated/type/tagLabel'; @@ -33,24 +29,15 @@ import { getCurrentUserId, getEntityName, getEntityPlaceHolder, - getHtmlForNonAdminAction, - isEven, } from '../../utils/CommonUtils'; import { getEntityFeedLink } from '../../utils/EntityUtils'; -import { - getDefaultValue, - getFieldThreadElement, -} from '../../utils/FeedElementUtils'; +import { getDefaultValue } from '../../utils/FeedElementUtils'; import { getEntityFieldThreadCounts } from '../../utils/FeedUtils'; -import SVGIcons, { Icons } from '../../utils/SvgUtils'; import { getTagsWithoutTier } from '../../utils/TableUtils'; import ActivityFeedList from '../ActivityFeed/ActivityFeedList/ActivityFeedList'; import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThreadPanel'; import Description from '../common/description/Description'; import EntityPageInfo from '../common/entityPageInfo/EntityPageInfo'; -import NonAdminAction from '../common/non-admin-action/NonAdminAction'; -import PopOver from '../common/popover/PopOver'; -import RichTextEditorPreviewer from '../common/rich-text-editor/RichTextEditorPreviewer'; import TabsPane from '../common/TabsPane/TabsPane'; import PageContainer from '../containers/PageContainer'; import Entitylineage from '../EntityLineage/EntityLineage.component'; @@ -59,6 +46,7 @@ import ManageTabComponent from '../ManageTab/ManageTab.component'; import { ModalWithMarkdownEditor } from '../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor'; import RequestDescriptionModal from '../Modals/RequestDescriptionModal/RequestDescriptionModal'; import PipelineStatusList from '../PipelineStatusList/PipelineStatusList.component'; +import TasksDAGView from '../TasksDAGView/TasksDAGView'; import { PipeLineDetailsProp } from './PipelineDetails.interface'; const PipelineDetails = ({ @@ -252,10 +240,6 @@ const PipelineDetails = ({ } }; - const handleUpdateTask = (task: Task, index: number) => { - setEditTask({ task, index }); - }; - const closeEditTaskModal = (): void => { setEditTask(undefined); }; @@ -384,15 +368,15 @@ const PipelineDetails = ({ versionHandler={versionHandler} onThreadLinkSelect={onThreadLinkSelect} /> -
+
-
-
+
+
{activeTab === 1 && ( <>
@@ -418,135 +402,9 @@ const PipelineDetails = ({ />
-
+
{tasks ? ( - - - - - - - - - - {tasks?.map((task, index) => ( - - - - - - ))} - -
Task NameDescriptionTask Type
- - - - {task.displayName} - - - - - -
-
- {task.description ? ( - - ) : ( - - No description{' '} - - )} -
- {!deleted && ( - - - - - {!isNil( - getFieldThreadElement( - task.name, - 'description', - getEntityFieldThreadCounts( - 'tasks', - entityFieldThreadCount - ) as EntityFieldThreads[], - onThreadLinkSelect - ) - ) && - !isUndefined(onEntityFieldSelect) && - !task.description ? ( - - ) : null} - {getFieldThreadElement( - task.name, - 'description', - getEntityFieldThreadCounts( - 'tasks', - entityFieldThreadCount - ) as EntityFieldThreads[], - onThreadLinkSelect, - EntityType.PIPELINE, - pipelineFQN, - `tasks/${task.name}/description`, - Boolean(task.description) - )} - - )} -
- 		- {task.taskType} -
+ ) : (
No task data is available diff --git a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx index 3b9831397a1..1753d11b497 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/PipelineDetails/PipelineDetails.test.tsx @@ -22,6 +22,15 @@ import { Paging } from '../../generated/type/paging'; import { TagLabel } from '../../generated/type/tagLabel'; import PipelineDetails from './PipelineDetails.component'; +/** + * mock implementation of ResizeObserver + */ +window.ResizeObserver = jest.fn().mockImplementation(() => ({ + observe: jest.fn(), + unobserve: jest.fn(), + disconnect: jest.fn(), +})); + jest.mock('../../authentication/auth-provider/AuthProvider', () => { return { useAuthContext: jest.fn(() => ({ @@ -149,6 +158,10 @@ jest.mock('../PipelineStatusList/PipelineStatusList.component', () => { .mockReturnValue(

Pipeline Status

); }); +jest.mock('../TasksDAGView/TasksDAGView', () => { + return jest.fn().mockReturnValue(

Tasks DAG

); +}); + jest.mock('../../utils/CommonUtils', () => ({ addToRecentViewed: jest.fn(), getCountBadge: jest.fn(), @@ -192,7 +205,7 @@ describe('Test PipelineDetails component', () => { wrapper: MemoryRouter, } ); - const taskDetail = await findByTestId(container, 'tasks-table'); + const taskDetail = await findByTestId(container, 'tasks-dag'); expect(taskDetail).toBeInTheDocument(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx new file mode 100644 index 00000000000..8473ff669c1 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.test.tsx @@ -0,0 +1,86 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { findByTestId, render } from '@testing-library/react'; +import React from 'react'; +import { MemoryRouter } from 'react-router-dom'; +import TasksDAGView from './TasksDAGView'; + +/** + * mock implementation of ResizeObserver + */ +window.ResizeObserver = jest.fn().mockImplementation(() => ({ + observe: jest.fn(), + unobserve: jest.fn(), + disconnect: jest.fn(), +})); + +jest.mock('../../authentication/auth-provider/AuthProvider', () => { + return { + useAuthContext: jest.fn(() => ({ + isAuthDisabled: false, + isAuthenticated: true, + isProtectedRoute: jest.fn().mockReturnValue(true), + isTourRoute: jest.fn().mockReturnValue(false), + onLogoutHandler: jest.fn(), + })), + }; +}); + +const TasksDAGViewProps = { + tasks: [ + { + name: 'task1', + }, + { + name: 'task2', + }, + { + name: 'task3', + }, + ], +}; + +jest.mock('../../utils/EntityLineageUtils', () => ({ + dragHandle: jest.fn(), + getDataLabel: jest + .fn() + .mockReturnValue(datalabel), + getDeletedLineagePlaceholder: jest + .fn() + .mockReturnValue(

Task data is not available for deleted entities.

), + getHeaderLabel: jest.fn().mockReturnValue(

Header label

), + getLayoutedElements: jest.fn().mockReturnValue([]), + getLineageData: jest.fn().mockReturnValue([]), + getModalBodyText: jest.fn(), + onLoad: jest.fn(), + onNodeContextMenu: jest.fn(), + onNodeMouseEnter: jest.fn(), + onNodeMouseLeave: jest.fn(), + onNodeMouseMove: jest.fn(), + getUniqueFlowElements: jest.fn().mockReturnValue([]), +})); + +describe('Test PipelineDetails component', () => { + it('Checks if the PipelineDetails component has all the proper components rendered', async () => { + const { container } = render(, { + wrapper: MemoryRouter, + }); + const reactFlowElement = await findByTestId( + container, + 'react-flow-component' + ); + + expect(reactFlowElement).toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.tsx new file mode 100644 index 00000000000..03934c7d151 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/TasksDAGView/TasksDAGView.tsx @@ -0,0 +1,100 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import ReactFlow, { + ArrowHeadType, + Edge, + Elements, + Node, +} from 'react-flow-renderer'; +import { Task } from '../../generated/entity/data/pipeline'; +import { EntityReference } from '../../generated/type/entityReference'; +import { getEntityName, replaceSpaceWith_ } from '../../utils/CommonUtils'; +import { getLayoutedElements, onLoad } from '../../utils/EntityLineageUtils'; + +export interface Props { + tasks: Task[]; +} + +const TasksDAGView = ({ tasks }: Props) => { + const [elements, setElements] = useState([]); + + const getNodeType = useCallback( + (index: number) => { + return index === 0 + ? 'input' + : index === tasks.length - 1 + ? 'output' + : 'default'; + }, + [tasks] + ); + + const nodes: Node[] = useMemo(() => { + const posY = 0; + let posX = 0; + const deltaX = 250; + + return tasks.map((task, index) => { + posX += deltaX; + + return { + className: 'leaf-node', + id: replaceSpaceWith_(task.name), + type: getNodeType(index), + data: { label: getEntityName(task as EntityReference) }, + position: { x: posX, y: posY }, + }; + }); + }, [tasks]); + + const edges: Edge[] = useMemo(() => { + return tasks.reduce((prev, task) => { + const src = replaceSpaceWith_(task.name); + const taskEdges = (task.downstreamTasks || []).map((dwTask) => { + const dest = replaceSpaceWith_(dwTask); + + return { + arrowHeadType: ArrowHeadType.ArrowClosed, + id: `${src}-${dest}`, + type: 'straight', + source: src, + target: dest, + label: '', + } as Edge; + }); + + return [...prev, ...taskEdges]; + }, [] as Edge[]); + }, [tasks]); + + useEffect(() => { + setElements(getLayoutedElements([...nodes, ...edges])); + }, [nodes, edges]); + + return ( + + ); +}; + +export default TasksDAGView; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx index 628bb23f28a..90ebda95d11 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx @@ -627,3 +627,7 @@ export const getExploreLinkByFilter = ( `${filter}=${getOwnerIds(filter, userDetails, nonSecureUserDetails).join()}` ); }; + +export const replaceSpaceWith_ = (text: string) => { + return text.replace(/\s/g, '_'); +};