Allow adding pipeline between all entities in lineage (#16960)

* fix tier information

* add tests

* fix tests

* update test as slow

openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts

@@ -28,15 +28,16 @@ import {
   activateColumnLayer,
   addColumnLineage,
   addPipelineBetweenNodes,
+  applyPipelineFromModal,
   connectEdgeBetweenNodes,
   deleteEdge,
   deleteNode,
-  editPipelineEdgeDescription,
   performZoomOut,
   removeColumnLineage,
   setupEntitiesForLineage,
   verifyColumnLayerInactive,
   verifyNodePresent,
+  visitLineageTab,
 } from '../../utils/lineage';
 
 // use the admin user to login
@@ -51,12 +52,28 @@ const entities = [
   SearchIndexClass,
 ] as const;
 
+const pipeline = new PipelineClass();
+
+test.beforeAll('Setup pre-requests', async ({ browser }) => {
+  const { apiContext, afterAction } = await createNewPage(browser);
+  await pipeline.create(apiContext);
+  await afterAction();
+});
+
+test.afterAll('Cleanup', async ({ browser }) => {
+  const { apiContext, afterAction } = await createNewPage(browser);
+  await pipeline.delete(apiContext);
+  await afterAction();
+});
+
 for (const EntityClass of entities) {
   const defaultEntity = new EntityClass();
 
   test(`Lineage creation from ${defaultEntity.getType()} entity`, async ({
     browser,
   }) => {
+    test.slow(true);
+
     const { page } = await createNewPage(browser);
     const { currentEntity, entities, cleanup } = await setupEntitiesForLineage(
       page,
@@ -66,7 +83,7 @@ for (const EntityClass of entities) {
     await test.step('Should create lineage for the entity', async () => {
       await redirectToHomePage(page);
       await currentEntity.visitEntityPage(page);
-      await page.click('[data-testid="lineage"]');
+      await visitLineageTab(page);
       await verifyColumnLayerInactive(page);
       await page.click('[data-testid="edit-lineage"]');
       await performZoomOut(page);
@@ -76,15 +93,29 @@ for (const EntityClass of entities) {
 
       await redirectToHomePage(page);
       await currentEntity.visitEntityPage(page);
-      await page.click('[data-testid="lineage"]');
+      await visitLineageTab(page);
-      await page.click('.react-flow__controls-fitview', { force: true });
+      await page
+        .locator('.react-flow__controls-fitview')
+        .dispatchEvent('click');
 
       for (const entity of entities) {
         await verifyNodePresent(page, entity);
       }
     });
 
+    await test.step('Should create pipeline between entities', async () => {
+      await page.click('[data-testid="edit-lineage"]');
+      await performZoomOut(page);
+
+      for (const entity of entities) {
+        await applyPipelineFromModal(page, currentEntity, entity, pipeline);
+      }
+    });
+
     await test.step('Remove lineage between nodes for the entity', async () => {
+      await redirectToHomePage(page);
+      await currentEntity.visitEntityPage(page);
+      await visitLineageTab(page);
       await page.click('[data-testid="edit-lineage"]');
       await performZoomOut(page);
 
@@ -97,56 +128,6 @@ for (const EntityClass of entities) {
   });
 }
 
-test('Lineage Add Pipeline Between Tables', async ({ browser }) => {
-  const { page } = await createNewPage(browser);
-  const { apiContext, afterAction } = await getApiContext(page);
-  const table1 = new TableClass();
-  const table2 = new TableClass();
-  const pipeline = new PipelineClass();
-  await table1.create(apiContext);
-  await table2.create(apiContext);
-  await pipeline.create(apiContext);
-  await redirectToHomePage(page);
-
-  await addPipelineBetweenNodes(page, table1, table2, pipeline, true);
-  await page.click('[data-testid="edit-lineage"]');
-  await deleteNode(page, table2);
-
-  await table1.delete(apiContext);
-  await table2.delete(apiContext);
-  await pipeline.delete(apiContext);
-
-  await afterAction();
-});
-
-test('Lineage Pipeline Between Table and Topic', async ({ browser }) => {
-  const { page } = await createNewPage(browser);
-  const { apiContext, afterAction } = await getApiContext(page);
-  const table = new TableClass();
-  const topic = new TopicClass();
-  const pipeline = new PipelineClass();
-  await table.create(apiContext);
-  await topic.create(apiContext);
-  await pipeline.create(apiContext);
-  await redirectToHomePage(page);
-
-  await addPipelineBetweenNodes(page, table, topic, pipeline, true);
-  await editPipelineEdgeDescription(
-    page,
-    table,
-    topic,
-    pipeline,
-    'Test Description'
-  );
-  await page.click('[data-testid="edit-lineage"]');
-  await deleteNode(page, topic);
-
-  await table.delete(apiContext);
-  await topic.delete(apiContext);
-  await pipeline.delete(apiContext);
-  await afterAction();
-});
-
 test('Verify column lineage between tables', async ({ browser }) => {
   const { page } = await createNewPage(browser);
   const { apiContext, afterAction } = await getApiContext(page);

openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts

@@ -53,23 +53,19 @@ export const deleteEdge = async (
   const fromNodeFqn = get(fromNode, 'entityResponseData.fullyQualifiedName');
   const toNodeFqn = get(toNode, 'entityResponseData.fullyQualifiedName');
 
-  page.click(`[data-testid="edge-${fromNodeFqn}-${toNodeFqn}"]`, {
+  await page
-    force: true,
+    .locator(`[data-testid="edge-${fromNodeFqn}-${toNodeFqn}"]`)
-  });
+    .dispatchEvent('click');
 
-  if (
+  await page.locator('[data-testid="add-pipeline"]').dispatchEvent('click');
-    ['Table', 'Topic'].indexOf(fromNode.getType()) > -1 &&
+
-    ['Table', 'Topic'].indexOf(toNode.getType()) > -1
+  await expect(page.locator('[role="dialog"]')).toBeVisible();
-  ) {
+
-    await page.locator('[data-testid="add-pipeline"]').dispatchEvent('click');
+  await page
-    await page
+    .locator(
-      .locator(
+      '[data-testid="add-edge-modal"] [data-testid="remove-edge-button"]'
-        '[data-testid="add-edge-modal"] [data-testid="remove-edge-button"]'
+    )
-      )
+    .dispatchEvent('click');
-      .dispatchEvent('click');
-  } else {
-    await page.locator('[data-testid="delete-button"]').dispatchEvent('click');
-  }
 
   await expect(page.locator('[role="dialog"]')).toBeVisible();
 
@@ -118,8 +114,14 @@ export const connectEdgeBetweenNodes = async (
 
   await page.locator('[data-testid="suggestion-node"]').dispatchEvent('click');
 
+  const waitForSearchResponse = page.waitForResponse(
+    `/api/v1/search/query?q=*&from=0&size=10&*`
+  );
+
   await page.locator('[data-testid="suggestion-node"] input').fill(toNodeName);
 
+  await waitForSearchResponse;
+
   await page
     .locator(`[data-testid="node-suggestion-${toNodeFqn}"]`)
     .dispatchEvent('click');
@@ -142,7 +144,7 @@ export const verifyNodePresent = async (page: Page, node: EntityClass) => {
     '[data-testid="entity-header-name"]'
   );
 
-  expect(entityHeaderName).toHaveText(name);
+  await expect(entityHeaderName).toHaveText(name);
 };
 
 export const setupEntitiesForLineage = async (
@@ -257,16 +259,20 @@ export const applyPipelineFromModal = async (
     'entityResponseData.fullyQualifiedName'
   );
 
-  await page.click(`[data-testid="edge-${fromNodeFqn}-${toNodeFqn}"]`, {
+  await page
-    force: true,
+    .locator(`[data-testid="edge-${fromNodeFqn}-${toNodeFqn}"]`)
-  });
+    .dispatchEvent('click');
+  await page.locator('[data-testid="add-pipeline"]').dispatchEvent('click');
 
-  await page.click('[data-testid="add-pipeline"]');
+  const waitForSearchResponse = page.waitForResponse(
-  const field = await page.locator(
+    `/api/v1/search/query?q=*&from=0&size=10&*`
-    '[data-testid="add-edge-modal"] [data-testid="field-input"]'
   );
-  await field.click();
+
-  await field.fill(pipelineName);
+  await page
+    .locator('[data-testid="add-edge-modal"] [data-testid="field-input"]')
+    .fill(pipelineName);
+
+  await waitForSearchResponse;
 
   await page.click(`[data-testid="pipeline-entry-${pipelineFqn}"]`);
 
@@ -306,10 +312,10 @@ export const addColumnLineage = async (
   await lineageRes;
 
   if (exitEditMode) {
-    page.click('[data-testid="edit-lineage"]');
+    await page.click('[data-testid="edit-lineage"]');
   }
 
-  expect(
+  await expect(
     page.locator(
       `[data-testid="column-edge-${btoa(fromColumnNode)}-${btoa(
         toColumnNode
@@ -341,7 +347,7 @@ export const removeColumnLineage = async (
 
   await page.click('[data-testid="edit-lineage"]');
 
-  expect(
+  await expect(
     page.locator(
       `[data-testid="column-edge-${btoa(fromColumnNode)}-${btoa(
         toColumnNode
@@ -381,3 +387,9 @@ export const addPipelineBetweenNodes = async (
     );
   }
 };
+
+export const visitLineageTab = async (page: Page) => {
+  const lineageRes = page.waitForResponse('/api/v1/lineage/getLineage?*');
+  await page.click('[data-testid="lineage"]');
+  await lineageRes;
+};

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EntityInfoDrawer.component.tsx

@@ -13,6 +13,7 @@
 
 import { CloseOutlined } from '@ant-design/icons';
 import { Col, Drawer, Row } from 'antd';
+import { cloneDeep } from 'lodash';
 import { EntityDetailUnion } from 'Models';
 import React, { useEffect, useMemo, useState } from 'react';
 import { EntityType } from '../../../enums/entity.enum';
@@ -25,6 +26,7 @@ import { SearchIndex } from '../../../generated/entity/data/searchIndex';
 import { StoredProcedure } from '../../../generated/entity/data/storedProcedure';
 import { Table } from '../../../generated/entity/data/table';
 import { Topic } from '../../../generated/entity/data/topic';
+import { TagLabel } from '../../../generated/type/tagLabel';
 import { SearchSourceAlias } from '../../../interface/search.interface';
 import entityUtilClassBase from '../../../utils/EntityUtilClassBase';
 import {
@@ -173,7 +175,14 @@ const EntityInfoDrawer = ({
   }, [entityDetail, tags, selectedNode]);
 
   useEffect(() => {
-    setEntityDetail(selectedNode);
+    const node = cloneDeep(selectedNode);
+    // Since selectedNode is a source object, modify the tags to contain tier information
+    node.tags = [
+      ...(node.tags ?? []),
+      ...(node.tier ? [node.tier as TagLabel] : []),
+    ];
+
+    setEntityDetail(node);
   }, [selectedNode]);
 
   return (

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx

@@ -15,8 +15,8 @@ import { Button, Input, Modal } from 'antd';
 import { AxiosError } from 'axios';
 import classNames from 'classnames';
 import { t } from 'i18next';
-import { isUndefined } from 'lodash';
+import { debounce, isUndefined } from 'lodash';
-import React, { useEffect, useMemo, useState } from 'react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import { Edge } from 'reactflow';
 import { PAGE_SIZE } from '../../../../constants/constants';
 import { ERROR_PLACEHOLDER_TYPE, SIZE } from '../../../../enums/common.enum';
@@ -108,9 +108,16 @@ const AddPipeLineModal = ({
     return;
   }, [selectedEdge, edgeSearchValue]);
 
+  const debounceOnSearch = useCallback(debounce(getSearchResults, 300), []);
+
+  const handleChange = (value: string): void => {
+    setEdgeSearchValue(value);
+    debounceOnSearch(value);
+  };
+
   useEffect(() => {
     getSearchResults(edgeSearchValue);
-  }, [edgeSearchValue]);
+  }, []);
 
   return (
     <Modal
@@ -145,7 +152,7 @@ const AddPipeLineModal = ({
         data-testid="field-input"
         placeholder={t('message.search-for-edge')}
         value={edgeSearchValue}
-        onChange={(e) => setEdgeSearchValue(e.target.value)}
+        onChange={(e) => handleChange(e.target.value)}
       />
 
       <div className="edge-option-container">

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx

@@ -22,7 +22,6 @@ import { ReactComponent as PipelineIcon } from '../../../assets/svg/pipeline-gre
 import { FOREIGN_OBJECT_SIZE } from '../../../constants/Lineage.constants';
 import { useLineageProvider } from '../../../context/LineageProvider/LineageProvider';
 import { LineageLayerView } from '../../../context/LineageProvider/LineageProvider.interface';
-import { EntityType } from '../../../enums/entity.enum';
 import { StatusType } from '../../../generated/entity/data/pipeline';
 import { useApplicationStore } from '../../../hooks/useApplicationStore';
 import { getColumnSourceTargetHandles } from '../../../utils/EntityLineageUtils';
@@ -78,8 +77,7 @@ export const CustomEdge = ({
   } = data;
   const offset = 4;
 
-  const { fromEntity, toEntity, pipeline, pipelineEntityType } =
+  const { pipeline, pipelineEntityType } = data?.edge ?? {};
-    data?.edge ?? {};
 
   const {
     tracedNodes,
@@ -152,18 +150,7 @@ export const CustomEdge = ({
     };
   }, [style, tracedNodes, edge, isColumnHighlighted, isColumnLineage]);
 
-  const isPipelineEdgeAllowed = (
+  const isColumnLineageAllowed = !isColumnLineage;
-    sourceType: EntityType,
-    targetType: EntityType
-  ) => {
-    return (
-      [EntityType.TABLE, EntityType.TOPIC].indexOf(sourceType) > -1 &&
-      [EntityType.TABLE, EntityType.TOPIC].indexOf(targetType) > -1
-    );
-  };
-
-  const isColumnLineageAllowed =
-    !isColumnLineage && isPipelineEdgeAllowed(fromEntity.type, toEntity.type);
 
   const hasLabel = useMemo(() => {
     if (isColumnLineage) {