update lineage export data to show common fields (#16207)

* fix data for lineage export

* add level in lineage export

* set default open entity popover card for pipeline lineage

openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.ts

@@ -64,7 +64,15 @@ const connectEdgeBetweenNodes = (fromNode, toNode) => {
     .should('contain', 'false');
 
   cy.get('[data-testid="suggestion-node"]').click();
+
+  interceptURL(
+    'GET',
+    `/api/v1/search/query?q=*${toNode.term}*&**`,
+    'nodeQuery'
+  );
   cy.get('[data-testid="suggestion-node"] input').click().type(toNode.term);
+  verifyResponseStatusCode('@nodeQuery', 200);
+
   cy.get(`[data-testid="node-suggestion-${toNode.fqn}"]`)
     .scrollIntoView()
     .click();

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx

@@ -243,6 +243,7 @@ export const CustomEdge = ({
             />
           ) : (
             <EntityPopOverCard
+              defaultOpen={isPipelineRootNode}
               entityFQN={pipeline?.fullyQualifiedName}
               entityType={pipelineEntityType}
               extraInfo={
@@ -267,7 +268,15 @@ export const CustomEdge = ({
         </LineageEdgeIcon>
       );
     },
-    [edgeCenterX, edgeCenterY, rest, pipeline, blinkingClass, isEditMode]
+    [
+      edgeCenterX,
+      edgeCenterY,
+      rest,
+      pipeline,
+      blinkingClass,
+      isEditMode,
+      isPipelineRootNode,
+    ]
   );
 
   const getEditLineageIcon = useCallback(

openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx

@@ -88,18 +88,11 @@ const NodeSuggestions: FC<EntitySuggestionProps> = ({
     }
   };
 
-  const debouncedOnSearch = useCallback((searchText: string): void => {
-    getSearchResults(searchText);
-  }, []);
-
-  const debounceOnSearch = useCallback(debounce(debouncedOnSearch, 300), [
-    debouncedOnSearch,
-  ]);
+  const debounceOnSearch = useCallback(debounce(getSearchResults, 300), []);
 
   const handleChange = (value: string): void => {
-    const searchText = value;
-    setSearchValue(searchText);
-    debounceOnSearch(searchText);
+    setSearchValue(value);
+    debounceOnSearch(value);
   };
 
   useEffect(() => {
@@ -164,7 +157,7 @@ const NodeSuggestions: FC<EntitySuggestionProps> = ({
         popupClassName="lineage-suggestion-select-menu"
         onChange={handleChange}
         onClick={(e) => e.stopPropagation()}
-        onSearch={debouncedOnSearch}
+        onSearch={handleChange}
       />
     </div>
   );

openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts

@@ -53,3 +53,8 @@ export interface EdgeDetails {
   description?: string;
   pipelineEntityType?: EntityType.PIPELINE | EntityType.STORED_PROCEDURE;
 }
+
+export type LineageSourceType = Omit<SourceType, 'service'> & {
+  direction: string;
+  depth: number;
+};

openmetadata-ui/src/main/resources/ui/src/components/common/PopOverCard/EntityPopOverCard.tsx

@@ -57,6 +57,7 @@ interface Props extends HTMLAttributes<HTMLDivElement> {
   entityType: string;
   entityFQN: string;
   extraInfo?: React.ReactNode;
+  defaultOpen?: boolean;
 }
 
 export const PopoverContent: React.FC<{
@@ -227,6 +228,7 @@ const EntityPopOverCard: FC<Props> = ({
   entityType,
   entityFQN,
   extraInfo,
+  defaultOpen = false,
 }) => {
   return (
     <Popover
@@ -238,6 +240,7 @@ const EntityPopOverCard: FC<Props> = ({
           extraInfo={extraInfo}
         />
       }
+      defaultOpen={defaultOpen}
       overlayClassName="entity-popover-card"
       trigger="hover"
       zIndex={9999}>

openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts

@@ -49,10 +49,6 @@ export const entityData = [
     type: SearchIndex.CONTAINER,
     label: t('label.container-plural'),
   },
-  {
-    type: SearchIndex.PIPELINE,
-    label: t('label.pipeline-plural'),
-  },
   {
     type: SearchIndex.SEARCH_INDEX,
     label: t('label.search-index-plural'),
@@ -102,3 +98,17 @@ export const LINEAGE_COLUMN_NODE_SUPPORTED = [
   EntityType.TOPIC,
   EntityType.SEARCH_INDEX,
 ];
+
+export const LINEAGE_EXPORT_HEADERS = [
+  { field: 'name', title: 'Name' },
+  { field: 'displayName', title: 'Display Name' },
+  { field: 'fullyQualifiedName', title: 'Fully Qualified Name' },
+  { field: 'entityType', title: 'Entity Type' },
+  { field: 'direction', title: 'Direction' },
+  { field: 'owner', title: 'Owner' },
+  { field: 'domain', title: 'Domain' },
+  { field: 'tags', title: 'Tags' },
+  { field: 'tier', title: 'Tier' },
+  { field: 'glossaryTerms', title: 'Glossary Terms' },
+  { field: 'depth', title: 'Level' },
+];

openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx

@@ -75,11 +75,7 @@ import {
   LineageDetails,
 } from '../../generated/type/entityLineage';
 import { useFqn } from '../../hooks/useFqn';
-import {
-  exportLineage,
-  getLineageDataByFQN,
-  updateLineageEdge,
-} from '../../rest/lineageAPI';
+import { getLineageDataByFQN, updateLineageEdge } from '../../rest/lineageAPI';
 import {
   addLineageHandler,
   createEdges,
@@ -91,6 +87,7 @@ import {
   getChildMap,
   getClassifiedEdge,
   getConnectedNodesEdges,
+  getExportData,
   getLayoutedElements,
   getLineageEdge,
   getLineageEdgeForAPI,
@@ -178,6 +175,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
   const [childMap, setChildMap] = useState<EntityReferenceChild>();
   const [paginationData, setPaginationData] = useState({});
   const { showModal } = useEntityExportModalProvider();
+  const [exportResult, setExportResult] = useState<string>('');
 
   const initLineageChildMaps = useCallback(
     (
@@ -230,13 +228,12 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
             entityType !== EntityType.PIPELINE &&
             entityType !== EntityType.STORED_PROCEDURE
           ) {
-            const childMapObj = getChildMap(
+            const { map: childMapObj, exportResult } = getChildMap(
               { ...res, nodes: allNodes },
               decodedFqn
             );
-
+            setExportResult(exportResult);
             setChildMap(childMapObj);
-
             const { nodes: newNodes, edges: newEdges } = getPaginatedChildMap(
               {
                 ...res,
@@ -253,6 +250,8 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
               edges: [...(res.edges ?? []), ...newEdges],
             });
           } else {
+            const csv = getExportData(allNodes);
+            setExportResult(csv);
             setEntityLineage({
               ...res,
               nodes: allNodes,
@@ -281,26 +280,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
   );
 
   const exportLineageData = useCallback(
-    async (name: string) => {
-      try {
-        return await exportLineage(
-          name,
-          entityType,
-          lineageConfig,
-          queryFilter
-        );
-      } catch (err) {
-        showErrorToast(
-          err as AxiosError,
-          t('server.entity-fetch-error', {
-            entity: t('label.lineage-data-lowercase'),
-          })
-        );
-
-        return '';
-      }
+    async (_: string) => {
+      return exportResult;
     },
-    [entityType, lineageConfig, queryFilter]
+    [exportResult]
   );
 
   const onExportClick = useCallback(() => {
@@ -310,7 +293,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
         onExport: exportLineageData,
       });
     }
-  }, [decodedFqn]);
+  }, [decodedFqn, exportResult]);
 
   const loadChildNodesHandler = useCallback(
     async (node: SourceType, direction: EdgeTypeEnum) => {

openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx

@@ -355,12 +355,16 @@ describe('Test EntityLineageUtils utility', () => {
   });
 
   it('getChildMap should return valid map object', () => {
-    expect(
-      getChildMap(
-        MOCK_LINEAGE_DATA_NEW,
-        's3_storage_sample.departments.media.movies'
-      )
-    ).toEqual(MOCK_CHILD_MAP);
+    const { map, exportResult } = getChildMap(
+      MOCK_LINEAGE_DATA_NEW,
+      's3_storage_sample.departments.media.movies'
+    );
+
+    expect(map).toEqual(MOCK_CHILD_MAP);
+    expect(exportResult).toEqual(
+      `Name,Display Name,Fully Qualified Name,Entity Type,Direction,Owner,Domain,Tags,Tier,Glossary Terms,Level
+"engineering","Engineering department","s3_storage_sample.departments.engineering","container","downstream","","","","","","1"`
+    );
   });
 
   it('getPaginatedChildMap should return valid map object', () => {

openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx

@@ -25,7 +25,7 @@ import {
   uniqWith,
   upperCase,
 } from 'lodash';
-import { LoadingState } from 'Models';
+import { EntityTags, LoadingState } from 'Models';
 import React, { MouseEvent as ReactMouseEvent } from 'react';
 import {
   Connection,
@@ -61,9 +61,11 @@ import { ExploreSearchIndex } from '../components/Explore/ExplorePage.interface'
 import {
   EdgeDetails,
   EntityLineageResponse,
+  LineageSourceType,
 } from '../components/Lineage/Lineage.interface';
 import { SourceType } from '../components/SearchedData/SearchedData.interface';
 import {
+  LINEAGE_EXPORT_HEADERS,
   NODE_HEIGHT,
   NODE_WIDTH,
   ZOOM_VALUE,
@@ -84,10 +86,12 @@ import { Column, Table } from '../generated/entity/data/table';
 import { Topic } from '../generated/entity/data/topic';
 import { ColumnLineage, LineageDetails } from '../generated/type/entityLineage';
 import { EntityReference } from '../generated/type/entityReference';
+import { TagSource } from '../generated/type/tagLabel';
 import { addLineage, deleteLineageEdge } from '../rest/miscAPI';
 import { getPartialNameFromTableFQN } from './CommonUtils';
 import { getEntityName } from './EntityUtils';
 import Fqn from './Fqn';
+import { jsonToCSV } from './StringsUtils';
 import { showErrorToast } from './ToastUtils';
 
 export const MAX_LINEAGE_LENGTH = 20;
@@ -1073,9 +1077,11 @@ export const getUpstreamDownstreamNodesEdges = (
 export const getLineageChildParents = (
   obj: EntityLineageResponse,
   nodeSet: Set<string>,
+  parsedNodes: LineageSourceType[],
   id: string,
   isParent = false,
-  index = 0
+  index = 0, // page index
+  depth = 1 // depth of lineage
 ) => {
   const edges = isParent ? obj.upstreamEdges || [] : obj.downstreamEdges || [];
   const filtered = edges.filter((edge) => {
@@ -1091,12 +1097,19 @@ export const getLineageChildParents = (
 
     if (node && !nodeSet.has(node.id)) {
       nodeSet.add(node.id);
+      parsedNodes.push({
+        ...(node as SourceType),
+        direction: isParent ? 'upstream' : 'downstream',
+        depth: depth,
+      });
       const childNodes = getLineageChildParents(
         obj,
         nodeSet,
+        parsedNodes,
         node.id,
         isParent,
-        i
+        i,
+        depth + 1
       );
       const lineage: EntityReferenceChild = { ...node, pageIndex: index + i };
 
@@ -1117,9 +1130,60 @@ export const removeDuplicates = (arr: EdgeDetails[] = []) => {
   return uniqWith(arr, isEqual);
 };
 
+export const getExportEntity = (entity: LineageSourceType) => {
+  const {
+    name,
+    displayName = '',
+    fullyQualifiedName = '',
+    entityType = '',
+    direction = '',
+    owner,
+    domain,
+    tier,
+    tags = [],
+    depth = '',
+  } = entity;
+
+  const classificationTags = [];
+  const glossaryTerms = [];
+
+  for (const tag of tags) {
+    if (tag.source === TagSource.Classification) {
+      classificationTags.push(tag.tagFQN);
+    } else if (tag.source === TagSource.Glossary) {
+      glossaryTerms.push(tag.tagFQN);
+    }
+  }
+
+  return {
+    name,
+    displayName,
+    fullyQualifiedName,
+    entityType,
+    direction,
+    owner: getEntityName(owner),
+    domain: domain?.fullyQualifiedName ?? '',
+    tags: classificationTags.join(', '),
+    tier: (tier as EntityTags)?.tagFQN ?? '',
+    glossaryTerms: glossaryTerms.join(', '),
+    depth,
+  };
+};
+
+export const getExportData = (
+  allNodes: LineageSourceType[] | EntityReference[]
+) => {
+  const exportResultData = allNodes.map((child) =>
+    getExportEntity(child as LineageSourceType)
+  );
+
+  return jsonToCSV(exportResultData, LINEAGE_EXPORT_HEADERS);
+};
+
 export const getChildMap = (obj: EntityLineageResponse, decodedFqn: string) => {
   const nodeSet = new Set<string>();
   nodeSet.add(obj.entity.id);
+  const parsedNodes: LineageSourceType[] = [];
 
   const data = getUpstreamDownstreamNodesEdges(
     obj.edges ?? [],
@@ -1134,6 +1198,7 @@ export const getChildMap = (obj: EntityLineageResponse, decodedFqn: string) => {
   const childMap: EntityReferenceChild[] = getLineageChildParents(
     newData,
     nodeSet,
+    parsedNodes,
     obj.entity.id,
     false
   );
@@ -1141,6 +1206,7 @@ export const getChildMap = (obj: EntityLineageResponse, decodedFqn: string) => {
   const parentsMap: EntityReferenceChild[] = getLineageChildParents(
     newData,
     nodeSet,
+    parsedNodes,
     obj.entity.id,
     true
   );
@@ -1151,7 +1217,10 @@ export const getChildMap = (obj: EntityLineageResponse, decodedFqn: string) => {
     parents: parentsMap,
   };
 
-  return map;
+  return {
+    map,
+    exportResult: getExportData(parsedNodes) ?? '',
+  };
 };
 
 export const flattenObj = (

openmetadata-ui/src/main/resources/ui/src/utils/StringsUtils.test.ts

@@ -14,6 +14,7 @@ import {
   formatJsonString,
   getDecodedFqn,
   getEncodedFqn,
+  jsonToCSV,
   replaceCallback,
 } from './StringsUtils';
 
@@ -100,4 +101,24 @@ describe('StringsUtils', () => {
       expect(formatJsonString(jsonString)).toStrictEqual(jsonString);
     });
   });
+
+  it('jsonToCSV should return expected csv', () => {
+    const jsonData = [
+      { name: 'John', age: 30, city: 'New York' },
+      { name: 'Jane', age: 25, city: 'San Francisco' },
+      { name: 'Bob', age: 35, city: 'Chicago' },
+    ];
+
+    const headers = [
+      { field: 'name', title: 'Name' },
+      { field: 'age', title: 'Age' },
+      { field: 'city', title: 'City' },
+    ];
+
+    const expectedCSV = `Name,Age,City\n"John","30","New York"\n"Jane","25","San Francisco"\n"Bob","35","Chicago"`;
+
+    expect(jsonToCSV(jsonData, headers)).toEqual(expectedCSV);
+    expect(jsonToCSV(jsonData, [])).toEqual('');
+    expect(jsonToCSV([], headers)).toEqual('');
+  });
 });

openmetadata-ui/src/main/resources/ui/src/utils/StringsUtils.ts

@@ -279,3 +279,46 @@ export const generateUUID = () => {
     replaceCallback
   );
 };
+
+type JSONRecord = Record<string, string | number | boolean>;
+type HeaderMap = {
+  field: string;
+  title: string;
+};
+
+export const jsonToCSV = <T extends JSONRecord>(
+  jsonArray: T[],
+  headers: HeaderMap[]
+): string => {
+  if (!Array.isArray(jsonArray) || jsonArray.length === 0) {
+    return '';
+  }
+
+  // Check if headers array is empty
+  if (headers.length === 0) {
+    return '';
+  }
+
+  // Create the header row from headers mapping
+  const headerRow = headers.map((h) => h.title);
+  const csvRows: string[] = [headerRow.join(',')];
+
+  // Convert each JSON object to a CSV row
+  jsonArray.forEach((obj) => {
+    const row = headers
+      .map((header) => {
+        const value = obj[header.field];
+        const escaped =
+          typeof value === 'string'
+            ? value.replace(/"/g, '\\"')
+            : value.toString(); // handle quotes in content
+
+        return `"${escaped}"`; // wrap each field in quotes
+      })
+      .join(',');
+    csvRows.push(row);
+  });
+
+  // Combine all CSV rows and add newline character to form final CSV string
+  return csvRows.join('\n');
+};