feat(react): add topological sort feature, update graphql, add tests (#2600)

datahub-web-react/src/Mocks.tsx

@@ -529,7 +529,6 @@ export const dataFlow1 = {
             },
         ],
     },
-    dataJobs: [],
 } as DataFlow;
 
 export const dataJob1 = {
@@ -569,6 +568,7 @@ export const dataJob1 = {
         __typename: 'DataJobInputOutput',
         inputDatasets: [dataset3],
         outputDatasets: [dataset3],
+        inputDatajobs: [],
     },
     upstreamLineage: null,
     downstreamLineage: null,
@@ -586,6 +586,126 @@ export const dataJob1 = {
     },
 } as DataJob;
 
+export const dataJob2 = {
+    __typename: 'DataJob',
+    urn: 'urn:li:dataJob:2',
+    type: EntityType.DataJob,
+    dataFlow: dataFlow1,
+    jobId: 'jobId2',
+    ownership: {
+        __typename: 'Ownership',
+        owners: [
+            {
+                owner: {
+                    ...user1,
+                },
+                type: 'DATAOWNER',
+            },
+            {
+                owner: {
+                    ...user2,
+                },
+                type: 'DELEGATE',
+            },
+        ],
+        lastModified: {
+            time: 0,
+        },
+    },
+    info: {
+        __typename: 'DataJobInfo',
+        name: 'DataJobInfoName2',
+        description: 'DataJobInfo2 Description',
+        externalUrl: null,
+        customProperties: [],
+    },
+    inputOutput: {
+        __typename: 'DataJobInputOutput',
+        inputDatasets: [dataset3],
+        outputDatasets: [dataset3],
+        inputDatajobs: [],
+    },
+    upstreamLineage: null,
+    downstreamLineage: null,
+    globalTags: {
+        tags: [
+            {
+                tag: {
+                    type: EntityType.Tag,
+                    urn: 'urn:li:tag:abc-sample-tag',
+                    name: 'abc-sample-tag',
+                    description: 'sample tag',
+                },
+            },
+        ],
+    },
+} as DataJob;
+
+export const dataJob3 = {
+    __typename: 'DataJob',
+    urn: 'urn:li:dataJob:3',
+    type: EntityType.DataJob,
+    dataFlow: dataFlow1,
+    jobId: 'jobId3',
+    ownership: {
+        __typename: 'Ownership',
+        owners: [
+            {
+                owner: {
+                    ...user1,
+                },
+                type: 'DATAOWNER',
+            },
+            {
+                owner: {
+                    ...user2,
+                },
+                type: 'DELEGATE',
+            },
+        ],
+        lastModified: {
+            time: 0,
+        },
+    },
+    info: {
+        __typename: 'DataJobInfo',
+        name: 'DataJobInfoName3',
+        description: 'DataJobInfo3 Description',
+        externalUrl: null,
+        customProperties: [],
+    },
+    inputOutput: {
+        __typename: 'DataJobInputOutput',
+        inputDatasets: [dataset3],
+        outputDatasets: [dataset3],
+        inputDatajobs: [],
+    },
+    upstreamLineage: null,
+    downstreamLineage: null,
+    globalTags: {
+        tags: [
+            {
+                tag: {
+                    type: EntityType.Tag,
+                    urn: 'urn:li:tag:abc-sample-tag',
+                    name: 'abc-sample-tag',
+                    description: 'sample tag',
+                },
+            },
+        ],
+    },
+} as DataJob;
+
+dataJob1.upstreamLineage = {
+    entities: [
+        {
+            created: {
+                time: 0,
+            },
+            entity: dataJob3,
+        },
+    ],
+};
 /*
     Define mock data to be returned by Apollo MockProvider. 
 */
@@ -635,6 +755,21 @@ export const mocks = [
             },
         },
     },
+    {
+        request: {
+            query: GetUserDocument,
+            variables: {
+                urn: 'urn:li:corpuser:datahub',
+            },
+        },
+        result: {
+            data: {
+                corpUser: {
+                    ...user1,
+                },
+            },
+        },
+    },
     {
         request: {
             query: GetBrowsePathsDocument,
@@ -1298,6 +1433,43 @@ export const mocks = [
             data: {
                 dataFlow: {
                     ...dataFlow1,
+                    dataJobs: {
+                        entities: [
+                            {
+                                created: {
+                                    time: 0,
+                                },
+                                entity: dataJob1,
+                            },
+                            {
+                                created: {
+                                    time: 0,
+                                },
+                                entity: dataJob2,
+                            },
+                            {
+                                created: {
+                                    time: 0,
+                                },
+                                entity: dataJob3,
+                            },
+                        ],
+                    },
+                },
+            },
+        },
+    },
+    {
+        request: {
+            query: GetDataJobDocument,
+            variables: {
+                urn: 'urn:li:dataJob:1',
+            },
+        },
+        result: {
+            data: {
+                dataJob: {
+                    ...dataJob1,
                 },
             },
         },
@@ -1355,21 +1527,6 @@ export const mocks = [
             } as GetSearchResultsQuery,
         },
     },
-    {
-        request: {
-            query: GetDataJobDocument,
-            variables: {
-                urn: 'urn:li:dataJob:1',
-            },
-        },
-        result: {
-            data: {
-                dataJob: {
-                    ...dataJob1,
-                },
-            },
-        },
-    },
     {
         request: {
             query: GetTagDocument,

datahub-web-react/src/app/entity/dataFlow/profile/DataFlowDataJobs.tsx

@@ -27,6 +27,7 @@ export default function DataFlowDataJobs({ dataJobs }: Props) {
             renderItem={(item) => (
                 <DataJobItem>{entityRegistry.renderPreview(EntityType.DataJob, PreviewType.PREVIEW, item)}</DataJobItem>
             )}
+            data-testid="dataflow-jobs-list"
         />
     );
 }

datahub-web-react/src/app/entity/dataFlow/profile/DataFlowProfile.tsx

@@ -15,6 +15,7 @@ import { Properties as PropertiesView } from '../../shared/Properties';
 import { Ownership as OwnershipView } from '../../shared/Ownership';
 import { useEntityRegistry } from '../../../useEntityRegistry';
 import analytics, { EventType, EntityActionType } from '../../../analytics';
+import { topologicalSort } from '../../../../utils/sort/topologicalSort';
 
 export enum TabType {
     Tasks = 'Tasks',
@@ -80,7 +81,7 @@ export const DataFlowProfile = ({ urn }: { urn: string }): JSX.Element => {
             {
                 name: TabType.Tasks,
                 path: TabType.Tasks.toLowerCase(),
-                content: <DataFlowDataJobs dataJobs={dataJobs?.entities || []} />,
+                content: <DataFlowDataJobs dataJobs={topologicalSort(dataJobs?.entities || [])} />,
             },
         ].filter((tab) => ENABLED_TAB_TYPES.includes(tab.name));
     };

datahub-web-react/src/app/entity/dataFlow/profile/__tests__/DataFlowProfile.test.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { render, waitFor } from '@testing-library/react';
+import { render, waitFor, fireEvent } from '@testing-library/react';
 import { MockedProvider } from '@apollo/client/testing';
 
 import { DataFlowProfile } from '../DataFlowProfile';
@@ -10,7 +10,7 @@ describe('DataJobProfile', () => {
     it('renders', async () => {
         const { getByText, queryAllByText } = render(
             <MockedProvider mocks={mocks} addTypename={false}>
-                <TestPageContainer initialEntries={['/pipeline/urn:li:dataFlow:1']}>
+                <TestPageContainer initialEntries={['/pipelines/urn:li:dataFlow:1']}>
                     <DataFlowProfile urn="urn:li:dataFlow:1" />
                 </TestPageContainer>
             </MockedProvider>,
@@ -19,4 +19,34 @@ describe('DataJobProfile', () => {
 
         expect(getByText('DataFlowInfo1 Description')).toBeInTheDocument();
     });
+
+    it('topological sort', async () => {
+        const { getByTestId, getByText, queryAllByText, getAllByTestId } = render(
+            <MockedProvider
+                mocks={mocks}
+                addTypename={false}
+                defaultOptions={{
+                    watchQuery: { fetchPolicy: 'no-cache' },
+                    query: { fetchPolicy: 'no-cache' },
+                }}
+            >
+                <TestPageContainer initialEntries={['/pipelines/urn:li:dataFlow:1']}>
+                    <DataFlowProfile urn="urn:li:dataFlow:1" />
+                </TestPageContainer>
+            </MockedProvider>,
+        );
+
+        await waitFor(() => expect(queryAllByText('DataFlowInfoName').length).toBeGreaterThanOrEqual(1));
+        const rawButton = getByText('Tasks');
+        fireEvent.click(rawButton);
+        await waitFor(() => expect(getByTestId('dataflow-jobs-list')).toBeInTheDocument());
+        await waitFor(() => expect(queryAllByText('DataJobInfoName3').length).toBeGreaterThanOrEqual(1));
+        await new Promise((r) => setTimeout(r, 1000));
+        const jobsList = getAllByTestId('datajob-item-preview');
+
+        expect(jobsList.length).toBe(3);
+        expect(jobsList[0].innerHTML).toMatch(/DataJobInfoName3/);
+        expect(jobsList[1].innerHTML).toMatch(/DataJobInfoName/);
+        expect(jobsList[2].innerHTML).toMatch(/DataJobInfoName2/);
+    });
 });

datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx

@@ -36,6 +36,7 @@ export const Preview = ({
             owners={owners}
             tags={globalTags || undefined}
             snippet={snippet}
+            dataTestID="datajob-item-preview"
         />
     );
 };

datahub-web-react/src/app/entity/dataJob/profile/__tests__/DataJobProfile.test.tsx

@@ -10,7 +10,7 @@ describe('DataJobProfile', () => {
     it('renders', async () => {
         const { getByText, queryAllByText } = render(
             <MockedProvider mocks={mocks} addTypename={false}>
-                <TestPageContainer initialEntries={['/task/urn:li:dataJob:1']}>
+                <TestPageContainer initialEntries={['/tasks/urn:li:dataJob:1']}>
                     <DataJobProfile urn="urn:li:dataJob:1" />
                 </TestPageContainer>
             </MockedProvider>,

datahub-web-react/src/app/preview/DefaultPreviewCard.tsx

@@ -20,6 +20,7 @@ interface Props {
     owners?: Array<Owner> | null;
     snippet?: React.ReactNode;
     glossaryTerms?: GlossaryTerms;
+    dataTestID?: string;
 }
 
 const DescriptionParagraph = styled(Typography.Paragraph)`
@@ -58,11 +59,12 @@ export default function DefaultPreviewCard({
     owners,
     snippet,
     glossaryTerms,
+    dataTestID,
 }: Props) {
     const entityRegistry = useEntityRegistry();
 
     return (
-        <Row style={styles.row} justify="space-between">
+        <Row style={styles.row} justify="space-between" data-testid={dataTestID}>
             <Space direction="vertical" align="start" size={28} style={styles.leftColumn}>
                 <Link to={url}>
                     <Space direction="horizontal" size={20} align="center">

datahub-web-react/src/graphql/fragments.graphql

@@ -200,6 +200,22 @@ fragment nonRecursiveDataFlowFields on DataFlow {
     }
 }
 
+fragment nonRecursiveDataJobFields on DataJob {
+    urn
+    info {
+        name
+        description
+        externalUrl
+        customProperties {
+            key
+            value
+        }
+    }
+    globalTags {
+        ...globalTagsFields
+    }
+}
+
 fragment dataJobFields on DataJob {
     urn
     type
@@ -217,6 +233,9 @@ fragment dataJobFields on DataJob {
         outputDatasets {
             ...nonRecursiveDatasetFields
         }
+        inputDatajobs {
+            ...nonRecursiveDataJobFields
+        }
     }
     info {
         name

datahub-web-react/src/utils/sort/topologicalSort.ts

@@ -0,0 +1,42 @@
+import { EntityRelationship } from '../../types.generated';
+
+// Sort helper function
+function topologicalSortHelper(
+    node: EntityRelationship,
+    explored: Set<string>,
+    result: Array<EntityRelationship>,
+    urnsArray: Array<string>,
+) {
+    if (!node.entity?.urn) {
+        return;
+    }
+    explored.add(node.entity?.urn);
+
+    (node.entity.upstreamLineage?.entities || [])
+        .filter((entity) => entity?.entity?.urn && urnsArray.includes(entity?.entity?.urn))
+        .forEach((n) => {
+            if (n?.entity?.urn && !explored.has(n?.entity?.urn)) {
+                topologicalSortHelper(n, explored, result, urnsArray);
+            }
+        });
+    if (urnsArray.includes(node?.entity?.urn)) {
+        result.push(node);
+    }
+}
+
+// Topological Sort function with array of EntityRelationship
+export function topologicalSort(input: Array<EntityRelationship | null>) {
+    const explored = new Set<string>();
+    const result: Array<EntityRelationship> = [];
+    const nodes: Array<EntityRelationship> = [...input] as Array<EntityRelationship>;
+    const urnsArray: Array<string> = nodes
+        .filter((node) => !!node.entity?.urn)
+        .map((node) => node.entity?.urn) as Array<string>;
+    nodes.forEach((node) => {
+        if (node.entity?.urn && !explored.has(node.entity?.urn)) {
+            topologicalSortHelper(node, explored, result, urnsArray);
+        }
+    });
+
+    return result;
+}