From d23c58734b8c09d61f5f912e097cd9fa42dcc688 Mon Sep 17 00:00:00 2001 From: Brian Date: Wed, 26 May 2021 09:58:43 +0800 Subject: [PATCH] feat(react): add topological sort feature, update graphql, add tests (#2600) --- datahub-web-react/src/Mocks.tsx | 189 ++++++++++++++++-- .../dataFlow/profile/DataFlowDataJobs.tsx | 1 + .../dataFlow/profile/DataFlowProfile.tsx | 3 +- .../__tests__/DataFlowProfile.test.tsx | 34 +++- .../app/entity/dataJob/preview/Preview.tsx | 1 + .../profile/__tests__/DataJobProfile.test.tsx | 2 +- .../src/app/preview/DefaultPreviewCard.tsx | 4 +- .../src/graphql/fragments.graphql | 19 ++ .../src/utils/sort/topologicalSort.ts | 42 ++++ 9 files changed, 274 insertions(+), 21 deletions(-) create mode 100644 datahub-web-react/src/utils/sort/topologicalSort.ts diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index c51ab6a155..b9d6116d9f 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -529,7 +529,6 @@ export const dataFlow1 = { }, ], }, - dataJobs: [], } as DataFlow; export const dataJob1 = { @@ -569,6 +568,7 @@ export const dataJob1 = { __typename: 'DataJobInputOutput', inputDatasets: [dataset3], outputDatasets: [dataset3], + inputDatajobs: [], }, upstreamLineage: null, downstreamLineage: null, @@ -586,6 +586,126 @@ export const dataJob1 = { }, } as DataJob; +export const dataJob2 = { + __typename: 'DataJob', + urn: 'urn:li:dataJob:2', + type: EntityType.DataJob, + dataFlow: dataFlow1, + jobId: 'jobId2', + ownership: { + __typename: 'Ownership', + owners: [ + { + owner: { + ...user1, + }, + type: 'DATAOWNER', + }, + { + owner: { + ...user2, + }, + type: 'DELEGATE', + }, + ], + lastModified: { + time: 0, + }, + }, + info: { + __typename: 'DataJobInfo', + name: 'DataJobInfoName2', + description: 'DataJobInfo2 Description', + externalUrl: null, + customProperties: [], + }, + inputOutput: { + __typename: 'DataJobInputOutput', + inputDatasets: [dataset3], + outputDatasets: [dataset3], + inputDatajobs: [], + }, + upstreamLineage: null, + downstreamLineage: null, + globalTags: { + tags: [ + { + tag: { + type: EntityType.Tag, + urn: 'urn:li:tag:abc-sample-tag', + name: 'abc-sample-tag', + description: 'sample tag', + }, + }, + ], + }, +} as DataJob; + +export const dataJob3 = { + __typename: 'DataJob', + urn: 'urn:li:dataJob:3', + type: EntityType.DataJob, + dataFlow: dataFlow1, + jobId: 'jobId3', + ownership: { + __typename: 'Ownership', + owners: [ + { + owner: { + ...user1, + }, + type: 'DATAOWNER', + }, + { + owner: { + ...user2, + }, + type: 'DELEGATE', + }, + ], + lastModified: { + time: 0, + }, + }, + info: { + __typename: 'DataJobInfo', + name: 'DataJobInfoName3', + description: 'DataJobInfo3 Description', + externalUrl: null, + customProperties: [], + }, + inputOutput: { + __typename: 'DataJobInputOutput', + inputDatasets: [dataset3], + outputDatasets: [dataset3], + inputDatajobs: [], + }, + upstreamLineage: null, + downstreamLineage: null, + globalTags: { + tags: [ + { + tag: { + type: EntityType.Tag, + urn: 'urn:li:tag:abc-sample-tag', + name: 'abc-sample-tag', + description: 'sample tag', + }, + }, + ], + }, +} as DataJob; + +dataJob1.upstreamLineage = { + entities: [ + { + created: { + time: 0, + }, + entity: dataJob3, + }, + ], +}; /* Define mock data to be returned by Apollo MockProvider. */ @@ -635,6 +755,21 @@ export const mocks = [ }, }, }, + { + request: { + query: GetUserDocument, + variables: { + urn: 'urn:li:corpuser:datahub', + }, + }, + result: { + data: { + corpUser: { + ...user1, + }, + }, + }, + }, { request: { query: GetBrowsePathsDocument, @@ -1298,6 +1433,43 @@ export const mocks = [ data: { dataFlow: { ...dataFlow1, + dataJobs: { + entities: [ + { + created: { + time: 0, + }, + entity: dataJob1, + }, + { + created: { + time: 0, + }, + entity: dataJob2, + }, + { + created: { + time: 0, + }, + entity: dataJob3, + }, + ], + }, + }, + }, + }, + }, + { + request: { + query: GetDataJobDocument, + variables: { + urn: 'urn:li:dataJob:1', + }, + }, + result: { + data: { + dataJob: { + ...dataJob1, }, }, }, @@ -1355,21 +1527,6 @@ export const mocks = [ } as GetSearchResultsQuery, }, }, - { - request: { - query: GetDataJobDocument, - variables: { - urn: 'urn:li:dataJob:1', - }, - }, - result: { - data: { - dataJob: { - ...dataJob1, - }, - }, - }, - }, { request: { query: GetTagDocument, diff --git a/datahub-web-react/src/app/entity/dataFlow/profile/DataFlowDataJobs.tsx b/datahub-web-react/src/app/entity/dataFlow/profile/DataFlowDataJobs.tsx index bc5bfa7760..4fe92ac9f0 100644 --- a/datahub-web-react/src/app/entity/dataFlow/profile/DataFlowDataJobs.tsx +++ b/datahub-web-react/src/app/entity/dataFlow/profile/DataFlowDataJobs.tsx @@ -27,6 +27,7 @@ export default function DataFlowDataJobs({ dataJobs }: Props) { renderItem={(item) => ( {entityRegistry.renderPreview(EntityType.DataJob, PreviewType.PREVIEW, item)} )} + data-testid="dataflow-jobs-list" /> ); } diff --git a/datahub-web-react/src/app/entity/dataFlow/profile/DataFlowProfile.tsx b/datahub-web-react/src/app/entity/dataFlow/profile/DataFlowProfile.tsx index 53a4c82491..a5e6f109ca 100644 --- a/datahub-web-react/src/app/entity/dataFlow/profile/DataFlowProfile.tsx +++ b/datahub-web-react/src/app/entity/dataFlow/profile/DataFlowProfile.tsx @@ -15,6 +15,7 @@ import { Properties as PropertiesView } from '../../shared/Properties'; import { Ownership as OwnershipView } from '../../shared/Ownership'; import { useEntityRegistry } from '../../../useEntityRegistry'; import analytics, { EventType, EntityActionType } from '../../../analytics'; +import { topologicalSort } from '../../../../utils/sort/topologicalSort'; export enum TabType { Tasks = 'Tasks', @@ -80,7 +81,7 @@ export const DataFlowProfile = ({ urn }: { urn: string }): JSX.Element => { { name: TabType.Tasks, path: TabType.Tasks.toLowerCase(), - content: , + content: , }, ].filter((tab) => ENABLED_TAB_TYPES.includes(tab.name)); }; diff --git a/datahub-web-react/src/app/entity/dataFlow/profile/__tests__/DataFlowProfile.test.tsx b/datahub-web-react/src/app/entity/dataFlow/profile/__tests__/DataFlowProfile.test.tsx index 740a85fc0e..d97023a798 100644 --- a/datahub-web-react/src/app/entity/dataFlow/profile/__tests__/DataFlowProfile.test.tsx +++ b/datahub-web-react/src/app/entity/dataFlow/profile/__tests__/DataFlowProfile.test.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { render, waitFor } from '@testing-library/react'; +import { render, waitFor, fireEvent } from '@testing-library/react'; import { MockedProvider } from '@apollo/client/testing'; import { DataFlowProfile } from '../DataFlowProfile'; @@ -10,7 +10,7 @@ describe('DataJobProfile', () => { it('renders', async () => { const { getByText, queryAllByText } = render( - + , @@ -19,4 +19,34 @@ describe('DataJobProfile', () => { expect(getByText('DataFlowInfo1 Description')).toBeInTheDocument(); }); + + it('topological sort', async () => { + const { getByTestId, getByText, queryAllByText, getAllByTestId } = render( + + + + + , + ); + + await waitFor(() => expect(queryAllByText('DataFlowInfoName').length).toBeGreaterThanOrEqual(1)); + const rawButton = getByText('Tasks'); + fireEvent.click(rawButton); + await waitFor(() => expect(getByTestId('dataflow-jobs-list')).toBeInTheDocument()); + await waitFor(() => expect(queryAllByText('DataJobInfoName3').length).toBeGreaterThanOrEqual(1)); + await new Promise((r) => setTimeout(r, 1000)); + const jobsList = getAllByTestId('datajob-item-preview'); + + expect(jobsList.length).toBe(3); + expect(jobsList[0].innerHTML).toMatch(/DataJobInfoName3/); + expect(jobsList[1].innerHTML).toMatch(/DataJobInfoName/); + expect(jobsList[2].innerHTML).toMatch(/DataJobInfoName2/); + }); }); diff --git a/datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx b/datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx index 4195451ee4..eecd5b8253 100644 --- a/datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx @@ -36,6 +36,7 @@ export const Preview = ({ owners={owners} tags={globalTags || undefined} snippet={snippet} + dataTestID="datajob-item-preview" /> ); }; diff --git a/datahub-web-react/src/app/entity/dataJob/profile/__tests__/DataJobProfile.test.tsx b/datahub-web-react/src/app/entity/dataJob/profile/__tests__/DataJobProfile.test.tsx index 539341f67d..781dae46db 100644 --- a/datahub-web-react/src/app/entity/dataJob/profile/__tests__/DataJobProfile.test.tsx +++ b/datahub-web-react/src/app/entity/dataJob/profile/__tests__/DataJobProfile.test.tsx @@ -10,7 +10,7 @@ describe('DataJobProfile', () => { it('renders', async () => { const { getByText, queryAllByText } = render( - + , diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index 027ec26928..81c36d2a2d 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -20,6 +20,7 @@ interface Props { owners?: Array | null; snippet?: React.ReactNode; glossaryTerms?: GlossaryTerms; + dataTestID?: string; } const DescriptionParagraph = styled(Typography.Paragraph)` @@ -58,11 +59,12 @@ export default function DefaultPreviewCard({ owners, snippet, glossaryTerms, + dataTestID, }: Props) { const entityRegistry = useEntityRegistry(); return ( - + diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index e0092df69e..48c2128595 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -200,6 +200,22 @@ fragment nonRecursiveDataFlowFields on DataFlow { } } +fragment nonRecursiveDataJobFields on DataJob { + urn + info { + name + description + externalUrl + customProperties { + key + value + } + } + globalTags { + ...globalTagsFields + } +} + fragment dataJobFields on DataJob { urn type @@ -217,6 +233,9 @@ fragment dataJobFields on DataJob { outputDatasets { ...nonRecursiveDatasetFields } + inputDatajobs { + ...nonRecursiveDataJobFields + } } info { name diff --git a/datahub-web-react/src/utils/sort/topologicalSort.ts b/datahub-web-react/src/utils/sort/topologicalSort.ts new file mode 100644 index 0000000000..792c7207d1 --- /dev/null +++ b/datahub-web-react/src/utils/sort/topologicalSort.ts @@ -0,0 +1,42 @@ +import { EntityRelationship } from '../../types.generated'; + +// Sort helper function +function topologicalSortHelper( + node: EntityRelationship, + explored: Set, + result: Array, + urnsArray: Array, +) { + if (!node.entity?.urn) { + return; + } + explored.add(node.entity?.urn); + + (node.entity.upstreamLineage?.entities || []) + .filter((entity) => entity?.entity?.urn && urnsArray.includes(entity?.entity?.urn)) + .forEach((n) => { + if (n?.entity?.urn && !explored.has(n?.entity?.urn)) { + topologicalSortHelper(n, explored, result, urnsArray); + } + }); + if (urnsArray.includes(node?.entity?.urn)) { + result.push(node); + } +} + +// Topological Sort function with array of EntityRelationship +export function topologicalSort(input: Array) { + const explored = new Set(); + const result: Array = []; + const nodes: Array = [...input] as Array; + const urnsArray: Array = nodes + .filter((node) => !!node.entity?.urn) + .map((node) => node.entity?.urn) as Array; + nodes.forEach((node) => { + if (node.entity?.urn && !explored.has(node.entity?.urn)) { + topologicalSortHelper(node, explored, result, urnsArray); + } + }); + + return result; +}