yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-31 12:52:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(react): add topological sort feature, update graphql, add tests (#2600)

Browse Source
This commit is contained in:
Brian 2021-05-26 09:58:43 +08:00 committed by GitHub
parent dbe42e07f6
commit d23c58734b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 274 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
datahub-web-react/src/Mocks.tsx
View File

@ -529,7 +529,6 @@ export const dataFlow1 = {
},
],
},
dataJobs: [],
} as DataFlow;
export const dataJob1 = {
@ -569,6 +568,7 @@ export const dataJob1 = {
__typename: 'DataJobInputOutput',
inputDatasets: [dataset3],
outputDatasets: [dataset3],
inputDatajobs: [],
},
upstreamLineage: null,
downstreamLineage: null,
@ -586,6 +586,126 @@ export const dataJob1 = {
},
} as DataJob;
export const dataJob2 = {
__typename: 'DataJob',
urn: 'urn:li:dataJob:2',
type: EntityType.DataJob,
dataFlow: dataFlow1,
jobId: 'jobId2',
ownership: {
__typename: 'Ownership',
owners: [
{
owner: {
...user1,
},
type: 'DATAOWNER',
},
{
owner: {
...user2,
},
type: 'DELEGATE',
},
],
lastModified: {
time: 0,
},
},
info: {
__typename: 'DataJobInfo',
name: 'DataJobInfoName2',
description: 'DataJobInfo2 Description',
externalUrl: null,
customProperties: [],
},
inputOutput: {
__typename: 'DataJobInputOutput',
inputDatasets: [dataset3],
outputDatasets: [dataset3],
inputDatajobs: [],
},
upstreamLineage: null,
downstreamLineage: null,
globalTags: {
tags: [
{
tag: {
type: EntityType.Tag,
urn: 'urn:li:tag:abc-sample-tag',
name: 'abc-sample-tag',
description: 'sample tag',
},
},
],
},
} as DataJob;
export const dataJob3 = {
__typename: 'DataJob',
urn: 'urn:li:dataJob:3',
type: EntityType.DataJob,
dataFlow: dataFlow1,
jobId: 'jobId3',
ownership: {
__typename: 'Ownership',
owners: [
{
owner: {
...user1,
},
type: 'DATAOWNER',
},
{
owner: {
...user2,
},
type: 'DELEGATE',
},
],
lastModified: {
time: 0,
},
},
info: {
__typename: 'DataJobInfo',
name: 'DataJobInfoName3',
description: 'DataJobInfo3 Description',
externalUrl: null,
customProperties: [],
},
inputOutput: {
__typename: 'DataJobInputOutput',
inputDatasets: [dataset3],
outputDatasets: [dataset3],
inputDatajobs: [],
},
upstreamLineage: null,
downstreamLineage: null,
globalTags: {
tags: [
{
tag: {
type: EntityType.Tag,
urn: 'urn:li:tag:abc-sample-tag',
name: 'abc-sample-tag',
description: 'sample tag',
},
},
],
},
} as DataJob;
dataJob1.upstreamLineage = {
entities: [
{
created: {
time: 0,
},
entity: dataJob3,
},
],
};
/*
Define mock data to be returned by Apollo MockProvider.
*/
@ -635,6 +755,21 @@ export const mocks = [
},
},
},
{
request: {
query: GetUserDocument,
variables: {
urn: 'urn:li:corpuser:datahub',
},
},
result: {
data: {
corpUser: {
...user1,
},
},
},
},
{
request: {
query: GetBrowsePathsDocument,
@ -1298,6 +1433,43 @@ export const mocks = [
data: {
dataFlow: {
...dataFlow1,
dataJobs: {
entities: [
{
created: {
time: 0,
},
entity: dataJob1,
},
{
created: {
time: 0,
},
entity: dataJob2,
},
{
created: {
time: 0,
},
entity: dataJob3,
},
],
},
},
},
},
},
{
request: {
query: GetDataJobDocument,
variables: {
urn: 'urn:li:dataJob:1',
},
},
result: {
data: {
dataJob: {
...dataJob1,
},
},
},
@ -1355,21 +1527,6 @@ export const mocks = [
} as GetSearchResultsQuery,
},
},
{
request: {
query: GetDataJobDocument,
variables: {
urn: 'urn:li:dataJob:1',
},
},
result: {
data: {
dataJob: {
...dataJob1,
},
},
},
},
{
request: {
query: GetTagDocument,

1
datahub-web-react/src/app/entity/dataFlow/profile/DataFlowDataJobs.tsx
View File

@ -27,6 +27,7 @@ export default function DataFlowDataJobs({ dataJobs }: Props) {
renderItem={(item) => (
<DataJobItem>{entityRegistry.renderPreview(EntityType.DataJob, PreviewType.PREVIEW, item)}</DataJobItem>
)}
data-testid="dataflow-jobs-list"
/>
);
}

3
datahub-web-react/src/app/entity/dataFlow/profile/DataFlowProfile.tsx
View File

@ -15,6 +15,7 @@ import { Properties as PropertiesView } from '../../shared/Properties';
import { Ownership as OwnershipView } from '../../shared/Ownership';
import { useEntityRegistry } from '../../../useEntityRegistry';
import analytics, { EventType, EntityActionType } from '../../../analytics';
import { topologicalSort } from '../../../../utils/sort/topologicalSort';
export enum TabType {
Tasks = 'Tasks',
@ -80,7 +81,7 @@ export const DataFlowProfile = ({ urn }: { urn: string }): JSX.Element => {
{
name: TabType.Tasks,
path: TabType.Tasks.toLowerCase(),
content: <DataFlowDataJobs dataJobs={dataJobs?.entities || []} />,
content: <DataFlowDataJobs dataJobs={topologicalSort(dataJobs?.entities || [])} />,
},
].filter((tab) => ENABLED_TAB_TYPES.includes(tab.name));
};

34
datahub-web-react/src/app/entity/dataFlow/profile/__tests__/DataFlowProfile.test.tsx
View File

@ -1,5 +1,5 @@
import React from 'react';
import { render, waitFor } from '@testing-library/react';
import { render, waitFor, fireEvent } from '@testing-library/react';
import { MockedProvider } from '@apollo/client/testing';
import { DataFlowProfile } from '../DataFlowProfile';
@ -10,7 +10,7 @@ describe('DataJobProfile', () => {
it('renders', async () => {
const { getByText, queryAllByText } = render(
<MockedProvider mocks={mocks} addTypename={false}>
<TestPageContainer initialEntries={['/pipeline/urn:li:dataFlow:1']}>
<TestPageContainer initialEntries={['/pipelines/urn:li:dataFlow:1']}>
<DataFlowProfile urn="urn:li:dataFlow:1" />
</TestPageContainer>
</MockedProvider>,
@ -19,4 +19,34 @@ describe('DataJobProfile', () => {
expect(getByText('DataFlowInfo1 Description')).toBeInTheDocument();
});
it('topological sort', async () => {
const { getByTestId, getByText, queryAllByText, getAllByTestId } = render(
<MockedProvider
mocks={mocks}
addTypename={false}
defaultOptions={{
watchQuery: { fetchPolicy: 'no-cache' },
query: { fetchPolicy: 'no-cache' },
}}
>
<TestPageContainer initialEntries={['/pipelines/urn:li:dataFlow:1']}>
<DataFlowProfile urn="urn:li:dataFlow:1" />
</TestPageContainer>
</MockedProvider>,
);
await waitFor(() => expect(queryAllByText('DataFlowInfoName').length).toBeGreaterThanOrEqual(1));
const rawButton = getByText('Tasks');
fireEvent.click(rawButton);
await waitFor(() => expect(getByTestId('dataflow-jobs-list')).toBeInTheDocument());
await waitFor(() => expect(queryAllByText('DataJobInfoName3').length).toBeGreaterThanOrEqual(1));
await new Promise((r) => setTimeout(r, 1000));
const jobsList = getAllByTestId('datajob-item-preview');
expect(jobsList.length).toBe(3);
expect(jobsList[0].innerHTML).toMatch(/DataJobInfoName3/);
expect(jobsList[1].innerHTML).toMatch(/DataJobInfoName/);
expect(jobsList[2].innerHTML).toMatch(/DataJobInfoName2/);
});
});

1
datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx
View File

@ -36,6 +36,7 @@ export const Preview = ({
owners={owners}
tags={globalTags || undefined}
snippet={snippet}
dataTestID="datajob-item-preview"
/>
);
};

2
datahub-web-react/src/app/entity/dataJob/profile/__tests__/DataJobProfile.test.tsx
View File

@ -10,7 +10,7 @@ describe('DataJobProfile', () => {
it('renders', async () => {
const { getByText, queryAllByText } = render(
<MockedProvider mocks={mocks} addTypename={false}>
<TestPageContainer initialEntries={['/task/urn:li:dataJob:1']}>
<TestPageContainer initialEntries={['/tasks/urn:li:dataJob:1']}>
<DataJobProfile urn="urn:li:dataJob:1" />
</TestPageContainer>
</MockedProvider>,

4
datahub-web-react/src/app/preview/DefaultPreviewCard.tsx
View File

@ -20,6 +20,7 @@ interface Props {
owners?: Array<Owner> | null;
snippet?: React.ReactNode;
glossaryTerms?: GlossaryTerms;
dataTestID?: string;
}
const DescriptionParagraph = styled(Typography.Paragraph)`
@ -58,11 +59,12 @@ export default function DefaultPreviewCard({
owners,
snippet,
glossaryTerms,
dataTestID,
}: Props) {
const entityRegistry = useEntityRegistry();
return (
<Row style={styles.row} justify="space-between">
<Row style={styles.row} justify="space-between" data-testid={dataTestID}>
<Space direction="vertical" align="start" size={28} style={styles.leftColumn}>
<Link to={url}>
<Space direction="horizontal" size={20} align="center">

19
datahub-web-react/src/graphql/fragments.graphql
View File

@ -200,6 +200,22 @@ fragment nonRecursiveDataFlowFields on DataFlow {
}
}
fragment nonRecursiveDataJobFields on DataJob {
urn
info {
name
description
externalUrl
customProperties {
key
value
}
}
globalTags {
...globalTagsFields
}
}
fragment dataJobFields on DataJob {
urn
type
@ -217,6 +233,9 @@ fragment dataJobFields on DataJob {
outputDatasets {
...nonRecursiveDatasetFields
}
inputDatajobs {
...nonRecursiveDataJobFields
}
}
info {
name

42
datahub-web-react/src/utils/sort/topologicalSort.ts Normal file
View File

@ -0,0 +1,42 @@
import { EntityRelationship } from '../../types.generated';
// Sort helper function
function topologicalSortHelper(
node: EntityRelationship,
explored: Set<string>,
result: Array<EntityRelationship>,
urnsArray: Array<string>,
) {
if (!node.entity?.urn) {
return;
}
explored.add(node.entity?.urn);
(node.entity.upstreamLineage?.entities || [])
.filter((entity) => entity?.entity?.urn && urnsArray.includes(entity?.entity?.urn))
.forEach((n) => {
if (n?.entity?.urn && !explored.has(n?.entity?.urn)) {
topologicalSortHelper(n, explored, result, urnsArray);
}
});
if (urnsArray.includes(node?.entity?.urn)) {
result.push(node);
}
}
// Topological Sort function with array of EntityRelationship
export function topologicalSort(input: Array<EntityRelationship | null>) {
const explored = new Set<string>();
const result: Array<EntityRelationship> = [];
const nodes: Array<EntityRelationship> = [...input] as Array<EntityRelationship>;
const urnsArray: Array<string> = nodes
.filter((node) => !!node.entity?.urn)
.map((node) => node.entity?.urn) as Array<string>;
nodes.forEach((node) => {
if (node.entity?.urn && !explored.has(node.entity?.urn)) {
topologicalSortHelper(node, explored, result, urnsArray);
}
});
return result;
}