yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 09:58:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ui) Fix displaying column level lineage for sibling nodes (#7955)

Browse Source
This commit is contained in:
Chris Collins 2023-05-04 10:53:18 -04:00 committed by GitHub
parent 4d63ea5220
commit 3f8a532bbc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 108 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
datahub-web-react/src/app/entity/EntityRegistry.tsx
View File

@ -3,7 +3,7 @@ import { FetchedEntity } from '../lineage/types';
import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from './Entity';
import { GLOSSARY_ENTITY_TYPES } from './shared/constants';
import { GenericEntityProperties } from './shared/types';
import { dictToQueryStringParams, urlEncodeUrn } from './shared/utils';
import { dictToQueryStringParams, getFineGrainedLineageWithSiblings, urlEncodeUrn } from './shared/utils';
function validatedGet<K, V>(key: K, map: Map<K, V>): V {
if (map.has(key)) {
@ -132,6 +132,11 @@ export default class EntityRegistry {
getLineageVizConfig<T>(type: EntityType, data: T): FetchedEntity | undefined {
const entity = validatedGet(type, this.entityTypeToEntity);
const genericEntityProperties = this.getGenericEntityProperties(type, data);
// combine fineGrainedLineages from this node as well as its siblings
const fineGrainedLineages = getFineGrainedLineageWithSiblings(
genericEntityProperties,
(t: EntityType, d: EntityInterface) => this.getGenericEntityProperties(t, d),
);
return (
({
...entity.getLineageVizConfig?.(data),
@ -161,7 +166,8 @@ export default class EntityRegistry {
(genericEntityProperties?.upstream?.filtered || 0),
status: genericEntityProperties?.status,
siblingPlatforms: genericEntityProperties?.siblingPlatforms,
fineGrainedLineages: genericEntityProperties?.fineGrainedLineages,
fineGrainedLineages,
siblings: genericEntityProperties?.siblings,
schemaMetadata: genericEntityProperties?.schemaMetadata,
inputFields: genericEntityProperties?.inputFields,
canEditLineage: genericEntityProperties?.privileges?.canEditLineage,

19
datahub-web-react/src/app/entity/shared/utils.ts
View File

@ -1,6 +1,6 @@
import * as QueryString from 'query-string';
import { MatchedField } from '../../../types.generated';
import { Entity, EntityType, MatchedField } from '../../../types.generated';
import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
import { FIELDS_TO_HIGHLIGHT } from '../dataset/search/highlights';
import { GenericEntityProperties } from './types';
@ -146,3 +146,20 @@ export const handleBatchError = (urns, e, defaultMessage) => {
}
return defaultMessage;
};
// put all of the fineGrainedLineages for a given entity and its siblings into one array so we have all of it in one place
export function getFineGrainedLineageWithSiblings(
entityData: GenericEntityProperties | null,
getGenericEntityProperties: (type: EntityType, data: Entity) => GenericEntityProperties | null,
) {
const fineGrainedLineages = [...(entityData?.fineGrainedLineages || [])];
entityData?.siblings?.siblings?.forEach((sibling) => {
if (sibling) {
const genericSiblingProps = getGenericEntityProperties(sibling.type, sibling);
if (genericSiblingProps && genericSiblingProps.fineGrainedLineages) {
fineGrainedLineages.push(...genericSiblingProps.fineGrainedLineages);
}
}
});
return fineGrainedLineages;
}

6
datahub-web-react/src/app/lineage/LineageExplorer.tsx
View File

@ -58,6 +58,7 @@ export default function LineageExplorer({ urn, type }: Props) {
const previousUrn = usePrevious(urn);
const history = useHistory();
const [fineGrainedMap] = useState<any>({ forward: {}, reverse: {} });
const [fineGrainedMapForSiblings] = useState<any>({});
const entityRegistry = useEntityRegistry();
const isHideSiblingMode = useIsSeparateSiblingsMode();
@ -100,6 +101,7 @@ export default function LineageExplorer({ urn, type }: Props) {
// record that we have added this entity
let newAsyncEntities = extendAsyncEntities(
fineGrainedMap,
fineGrainedMapForSiblings,
asyncEntities,
entityRegistry,
entityAndType,
@ -110,6 +112,7 @@ export default function LineageExplorer({ urn, type }: Props) {
config?.downstreamChildren?.forEach((downstream) => {
newAsyncEntities = extendAsyncEntities(
fineGrainedMap,
fineGrainedMapForSiblings,
newAsyncEntities,
entityRegistry,
downstream,
@ -119,6 +122,7 @@ export default function LineageExplorer({ urn, type }: Props) {
config?.upstreamChildren?.forEach((downstream) => {
newAsyncEntities = extendAsyncEntities(
fineGrainedMap,
fineGrainedMapForSiblings,
newAsyncEntities,
entityRegistry,
downstream,
@ -128,7 +132,7 @@ export default function LineageExplorer({ urn, type }: Props) {
setAsyncEntities(newAsyncEntities);
}
},
[asyncEntities, setAsyncEntities, entityRegistry, fineGrainedMap],
[asyncEntities, setAsyncEntities, entityRegistry, fineGrainedMap, fineGrainedMapForSiblings],
);
// set asyncEntity to have fullyFetched: false so we can update it in maybeAddAsyncLoadedEntity

1
datahub-web-react/src/app/lineage/__tests__/LineageTree.test.tsx
View File

@ -43,6 +43,7 @@ describe('LineageTree', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,

5
datahub-web-react/src/app/lineage/__tests__/adjustVXTreeLayout.test.tsx
View File

@ -30,6 +30,7 @@ describe('adjustVXTreeLayout', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -80,6 +81,7 @@ describe('adjustVXTreeLayout', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -135,6 +137,7 @@ describe('adjustVXTreeLayout', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -179,6 +182,7 @@ describe('adjustVXTreeLayout', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -223,6 +227,7 @@ describe('adjustVXTreeLayout', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,

7
datahub-web-react/src/app/lineage/__tests__/constructTree.test.ts
View File

@ -54,6 +54,7 @@ describe('constructTree', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -105,6 +106,7 @@ describe('constructTree', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -157,6 +159,7 @@ describe('constructTree', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -251,6 +254,7 @@ describe('constructTree', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -281,6 +285,7 @@ describe('constructTree', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -367,6 +372,7 @@ describe('constructTree', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,
@ -421,6 +427,7 @@ describe('constructTree', () => {
const mockFetchedEntities = fetchedEntities.reduce(
(acc, entry) =>
extendAsyncEntities(
{},
{},
acc,
testEntityRegistry,

2
datahub-web-react/src/app/lineage/types.ts
View File

@ -18,6 +18,7 @@ import {
InputFields,
Entity,
LineageRelationship,
SiblingProperties,
} from '../../types.generated';
export type EntitySelectParams = {
@ -51,6 +52,7 @@ export type FetchedEntity = {
status?: Maybe<Status>;
siblingPlatforms?: Maybe<DataPlatform[]>;
fineGrainedLineages?: [FineGrainedLineage];
siblings?: Maybe<SiblingProperties>;
schemaMetadata?: SchemaMetadata;
inputFields?: InputFields;
canEditLineage?: boolean;

1
datahub-web-react/src/app/lineage/utils/constructTree.ts
View File

@ -72,6 +72,7 @@ export default function constructTree(
(updatedLineage as any).entitiesToAdd.forEach((entity) => {
if (!(entity.urn in updatedFetchedEntities)) {
updatedFetchedEntities = extendAsyncEntities(
{},
{},
updatedFetchedEntities,
entityRegistry,

67
datahub-web-react/src/app/lineage/utils/extendAsyncEntities.ts
View File

@ -23,7 +23,7 @@ function updateFineGrainedMap(
) {
const mapForUrn = fineGrainedMap.forward[upstreamEntityUrn] || {};
const mapForField = mapForUrn[upstreamField] || {};
const listForDownstream = mapForField[downstreamEntityUrn] || [];
const listForDownstream = [...(mapForField[downstreamEntityUrn] || [])];
listForDownstream.push(downstreamField);
// eslint-disable-next-line no-param-reassign
@ -33,7 +33,7 @@ function updateFineGrainedMap(
const mapForUrnReverse = fineGrainedMap.reverse[downstreamEntityUrn] || {};
const mapForFieldReverse = mapForUrnReverse[downstreamField] || {};
const listForDownstreamReverse = mapForFieldReverse[upstreamEntityUrn] || [];
const listForDownstreamReverse = [...(mapForFieldReverse[upstreamEntityUrn] || [])];
listForDownstreamReverse.push(upstreamField);
// eslint-disable-next-line no-param-reassign
@ -42,24 +42,78 @@ function updateFineGrainedMap(
mapForFieldReverse[upstreamEntityUrn] = listForDownstreamReverse;
}
function extendColumnLineage(lineageVizConfig: FetchedEntity, fineGrainedMap: any) {
function extendColumnLineage(
lineageVizConfig: FetchedEntity,
fineGrainedMap: any,
fineGrainedMapForSiblings: any,
fetchedEntities: FetchedEntities,
) {
if (lineageVizConfig.fineGrainedLineages && lineageVizConfig.fineGrainedLineages.length > 0) {
lineageVizConfig.fineGrainedLineages.forEach((fineGrainedLineage) => {
fineGrainedLineage.upstreams?.forEach((upstream) => {
const [upstreamEntityUrn, upstreamField] = breakFieldUrn(upstream);
fineGrainedLineage.downstreams?.forEach((downstream) => {
const [downstreamEntityUrn, downstreamField] = breakFieldUrn(downstream);
const downstreamField = breakFieldUrn(downstream)[1];
// fineGrainedLineage always belongs on the downstream urn with upstreams pointing to another entity
// pass in the visualized node's urn and not the urn from the schema field as the downstream urn,
// as they will either be the same or if they are different, it belongs to a "hidden" sibling
updateFineGrainedMap(
fineGrainedMap,
upstreamEntityUrn,
upstreamField,
downstreamEntityUrn,
lineageVizConfig.urn,
downstreamField,
);
// upstreamEntityUrn could belong to a sibling we don't "render", so store its inputs to updateFineGrainedMap
// and update the fine grained map later when we see the entity with these siblings
// eslint-disable-next-line no-param-reassign
fineGrainedMapForSiblings[upstreamEntityUrn] = [
...(fineGrainedMapForSiblings[upstreamEntityUrn] || []),
{
upstreamField,
downstreamEntityUrn: lineageVizConfig.urn,
downstreamField,
},
];
// if this upstreamEntityUrn is a sibling of one of the already rendered nodes,
// update the fine grained map with the rendered node instead of its sibling
Object.keys(fetchedEntities).forEach((urn) => {
fetchedEntities[urn].siblings?.siblings?.forEach((sibling) => {
if (sibling && sibling.urn === upstreamEntityUrn) {
updateFineGrainedMap(
fineGrainedMap,
urn,
upstreamField,
lineageVizConfig.urn,
downstreamField,
);
}
});
});
});
});
});
}
// if we've seen fineGrainedMappings for this current entity's siblings, update the
// fine grained map with the rendered urn instead of the "hidden" sibling urn
lineageVizConfig.siblings?.siblings?.forEach((sibling) => {
if (sibling && fineGrainedMapForSiblings[sibling.urn]) {
fineGrainedMapForSiblings[sibling.urn].forEach((entry) => {
updateFineGrainedMap(
fineGrainedMap,
lineageVizConfig.urn,
entry.upstreamField,
entry.downstreamEntityUrn,
entry.downstreamField,
);
});
}
});
// below is to update the fineGrainedMap for Data Jobs
if (lineageVizConfig.inputFields?.fields && lineageVizConfig.inputFields.fields.length > 0) {
lineageVizConfig.inputFields.fields.forEach((inputField) => {
if (inputField?.schemaFieldUrn && inputField.schemaField) {
@ -80,6 +134,7 @@ function extendColumnLineage(lineageVizConfig: FetchedEntity, fineGrainedMap: an
export default function extendAsyncEntities(
fineGrainedMap: any,
fineGrainedMapForSiblings: any,
fetchedEntities: FetchedEntities,
entityRegistry: EntityRegistry,
entityAndType: EntityAndType,
@ -93,7 +148,7 @@ export default function extendAsyncEntities(
if (!lineageVizConfig) return fetchedEntities;
extendColumnLineage(lineageVizConfig, fineGrainedMap);
extendColumnLineage(lineageVizConfig, fineGrainedMap, fineGrainedMapForSiblings, fetchedEntities);
return {
...fetchedEntities,