yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 09:58:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ui) Fix lineage graph rendering with duplicate nodes (#6480)

Browse Source
This commit is contained in:
Chris Collins 2022-11-18 12:28:30 -05:00 committed by GitHub
parent 250f7ce1a8
commit 970e9f0aa7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 35 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
datahub-web-react/src/app/lineage/LineageTreeNodeAndEdgeRenderer.tsx
View File

@ -73,15 +73,14 @@ export default function LineageTreeNodeAndEdgeRenderer({
// concept of a z-index
...edgesToRender.filter((link) => !isLinkHighlighted(link)),
...edgesToRender.filter(isLinkHighlighted),
].map((link) => {
].map((link, idx) => {
const isHighlighted = isLinkHighlighted(link);
const key = `edge-${idx}-${link.source.data.urn}${link.sourceField && `-${link.sourceField}`}-${
link.target.data.urn
}${link.targetField && `-${link.targetField}`}-${direction}`;
return (
<Group
key={`edge-${link.source.data.urn}${link.sourceField && `-${link.sourceField}`}-${
link.target.data.urn
}${link.targetField && `-${link.targetField}`}-${direction}`}
>
<Group key={key}>
<LinePath
// we rotated the svg 90 degrees so we need to switch x & y for the last mile
x={(d) => d.y}

45
metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java
View File

@ -8,7 +8,7 @@ import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.metadata.entity.EntityService;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -16,6 +16,8 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -89,32 +91,39 @@ public class SiblingGraphService {
// takes a lineage result and removes any nodes that are siblings of some other node already in the result
private EntityLineageResult filterLineageResultFromSiblings(
Urn urn,
Set<Urn> allSiblingsInGroup,
EntityLineageResult entityLineageResult,
EntityLineageResult existingResult
@Nonnull final Urn urn,
@Nonnull final Set<Urn> allSiblingsInGroup,
@Nonnull final EntityLineageResult entityLineageResult,
@Nullable final EntityLineageResult existingResult
) {
// 1) remove the source entities siblings from this entity's downstreams
List<LineageRelationship> filteredRelationships = entityLineageResult.getRelationships()
final List<LineageRelationship> filteredRelationships = entityLineageResult.getRelationships()
.stream()
.filter(lineageRelationship -> !allSiblingsInGroup.contains(lineageRelationship.getEntity())
|| lineageRelationship.getEntity().equals(urn))
.collect(Collectors.toList());
// 2) combine this entity's lineage with the lineage we've already seen and remove duplicates
List<LineageRelationship> combinedResults = new ArrayList<>(Stream.concat(
filteredRelationships.stream(),
// 2) filter out existing lineage to avoid duplicates in our combined result
final Set<Urn> existingUrns = existingResult != null
? existingResult.getRelationships().stream().map(LineageRelationship::getEntity).collect(Collectors.toSet())
: new HashSet<>();
List<LineageRelationship> uniqueFilteredRelationships = filteredRelationships.stream().filter(
lineageRelationship -> !existingUrns.contains(lineageRelationship.getEntity())).collect(Collectors.toList());
// 3) combine this entity's lineage with the lineage we've already seen and remove duplicates
final List<LineageRelationship> combinedResults = Stream.concat(
uniqueFilteredRelationships.stream(),
existingResult != null ? existingResult.getRelationships().stream() : ImmutableList.<LineageRelationship>of().stream())
.collect(Collectors.toSet()));
.collect(Collectors.toList());
// 3) fetch the siblings of each lineage result
Set<Urn> combinedResultUrns = combinedResults.stream().map(result -> result.getEntity()).collect(Collectors.toSet());
// 4) fetch the siblings of each lineage result
final Set<Urn> combinedResultUrns = combinedResults.stream().map(result -> result.getEntity()).collect(Collectors.toSet());
Map<Urn, List<RecordTemplate>> siblingAspects =
final Map<Urn, List<RecordTemplate>> siblingAspects =
_entityService.getLatestAspects(combinedResultUrns, ImmutableSet.of(SIBLINGS_ASPECT_NAME));
// 4) if you are not primary & your sibling is in the results, filter yourself out of the return set
filteredRelationships = combinedResults.stream().filter(result -> {
// 5) if you are not primary & your sibling is in the results, filter yourself out of the return set
uniqueFilteredRelationships = combinedResults.stream().filter(result -> {
Optional<RecordTemplate> optionalSiblingsAspect = siblingAspects.get(result.getEntity()).stream().filter(
aspect -> aspect instanceof Siblings
).findAny();
@ -124,7 +133,7 @@ public class SiblingGraphService {
}
Siblings siblingsAspect = (Siblings) optionalSiblingsAspect.get();
final Siblings siblingsAspect = (Siblings) optionalSiblingsAspect.get();
if (siblingsAspect.isPrimary()) {
return true;
@ -140,9 +149,9 @@ public class SiblingGraphService {
return true;
}).collect(Collectors.toList());
entityLineageResult.setRelationships(new LineageRelationshipArray(filteredRelationships));
entityLineageResult.setRelationships(new LineageRelationshipArray(uniqueFilteredRelationships));
entityLineageResult.setTotal(entityLineageResult.getTotal() + (existingResult != null ? existingResult.getTotal() : 0));
entityLineageResult.setCount(filteredRelationships.size());
entityLineageResult.setCount(uniqueFilteredRelationships.size());
return entityLineageResult;
}

5
metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java
View File

@ -259,7 +259,7 @@ public class SiblingGraphServiceTest {
EntityLineageResult expectedResult = mockResult.clone();
expectedResult.setTotal(3);
expectedResult.setCount(2);
expectedResult.setRelationships(new LineageRelationshipArray(relationship2, relationship1));
expectedResult.setRelationships(new LineageRelationshipArray(relationship1, relationship2));
EntityLineageResult upstreamLineage = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1);
@ -302,7 +302,7 @@ public class SiblingGraphServiceTest {
expectedRelationships.add(relationship2);
expectedRelationships.add(relationship4);
expectedRelationships.add(relationship1);
expectedRelationships.add(relationship1); // expect just one relationship1 despite duplicates in sibling lineage
expectedResult.setCount(3);
expectedResult.setStart(0);
@ -316,6 +316,7 @@ public class SiblingGraphServiceTest {
siblingRelationships.add(relationship2);
siblingRelationships.add(relationship4);
siblingRelationships.add(relationship1); // duplicate from sibling's lineage, we should not see duplicates in result
siblingMockResult.setStart(0);
siblingMockResult.setTotal(2);
siblingMockResult.setCount(2);