diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java index b9683e0148..d44f2b7702 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java @@ -79,6 +79,7 @@ public class EntityLineageResultResolver implements DataFetcher relationship.entity) - // eslint-disable-next-line @typescript-eslint/dot-notation - ?.filter((relationship) => !relationship.entity?.['status']?.removed) ?.map((relationship) => ({ entity: relationship.entity as EntityInterface, type: (relationship.entity as EntityInterface).type, })), - downstreamRelationships: genericEntityProperties?.downstream?.relationships - ?.filter((relationship) => relationship.entity) - // eslint-disable-next-line @typescript-eslint/dot-notation - ?.filter((relationship) => !relationship.entity?.['status']?.removed), - numDownstreamChildren: genericEntityProperties?.downstream?.total, + downstreamRelationships: genericEntityProperties?.downstream?.relationships?.filter( + (relationship) => relationship.entity, + ), + numDownstreamChildren: + (genericEntityProperties?.downstream?.total || 0) - + (genericEntityProperties?.downstream?.filtered || 0), upstreamChildren: genericEntityProperties?.upstream?.relationships ?.filter((relationship) => relationship.entity) - // eslint-disable-next-line @typescript-eslint/dot-notation - ?.filter((relationship) => !relationship.entity?.['status']?.removed) ?.map((relationship) => ({ entity: relationship.entity as EntityInterface, type: (relationship.entity as EntityInterface).type, })), - upstreamRelationships: genericEntityProperties?.upstream?.relationships - ?.filter((relationship) => relationship.entity) - // eslint-disable-next-line @typescript-eslint/dot-notation - ?.filter((relationship) => !relationship.entity?.['status']?.removed), - numUpstreamChildren: genericEntityProperties?.upstream?.total, + upstreamRelationships: genericEntityProperties?.upstream?.relationships?.filter( + (relationship) => relationship.entity, + ), + numUpstreamChildren: + (genericEntityProperties?.upstream?.total || 0) - + (genericEntityProperties?.upstream?.filtered || 0), status: genericEntityProperties?.status, siblingPlatforms: genericEntityProperties?.siblingPlatforms, fineGrainedLineages: genericEntityProperties?.fineGrainedLineages, diff --git a/datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts b/datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts index 5d61983f57..40b0351871 100644 --- a/datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts +++ b/datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts @@ -44,7 +44,6 @@ export const navigateToLineageUrl = ({ } const newSearchStringified = QueryString.stringify(newSearch, { arrayFormat: 'comma' }); - console.log(location.pathname); history.push({ pathname: location.pathname, search: newSearchStringified, diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index d6a84d9d8b..b18a2d513d 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -251,6 +251,7 @@ fragment fullLineageResults on EntityLineageResult { start count total + filtered relationships { type createdOn @@ -308,6 +309,7 @@ fragment leafLineageResults on EntityLineageResult { start count total + filtered relationships { type entity { @@ -321,6 +323,7 @@ fragment partialLineageResults on EntityLineageResult { start count total + filtered } query getEntityLineage( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index 39808565ed..e87fb5e09a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -76,6 +76,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -1608,6 +1609,12 @@ public class EntityService { return new RollbackRunResult(removedAspects, rowsDeletedFromEntityDeletion); } + /** + * Returns true if the entity exists (has materialized aspects) + * + * @param urn the urn of the entity to check + * @return true if the entity exists, false otherwise + */ public Boolean exists(Urn urn) { final Set aspectsToFetch = getEntityAspectNames(urn); final List dbKeys = aspectsToFetch.stream() @@ -1618,6 +1625,18 @@ public class EntityService { return aspects.values().stream().anyMatch(aspect -> aspect != null); } + /** + * Returns true if an entity is soft-deleted. + * + * @param urn the urn to check + * @return true is the entity is soft deleted, false otherwise. + */ + public Boolean isSoftDeleted(@Nonnull final Urn urn) { + Objects.requireNonNull(urn, "urn is required"); + final RecordTemplate statusAspect = getLatestAspect(urn, STATUS_ASPECT_NAME); + return statusAspect != null && ((Status) statusAspect).isRemoved(); + } + @Nullable public RollbackResult deleteAspect(String urn, String aspectName, @Nonnull Map conditions, boolean hardDelete) { // Validate pre-conditions before running queries diff --git a/metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java index 0967c32985..64e9687151 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java @@ -119,8 +119,11 @@ public class ValidationUtils { final LineageRelationshipArray validatedRelationships = entityLineageResult.getRelationships().stream() .filter(relationship -> entityService.exists(relationship.getEntity())) + .filter(relationship -> !entityService.isSoftDeleted(relationship.getEntity())) .collect(Collectors.toCollection(LineageRelationshipArray::new)); + validatedEntityLineageResult.setFiltered( + entityLineageResult.getRelationships().size() - validatedRelationships.size()); validatedEntityLineageResult.setRelationships(validatedRelationships); return validatedEntityLineageResult; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java index d2237c9a44..388763789f 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java @@ -93,6 +93,7 @@ public class SiblingGraphServiceTest { mockResult.setStart(0); mockResult.setTotal(200); mockResult.setCount(3); + mockResult.setFiltered(0); mockResult.setRelationships(relationships); when(_graphService.getLineage( @@ -137,6 +138,7 @@ public class SiblingGraphServiceTest { mockResult.setStart(0); mockResult.setTotal(200); mockResult.setCount(3); + mockResult.setFiltered(0); mockResult.setRelationships(relationships); when(_graphService.getLineage( @@ -261,6 +263,7 @@ public class SiblingGraphServiceTest { EntityLineageResult expectedResult = mockResult.clone(); expectedResult.setTotal(3); expectedResult.setCount(2); + expectedResult.setFiltered(0); expectedResult.setRelationships(new LineageRelationshipArray(relationship1, relationship2)); EntityLineageResult upstreamLineage = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1); @@ -309,6 +312,7 @@ public class SiblingGraphServiceTest { expectedResult.setCount(3); expectedResult.setStart(0); expectedResult.setTotal(3); + expectedResult.setFiltered(0); expectedResult.setRelationships(expectedRelationships); mockResult.setStart(0); @@ -406,6 +410,7 @@ public class SiblingGraphServiceTest { expectedResult.setCount(2); expectedResult.setStart(0); expectedResult.setTotal(3); + expectedResult.setFiltered(0); expectedResult.setRelationships(expectedRelationships); mockResult.setStart(0); @@ -491,6 +496,7 @@ public class SiblingGraphServiceTest { expectedResult.setCount(1); expectedResult.setStart(0); expectedResult.setTotal(1); + expectedResult.setFiltered(0); expectedResult.setRelationships(expectedRelationships); mockResult.setStart(0); diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/EntityLineageResult.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/EntityLineageResult.pdl index dc2301db41..78a7bc6ef2 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/EntityLineageResult.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/EntityLineageResult.pdl @@ -19,6 +19,11 @@ record EntityLineageResult { */ total: int + /** + * The number of results that were filtered out of the page (soft-deleted or non-existent) + */ + filtered: optional int = 0 + /** * Relationships in the result set */ diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json index 840900547a..52b9dfc395 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json @@ -96,6 +96,11 @@ "name" : "total", "type" : "int", "doc" : "Total number of results in the result set" + }, { + "name" : "filtered", + "type" : "int", + "doc" : "The number of results that were filtered out of the page (soft-deleted or non-existent)", + "optional" : true }, { "name" : "relationships", "type" : {