From c0aa3ecb4b5ebdfc6024a990607491725ccf1a6d Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Fri, 17 Sep 2021 22:54:24 +0200 Subject: [PATCH] test(GraphService): Thorough graph service tests (#3011) --- .../linkedin/metadata/graph/GraphService.java | 64 + .../metadata/graph/Neo4jGraphService.java | 2 +- .../graph/ElasticSearchGraphServiceTest.java | 153 +- .../metadata/graph/GraphServiceTestBase.java | 1480 +++++++++++++++-- .../metadata/graph/Neo4jGraphServiceTest.java | 116 +- 5 files changed, 1701 insertions(+), 114 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java index b8bff02aa6..ddf369ca96 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java @@ -9,8 +9,58 @@ import javax.annotation.Nullable; public interface GraphService { + /** + * Adds an edge to the graph. This creates the source and destination nodes, if they do not exist. + */ void addEdge(final Edge edge); + /** + * Find related entities (nodes) connected to a source entity via edges of given relationship types. Related entities + * can be filtered by source and destination type (use `null` for any type), by source and destination entity filter + * and relationship filter. Pagination of the result is controlled via `offset` and `count`. + * + * Starting from a node as the source entity, determined by `sourceType` and `sourceEntityFilter`, + * related entities are found along the direction of edges (`RelationshipDirection.OUTGOING`) or in opposite + * direction of edges (`RelationshipDirection.INCOMING`). The destination entities are further filtered by `destinationType` + * and `destinationEntityFilter`, and then returned as related entities. + * + * This does not return duplicate related entities, even if entities are connected to source entities via multiple edges. + * An empty list of relationship types returns an empty result. + * + * In other words, the source and destination entity is not to be understood as the source and destination of the edge, + * but as the source and destination of "finding related entities", where always the destination entities are returned. + * This understanding is important when it comes to `RelationshipDirection.INCOMING`. The origin of the edge becomes + * the destination entity and the source entity is where the edge points to. + * + * Example I: + * dataset one --DownstreamOf-> dataset two --DownstreamOf-> dataset three + * + * findRelatedEntities(null, EMPTY_FILTER, null, EMPTY_FILTER, ["DownstreamOf"], RelationshipFilter.setDirection(RelationshipDirection.OUTGOING), 0, 100) + * - RelatedEntity("DownstreamOf", "dataset two") + * - RelatedEntity("DownstreamOf", "dataset three") + * + * findRelatedEntities(null, EMPTY_FILTER, null, EMPTY_FILTER, ["DownstreamOf"], RelationshipFilter.setDirection(RelationshipDirection.INCOMING), 0, 100) + * - RelatedEntity("DownstreamOf", "dataset one") + * - RelatedEntity("DownstreamOf", "dataset two") + * + * Example II: + * dataset one --HasOwner-> user one + * + * findRelatedEntities(null, EMPTY_FILTER, null, EMPTY_FILTER, ["HasOwner"], RelationshipFilter.setDirection(RelationshipDirection.OUTGOING), 0, 100) + * - RelatedEntity("HasOwner", "user one") + * + * findRelatedEntities(null, EMPTY_FILTER, null, EMPTY_FILTER, ["HasOwner"], RelationshipFilter.setDirection(RelationshipDirection.INCOMING), 0, 100) + * - RelatedEntity("HasOwner", "dataset one") + * + * Calling this method with {@link com.linkedin.metadata.query.RelationshipDirection} `UNDIRECTED` in `relationshipFilter` + * is equivalent to the union of `OUTGOING` and `INCOMING` (without duplicates). + * + * Example III: + * findRelatedEntities(null, EMPTY_FILTER, null, EMPTY_FILTER, ["DownstreamOf"], RelationshipFilter.setDirection(RelationshipDirection.UNDIRECTED), 0, 100) + * - RelatedEntity("DownstreamOf", "dataset one") + * - RelatedEntity("DownstreamOf", "dataset two") + * - RelatedEntity("DownstreamOf", "dataset three") + */ @Nonnull RelatedEntitiesResult findRelatedEntities( @Nullable final String sourceType, @@ -22,8 +72,19 @@ public interface GraphService { final int offset, final int count); + /** + * Removes the given node (if it exists) as well as all edges (incoming and outgoing) of the node. + */ void removeNode(@Nonnull final Urn urn); + /** + * Removes edges of the given relationship types from the given node after applying the relationship filter. + * + * An empty list of relationship types removes nothing from the node. + * + * Calling this method with a {@link com.linkedin.metadata.query.RelationshipDirection} `UNDIRECTED` in `relationshipFilter` + * is equivalent to the union of `OUTGOING` and `INCOMING` (without duplicates). + */ void removeEdgesFromNode( @Nonnull final Urn urn, @Nonnull final List relationshipTypes, @@ -31,5 +92,8 @@ public interface GraphService { void configure(); + /** + * Removes all edges and nodes from the graph. + */ void clear(); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/Neo4jGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/Neo4jGraphService.java index f22b3205b8..5e4f3c0213 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/Neo4jGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/Neo4jGraphService.java @@ -114,7 +114,7 @@ public class Neo4jGraphService implements GraphService { matchTemplate = "MATCH (src%s %s)-[r%s %s]->(dest%s %s)"; } - final String returnNodes = "RETURN dest, type(r)"; // Return both related entity and the relationship type. + final String returnNodes = String.format("RETURN dest%s, type(r)", destinationType); // Return both related entity and the relationship type. final String returnCount = "RETURN count(*)"; // For getting the total results. String relationshipTypeFilter = ""; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/ElasticSearchGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/ElasticSearchGraphServiceTest.java index 8d3a8125aa..c3768cd63d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/ElasticSearchGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/ElasticSearchGraphServiceTest.java @@ -1,9 +1,13 @@ package com.linkedin.metadata.graph; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.ElasticSearchTestUtils; import com.linkedin.metadata.graph.elastic.ESGraphQueryDAO; import com.linkedin.metadata.graph.elastic.ESGraphWriteDAO; import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; +import com.linkedin.metadata.query.Filter; +import com.linkedin.metadata.query.RelationshipDirection; +import com.linkedin.metadata.query.RelationshipFilter; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; import org.apache.http.HttpHost; @@ -12,11 +16,18 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testng.SkipException; import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; import javax.annotation.Nonnull; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; + +import static org.testng.Assert.assertEquals; import static com.linkedin.metadata.graph.elastic.ElasticSearchGraphService.INDEX_NAME; @@ -31,12 +42,6 @@ public class ElasticSearchGraphServiceTest extends GraphServiceTestBase { private static final String IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch:7.9.3"; private static final int HTTP_PORT = 9200; - @BeforeMethod - public void wipe() throws Exception { - _client.clear(); - syncAfterWrite(); - } - @BeforeTest public void setup() { _elasticsearchContainer = new ElasticsearchContainer(IMAGE_NAME); @@ -46,6 +51,12 @@ public class ElasticSearchGraphServiceTest extends GraphServiceTestBase { _client.configure(); } + @BeforeMethod + public void wipe() throws Exception { + _client.clear(); + syncAfterWrite(); + } + @Nonnull private RestHighLevelClient buildRestClient() { final RestClientBuilder builder = @@ -81,4 +92,134 @@ public class ElasticSearchGraphServiceTest extends GraphServiceTestBase { ElasticSearchTestUtils.syncAfterWrite(_searchClient, _indexName); } + @Override + protected void assertEqualsAnyOrder(RelatedEntitiesResult actual, RelatedEntitiesResult expected) { + // https://github.com/linkedin/datahub/issues/3115 + // ElasticSearchGraphService produces duplicates, which is here ignored until fixed + // actual.count and actual.total not tested due to duplicates + assertEquals(actual.start, expected.start); + assertEqualsAnyOrder(actual.entities, expected.entities, RELATED_ENTITY_COMPARATOR); + } + + @Override + protected void assertEqualsAnyOrder(List actual, List expected, Comparator comparator) { + // https://github.com/linkedin/datahub/issues/3115 + // ElasticSearchGraphService produces duplicates, which is here ignored until fixed + assertEquals( + new HashSet<>(actual), + new HashSet<>(expected) + ); + } + + @Override + public void testFindRelatedEntitiesSourceEntityFilter(Filter sourceEntityFilter, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + if (relationships.getDirection() == RelationshipDirection.UNDIRECTED) { + // https://github.com/linkedin/datahub/issues/3114 + throw new SkipException("ElasticSearchGraphService does not implement UNDIRECTED relationship filter"); + } + super.testFindRelatedEntitiesSourceEntityFilter(sourceEntityFilter, relationshipTypes, relationships, expectedRelatedEntities); + } + + @Override + public void testFindRelatedEntitiesDestinationEntityFilter(Filter destinationEntityFilter, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + if (relationships.getDirection() == RelationshipDirection.UNDIRECTED) { + // https://github.com/linkedin/datahub/issues/3114 + throw new SkipException("ElasticSearchGraphService does not implement UNDIRECTED relationship filter"); + } + super.testFindRelatedEntitiesDestinationEntityFilter(destinationEntityFilter, relationshipTypes, relationships, expectedRelatedEntities); + } + + @Override + public void testFindRelatedEntitiesSourceType(String datasetType, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + if (relationships.getDirection() == RelationshipDirection.UNDIRECTED) { + // https://github.com/linkedin/datahub/issues/3114 + throw new SkipException("ElasticSearchGraphService does not implement UNDIRECTED relationship filter"); + } + if (datasetType != null && datasetType.isEmpty()) { + // https://github.com/linkedin/datahub/issues/3116 + throw new SkipException("ElasticSearchGraphService does not support empty source type"); + } + super.testFindRelatedEntitiesSourceType(datasetType, relationshipTypes, relationships, expectedRelatedEntities); + } + + @Override + public void testFindRelatedEntitiesDestinationType(String datasetType, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + if (relationships.getDirection() == RelationshipDirection.UNDIRECTED) { + // https://github.com/linkedin/datahub/issues/3114 + throw new SkipException("ElasticSearchGraphService does not implement UNDIRECTED relationship filter"); + } + if (datasetType != null && datasetType.isEmpty()) { + // https://github.com/linkedin/datahub/issues/3116 + throw new SkipException("ElasticSearchGraphService does not support empty destination type"); + } + super.testFindRelatedEntitiesDestinationType(datasetType, relationshipTypes, relationships, expectedRelatedEntities); + } + + @Test + @Override + public void testFindRelatedEntitiesNoRelationshipTypes() { + // https://github.com/linkedin/datahub/issues/3117 + throw new SkipException("ElasticSearchGraphService does not support empty list of relationship types"); + } + + @Override + public void testRemoveEdgesFromNode(@Nonnull Urn nodeToRemoveFrom, + @Nonnull List relationTypes, + @Nonnull RelationshipFilter relationshipFilter, + List expectedOutgoingRelatedUrnsBeforeRemove, + List expectedIncomingRelatedUrnsBeforeRemove, + List expectedOutgoingRelatedUrnsAfterRemove, + List expectedIncomingRelatedUrnsAfterRemove) throws Exception { + if (relationshipFilter.getDirection() == RelationshipDirection.UNDIRECTED) { + // https://github.com/linkedin/datahub/issues/3114 + throw new SkipException("ElasticSearchGraphService does not implement UNDIRECTED relationship filter"); + } + super.testRemoveEdgesFromNode( + nodeToRemoveFrom, + relationTypes, relationshipFilter, + expectedOutgoingRelatedUrnsBeforeRemove, expectedIncomingRelatedUrnsBeforeRemove, + expectedOutgoingRelatedUrnsAfterRemove, expectedIncomingRelatedUrnsAfterRemove + ); + } + + @Test + @Override + public void testRemoveEdgesFromNodeNoRelationshipTypes() { + // https://github.com/linkedin/datahub/issues/3117 + throw new SkipException("ElasticSearchGraphService does not support empty list of relationship types"); + } + + @Test + @Override + public void testConcurrentAddEdge() { + // https://github.com/linkedin/datahub/issues/3124 + throw new SkipException("This test is flaky for ElasticSearchGraphService, ~5% of the runs fail on a race condition"); + } + + @Test + @Override + public void testConcurrentRemoveEdgesFromNode() { + // https://github.com/linkedin/datahub/issues/3118 + throw new SkipException("ElasticSearchGraphService produces duplicates"); + } + + @Test + @Override + public void testConcurrentRemoveNodes() { + // https://github.com/linkedin/datahub/issues/3118 + throw new SkipException("ElasticSearchGraphService produces duplicates"); + } + } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java index e258529cfd..754007b9ab 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java @@ -1,19 +1,34 @@ package com.linkedin.metadata.graph; import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.query.Filter; import com.linkedin.metadata.query.RelationshipDirection; import com.linkedin.metadata.query.RelationshipFilter; -import java.util.stream.Collectors; +import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.net.URISyntaxException; +import java.util.Arrays; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.linkedin.metadata.dao.utils.QueryUtils.EMPTY_FILTER; import static com.linkedin.metadata.dao.utils.QueryUtils.newFilter; -import static org.testng.Assert.assertEquals; - +import static com.linkedin.metadata.dao.utils.QueryUtils.newRelationshipFilter; +import static org.testng.Assert.*; /** * Base class for testing any GraphService implementation. @@ -22,9 +37,107 @@ import static org.testng.Assert.assertEquals; * * You can add implementation specific tests in derived classes, or add general tests * here and have all existing implementations tested in the same way. + * + * The `getPopulatedGraphService` method calls `GraphService.addEdge` to provide a populated Graph. + * Feel free to add a test to your test implementation that calls `getPopulatedGraphService` and + * asserts the state of the graph in an implementation specific way. */ abstract public class GraphServiceTestBase { + private static class RelatedEntityComparator implements Comparator { + @Override + public int compare(RelatedEntity left, RelatedEntity right) { + int cmp = left.relationshipType.compareTo(right.relationshipType); + if (cmp != 0) { + return cmp; + } + return left.urn.compareTo(right.urn); + } + } + + protected static final RelatedEntityComparator RELATED_ENTITY_COMPARATOR = new RelatedEntityComparator(); + + /** + * Some test URN types. + */ + protected static String datasetType = "dataset"; + protected static String userType = "user"; + + /** + * Some test datasets. + */ + protected static String datasetOneUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetOne,PROD)"; + protected static String datasetTwoUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetTwo,PROD)"; + protected static String datasetThreeUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetThree,PROD)"; + protected static String datasetFourUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetFour,PROD)"; + + protected static Urn datasetOneUrn = createFromString(datasetOneUrnString); + protected static Urn datasetTwoUrn = createFromString(datasetTwoUrnString); + protected static Urn datasetThreeUrn = createFromString(datasetThreeUrnString); + protected static Urn datasetFourUrn = createFromString(datasetFourUrnString); + + protected static String unknownUrnString = "urn:li:unknown:(urn:li:unknown:Unknown)"; + + /** + * Some dataset owners. + */ + protected static String userOneUrnString = "urn:li:" + userType + ":(urn:li:user:system,Ingress,PROD)"; + protected static String userTwoUrnString = "urn:li:" + userType + ":(urn:li:user:individual,UserA,DEV)"; + + protected static Urn userOneUrn = createFromString(userOneUrnString); + protected static Urn userTwoUrn = createFromString(userTwoUrnString); + + protected static Urn unknownUrn = createFromString(unknownUrnString); + + /** + * Some test relationships. + */ + protected static String downstreamOf = "DownstreamOf"; + protected static String hasOwner = "HasOwner"; + protected static String knowsUser = "KnowsUser"; + protected static Set allRelationshipTypes = new HashSet<>(Arrays.asList(downstreamOf, hasOwner, knowsUser)); + + /** + * Some expected related entities. + */ + protected static RelatedEntity downstreamOfDatasetOneRelatedEntity = new RelatedEntity(downstreamOf, datasetOneUrnString); + protected static RelatedEntity downstreamOfDatasetTwoRelatedEntity = new RelatedEntity(downstreamOf, datasetTwoUrnString); + protected static RelatedEntity downstreamOfDatasetThreeRelatedEntity = new RelatedEntity(downstreamOf, datasetThreeUrnString); + protected static RelatedEntity downstreamOfDatasetFourRelatedEntity = new RelatedEntity(downstreamOf, datasetFourUrnString); + + protected static RelatedEntity hasOwnerDatasetOneRelatedEntity = new RelatedEntity(hasOwner, datasetOneUrnString); + protected static RelatedEntity hasOwnerDatasetTwoRelatedEntity = new RelatedEntity(hasOwner, datasetTwoUrnString); + protected static RelatedEntity hasOwnerDatasetThreeRelatedEntity = new RelatedEntity(hasOwner, datasetThreeUrnString); + protected static RelatedEntity hasOwnerDatasetFourRelatedEntity = new RelatedEntity(hasOwner, datasetFourUrnString); + protected static RelatedEntity hasOwnerUserOneRelatedEntity = new RelatedEntity(hasOwner, userOneUrnString); + protected static RelatedEntity hasOwnerUserTwoRelatedEntity = new RelatedEntity(hasOwner, userTwoUrnString); + + protected static RelatedEntity knowsUserOneRelatedEntity = new RelatedEntity(knowsUser, userOneUrnString); + protected static RelatedEntity knowsUserTwoRelatedEntity = new RelatedEntity(knowsUser, userTwoUrnString); + + /** + * Some relationship filters. + */ + protected static RelationshipFilter outgoingRelationships = newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.OUTGOING); + protected static RelationshipFilter incomingRelationships = newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING); + protected static RelationshipFilter undirectedRelationships = newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.UNDIRECTED); + + /** + * Any source and destination type value. + */ + protected static @Nullable String anyType = null; + + @Test + public void testStaticUrns() { + assertNotNull(datasetOneUrn); + assertNotNull(datasetTwoUrn); + assertNotNull(datasetThreeUrn); + assertNotNull(datasetFourUrn); + + assertNotNull(userOneUrn); + assertNotNull(userTwoUrn); + } + /** * Provides the current GraphService instance to test. This is being called by the test method * at most once. The serviced graph should be empty. @@ -43,129 +156,1286 @@ abstract public class GraphServiceTestBase { */ abstract protected void syncAfterWrite() throws Exception; - @Test - public void testAddEdge() throws Exception { - GraphService client = getGraphService(); + /** + * Calls getGraphService to retrieve the test GraphService and populates it + * with edges via `GraphService.addEdge`. + * + * @return test GraphService + * @throws Exception on failure + */ + protected GraphService getPopulatedGraphService() throws Exception { + GraphService service = getGraphService(); - Edge edge1 = new Edge( - Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"), - Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"), - "DownstreamOf"); + List edges = Arrays.asList( + new Edge(datasetTwoUrn, datasetOneUrn, downstreamOf), + new Edge(datasetThreeUrn, datasetTwoUrn, downstreamOf), + new Edge(datasetFourUrn, datasetTwoUrn, downstreamOf), - client.addEdge(edge1); + new Edge(datasetOneUrn, userOneUrn, hasOwner), + new Edge(datasetTwoUrn, userOneUrn, hasOwner), + new Edge(datasetThreeUrn, userTwoUrn, hasOwner), + new Edge(datasetFourUrn, userTwoUrn, hasOwner), + + new Edge(userOneUrn, userTwoUrn, knowsUser), + new Edge(userTwoUrn, userOneUrn, knowsUser) + ); + + edges.forEach(service::addEdge); syncAfterWrite(); - List edgeTypes = new ArrayList<>(); - edgeTypes.add("DownstreamOf"); - RelationshipFilter relationshipFilter = new RelationshipFilter(); - relationshipFilter.setDirection(RelationshipDirection.OUTGOING); - relationshipFilter.setCriteria(EMPTY_FILTER.getCriteria()); + return service; + } - List relatedUrns = client.findRelatedEntities( - "", - newFilter("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"), - "", - EMPTY_FILTER, - edgeTypes, - relationshipFilter, - 0, - 10).getEntities() - .stream() - .map(RelatedEntity::getUrn) - .collect(Collectors.toList()); + protected static @Nullable + Urn createFromString(@Nonnull String rawUrn) { + try { + return Urn.createFromString(rawUrn); + } catch (URISyntaxException e) { + return null; + } + } - assertEquals(relatedUrns.size(), 1); + protected void assertEqualsAnyOrder(RelatedEntitiesResult actual, List expected) { + assertEqualsAnyOrder(actual, new RelatedEntitiesResult(0, expected.size(), expected.size(), expected)); + } + + protected void assertEqualsAnyOrder(RelatedEntitiesResult actual, RelatedEntitiesResult expected) { + assertEquals(actual.start, expected.start); + assertEquals(actual.count, expected.count); + assertEquals(actual.total, expected.total); + assertEqualsAnyOrder(actual.entities, expected.entities, RELATED_ENTITY_COMPARATOR); + } + + protected void assertEqualsAnyOrder(List actual, List expected) { + assertEquals( + actual.stream().sorted().collect(Collectors.toList()), + expected.stream().sorted().collect(Collectors.toList()) + ); + } + + protected void assertEqualsAnyOrder(List actual, List expected, Comparator comparator) { + assertEquals( + actual.stream().sorted(comparator).collect(Collectors.toList()), + expected.stream().sorted(comparator).collect(Collectors.toList()) + ); + } + + @DataProvider(name = "AddEdgeTests") + public Object[][] getAddEdgeTests() { + return new Object[][]{ + new Object[]{ + Arrays.asList(), + Arrays.asList(), + Arrays.asList() + }, + new Object[]{ + Arrays.asList(new Edge(datasetOneUrn, datasetTwoUrn, downstreamOf)), + Arrays.asList(downstreamOfDatasetTwoRelatedEntity), + Arrays.asList(downstreamOfDatasetOneRelatedEntity) + }, + new Object[]{ + Arrays.asList( + new Edge(datasetOneUrn, datasetTwoUrn, downstreamOf), + new Edge(datasetTwoUrn, datasetThreeUrn, downstreamOf) + ), + Arrays.asList(downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity), + Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity) + }, + new Object[]{ + Arrays.asList( + new Edge(datasetOneUrn, datasetTwoUrn, downstreamOf), + new Edge(datasetOneUrn, userOneUrn, hasOwner), + new Edge(datasetTwoUrn, userTwoUrn, hasOwner), + new Edge(userOneUrn, userTwoUrn, knowsUser) + ), + Arrays.asList( + downstreamOfDatasetTwoRelatedEntity, + hasOwnerUserOneRelatedEntity, hasOwnerUserTwoRelatedEntity, + knowsUserTwoRelatedEntity + ), + Arrays.asList( + downstreamOfDatasetOneRelatedEntity, + hasOwnerDatasetOneRelatedEntity, + hasOwnerDatasetTwoRelatedEntity, + knowsUserOneRelatedEntity + ) + }, + new Object[]{ + Arrays.asList( + new Edge(userOneUrn, userOneUrn, knowsUser), + new Edge(userOneUrn, userOneUrn, knowsUser), + new Edge(userOneUrn, userOneUrn, knowsUser) + ), + Arrays.asList(knowsUserOneRelatedEntity), + Arrays.asList(knowsUserOneRelatedEntity) + } + }; + } + + @Test(dataProvider = "AddEdgeTests") + public void testAddEdge(List edges, List expectedOutgoing, List expectedIncoming) throws Exception { + GraphService service = getGraphService(); + + edges.forEach(service::addEdge); + syncAfterWrite(); + + RelatedEntitiesResult relatedOutgoing = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), + outgoingRelationships, + 0, 100 + ); + assertEqualsAnyOrder(relatedOutgoing, expectedOutgoing); + + RelatedEntitiesResult relatedIncoming = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), + incomingRelationships, + 0, 100 + ); + assertEqualsAnyOrder(relatedIncoming, expectedIncoming); } @Test - public void testAddEdgeReverse() throws Exception { - GraphService client = getGraphService(); + public void testPopulatedGraphService() throws Exception { + GraphService service = getPopulatedGraphService(); - Edge edge1 = new Edge( - Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"), - Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"), - "DownstreamOf"); + RelatedEntitiesResult relatedOutgoingEntitiesBeforeRemove = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100); + assertEqualsAnyOrder( + relatedOutgoingEntitiesBeforeRemove, + Arrays.asList( + downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, + hasOwnerUserOneRelatedEntity, hasOwnerUserTwoRelatedEntity, + knowsUserOneRelatedEntity, knowsUserTwoRelatedEntity + ) + ); + RelatedEntitiesResult relatedIncomingEntitiesBeforeRemove = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), incomingRelationships, + 0, 100); + assertEqualsAnyOrder( + relatedIncomingEntitiesBeforeRemove, + Arrays.asList( + downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity, + hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, hasOwnerDatasetThreeRelatedEntity, hasOwnerDatasetFourRelatedEntity, + knowsUserOneRelatedEntity, knowsUserTwoRelatedEntity + ) + ); + } - client.addEdge(edge1); - syncAfterWrite(); + @DataProvider(name = "FindRelatedEntitiesSourceEntityFilterTests") + public Object[][] getFindRelatedEntitiesSourceEntityFilterTests() { + return new Object[][] { + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity) + }, + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) + }, + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) + }, - List edgeTypes = new ArrayList<>(); - edgeTypes.add("DownstreamOf"); - RelationshipFilter relationshipFilter = new RelationshipFilter(); - relationshipFilter.setDirection(RelationshipDirection.INCOMING); - relationshipFilter.setCriteria(EMPTY_FILTER.getCriteria()); + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(hasOwner), + outgoingRelationships, + Arrays.asList(hasOwnerUserOneRelatedEntity) + }, + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(hasOwner), + incomingRelationships, + Arrays.asList() + }, + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(hasOwner), + undirectedRelationships, + Arrays.asList(hasOwnerUserOneRelatedEntity) + }, - List relatedUrns = client.findRelatedEntities( - "", - newFilter("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"), - "", - EMPTY_FILTER, - edgeTypes, - relationshipFilter, - 0, - 10) - .getEntities() - .stream() - .map(RelatedEntity::getUrn) - .collect(Collectors.toList()); + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(hasOwner), + outgoingRelationships, + Arrays.asList() + }, + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(hasOwner), + incomingRelationships, + Arrays.asList(hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity) + }, + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(hasOwner), + undirectedRelationships, + Arrays.asList(hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity) + } + }; + } - assertEquals(relatedUrns.size(), 1); + @Test(dataProvider = "FindRelatedEntitiesSourceEntityFilterTests") + public void testFindRelatedEntitiesSourceEntityFilter(Filter sourceEntityFilter, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + doTestFindRelatedEntities( + sourceEntityFilter, + EMPTY_FILTER, + relationshipTypes, + relationships, + expectedRelatedEntities + ); + } + + @DataProvider(name = "FindRelatedEntitiesDestinationEntityFilterTests") + public Object[][] getFindRelatedEntitiesDestinationEntityFilterTests() { + return new Object[][] { + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList(downstreamOfDatasetTwoRelatedEntity) + }, + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList(downstreamOfDatasetTwoRelatedEntity) + }, + new Object[] { + newFilter("urn", datasetTwoUrnString), + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList(downstreamOfDatasetTwoRelatedEntity) + }, + + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList() + }, + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList() + }, + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList() + }, + + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(hasOwner), + outgoingRelationships, + Arrays.asList(hasOwnerUserOneRelatedEntity) + }, + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(hasOwner), + incomingRelationships, + Arrays.asList() + }, + new Object[] { + newFilter("urn", userOneUrnString), + Arrays.asList(hasOwner), + undirectedRelationships, + Arrays.asList(hasOwnerUserOneRelatedEntity) + } + }; + } + + @Test(dataProvider = "FindRelatedEntitiesDestinationEntityFilterTests") + public void testFindRelatedEntitiesDestinationEntityFilter(Filter destinationEntityFilter, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + doTestFindRelatedEntities( + EMPTY_FILTER, + destinationEntityFilter, + relationshipTypes, + relationships, + expectedRelatedEntities + ); + } + + private void doTestFindRelatedEntities( + final Filter sourceEntityFilter, + final Filter destinationEntityFilter, + List relationshipTypes, + final RelationshipFilter relationshipFilter, + List expectedRelatedEntities + ) throws Exception { + GraphService service = getPopulatedGraphService(); + + RelatedEntitiesResult relatedEntities = service.findRelatedEntities( + anyType, sourceEntityFilter, + anyType, destinationEntityFilter, + relationshipTypes, relationshipFilter, + 0, 10 + ); + + assertEqualsAnyOrder(relatedEntities, expectedRelatedEntities); + } + + @DataProvider(name = "FindRelatedEntitiesSourceTypeTests") + public Object[][] getFindRelatedEntitiesSourceTypeTests() { + return new Object[][]{ + new Object[] { + null, + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity) + }, + new Object[] { + null, + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList(downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) + }, + new Object[] { + null, + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList( + downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, + downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity + ) + }, + + // "" used to be any type before v0.9.0, which is now encoded by null + new Object[] { + "", + Arrays.asList(downstreamOf), + outgoingRelationships, + Collections.emptyList() + }, + new Object[] { + "", + Arrays.asList(downstreamOf), + incomingRelationships, + Collections.emptyList() + }, + new Object[] { + "", + Arrays.asList(downstreamOf), + undirectedRelationships, + Collections.emptyList() + }, + + new Object[]{ + datasetType, + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity) + }, + new Object[]{ + datasetType, + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList(downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) + }, + new Object[]{ + datasetType, + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList( + downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, + downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity + ) + }, + + new Object[]{ + userType, + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList() + }, + new Object[]{ + userType, + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList() + }, + new Object[]{ + userType, + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList() + }, + + new Object[]{ + userType, + Arrays.asList(hasOwner), + outgoingRelationships, + Arrays.asList() + }, + new Object[]{ + userType, + Arrays.asList(hasOwner), + incomingRelationships, + Arrays.asList( + hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, + hasOwnerDatasetThreeRelatedEntity, hasOwnerDatasetFourRelatedEntity + ) + }, + new Object[]{ + userType, + Arrays.asList(hasOwner), + undirectedRelationships, + Arrays.asList( + hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, + hasOwnerDatasetThreeRelatedEntity, hasOwnerDatasetFourRelatedEntity + ) + } + }; + } + + @Test(dataProvider = "FindRelatedEntitiesSourceTypeTests") + public void testFindRelatedEntitiesSourceType(String datasetType, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + doTestFindRelatedEntities( + datasetType, + anyType, + relationshipTypes, + relationships, + expectedRelatedEntities + ); + } + + @DataProvider(name = "FindRelatedEntitiesDestinationTypeTests") + public Object[][] getFindRelatedEntitiesDestinationTypeTests() { + return new Object[][] { + new Object[] { + null, + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity) + }, + new Object[] { + null, + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList(downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) + }, + new Object[] { + null, + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList( + downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, + downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity + ) + }, + + new Object[] { + "", + Arrays.asList(downstreamOf), + outgoingRelationships, + Collections.emptyList() + }, + new Object[] { + "", + Arrays.asList(downstreamOf), + incomingRelationships, + Collections.emptyList() + }, + new Object[] { + "", + Arrays.asList(downstreamOf), + undirectedRelationships, + Collections.emptyList() + }, + + new Object[] { + datasetType, + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity) + }, + new Object[] { + datasetType, + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList(downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) + }, + new Object[] { + datasetType, + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList( + downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, + downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity + ) + }, + + new Object[] { + datasetType, + Arrays.asList(hasOwner), + outgoingRelationships, + Arrays.asList() + }, + new Object[] { + datasetType, + Arrays.asList(hasOwner), + incomingRelationships, + Arrays.asList( + hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, + hasOwnerDatasetThreeRelatedEntity, hasOwnerDatasetFourRelatedEntity + ) + }, + new Object[] { + datasetType, + Arrays.asList(hasOwner), + undirectedRelationships, + Arrays.asList( + hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, + hasOwnerDatasetThreeRelatedEntity, hasOwnerDatasetFourRelatedEntity + ) + }, + + new Object[] { + userType, + Arrays.asList(hasOwner), + outgoingRelationships, + Arrays.asList(hasOwnerUserOneRelatedEntity, hasOwnerUserTwoRelatedEntity) + }, + new Object[] { + userType, + Arrays.asList(hasOwner), + incomingRelationships, + Arrays.asList() + }, + new Object[] { + userType, + Arrays.asList(hasOwner), + undirectedRelationships, + Arrays.asList(hasOwnerUserOneRelatedEntity, hasOwnerUserTwoRelatedEntity) + } + }; + } + + @Test(dataProvider = "FindRelatedEntitiesDestinationTypeTests") + public void testFindRelatedEntitiesDestinationType(String datasetType, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + doTestFindRelatedEntities( + anyType, + datasetType, + relationshipTypes, + relationships, + expectedRelatedEntities + ); + } + + private void doTestFindRelatedEntities( + final String sourceType, + final String destinationType, + final List relationshipTypes, + final RelationshipFilter relationshipFilter, + List expectedRelatedEntities + ) throws Exception { + GraphService service = getPopulatedGraphService(); + + RelatedEntitiesResult relatedEntities = service.findRelatedEntities( + sourceType, EMPTY_FILTER, + destinationType, EMPTY_FILTER, + relationshipTypes, relationshipFilter, + 0, 10 + ); + + assertEqualsAnyOrder(relatedEntities, expectedRelatedEntities); + } + + private void doTestFindRelatedEntitiesEntityType(@Nullable String sourceType, + @Nullable String destinationType, + @Nonnull String relationshipType, + @Nonnull RelationshipFilter relationshipFilter, + @Nonnull GraphService service, + @Nonnull RelatedEntity... expectedEntities) { + RelatedEntitiesResult actualEntities = service.findRelatedEntities( + sourceType, EMPTY_FILTER, + destinationType, EMPTY_FILTER, + Arrays.asList(relationshipType), relationshipFilter, + 0, 100 + ); + assertEqualsAnyOrder(actualEntities, Arrays.asList(expectedEntities)); } @Test - public void testRemoveEdgesFromNode() throws Exception { - GraphService client = getGraphService(); + public void testFindRelatedEntitiesNullSourceType() throws Exception { + GraphService service = getGraphService(); - Edge edge1 = new Edge( - Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)"), - Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"), - "DownstreamOf"); + Urn nullUrn = createFromString("urn:li:null:(urn:li:null:Null)"); + assertNotNull(nullUrn); + RelatedEntity nullRelatedEntity = new RelatedEntity(downstreamOf, nullUrn.toString()); - client.addEdge(edge1); + doTestFindRelatedEntitiesEntityType(anyType, "null", downstreamOf, outgoingRelationships, service); + doTestFindRelatedEntitiesEntityType(anyType, null, downstreamOf, outgoingRelationships, service); + + service.addEdge(new Edge(datasetTwoUrn, datasetOneUrn, downstreamOf)); syncAfterWrite(); + doTestFindRelatedEntitiesEntityType(anyType, "null", downstreamOf, outgoingRelationships, service); + doTestFindRelatedEntitiesEntityType(anyType, null, downstreamOf, outgoingRelationships, service, downstreamOfDatasetOneRelatedEntity); - List edgeTypes = new ArrayList<>(); - edgeTypes.add("DownstreamOf"); - RelationshipFilter relationshipFilter = new RelationshipFilter(); - relationshipFilter.setDirection(RelationshipDirection.INCOMING); - relationshipFilter.setCriteria(EMPTY_FILTER.getCriteria()); - - List relatedUrns = client.findRelatedEntities( - "", - newFilter("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"), - "", - EMPTY_FILTER, - edgeTypes, - relationshipFilter, - 0, - 10) - .getEntities() - .stream() - .map(RelatedEntity::getUrn) - .collect(Collectors.toList()); - - assertEquals(relatedUrns.size(), 1); - - client.removeEdgesFromNode(Urn.createFromString( - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"), - edgeTypes, - relationshipFilter); + service.addEdge(new Edge(datasetOneUrn, nullUrn, downstreamOf)); syncAfterWrite(); - - List relatedUrnsPostDelete = client.findRelatedEntities( - "", - newFilter("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"), - "", - EMPTY_FILTER, - edgeTypes, - relationshipFilter, - 0, - 10) - .getEntities() - .stream() - .map(RelatedEntity::getUrn) - .collect(Collectors.toList()); - - assertEquals(relatedUrnsPostDelete.size(), 0); + doTestFindRelatedEntitiesEntityType(anyType, "null", downstreamOf, outgoingRelationships, service, nullRelatedEntity); + doTestFindRelatedEntitiesEntityType(anyType, null, downstreamOf, outgoingRelationships, service, nullRelatedEntity, downstreamOfDatasetOneRelatedEntity); } + + @Test + public void testFindRelatedEntitiesNullDestinationType() throws Exception { + GraphService service = getGraphService(); + + Urn nullUrn = createFromString("urn:li:null:(urn:li:null:Null)"); + assertNotNull(nullUrn); + RelatedEntity nullRelatedEntity = new RelatedEntity(downstreamOf, nullUrn.toString()); + + doTestFindRelatedEntitiesEntityType(anyType, "null", downstreamOf, outgoingRelationships, service); + doTestFindRelatedEntitiesEntityType(anyType, null, downstreamOf, outgoingRelationships, service); + + service.addEdge(new Edge(datasetTwoUrn, datasetOneUrn, downstreamOf)); + syncAfterWrite(); + doTestFindRelatedEntitiesEntityType(anyType, "null", downstreamOf, outgoingRelationships, service); + doTestFindRelatedEntitiesEntityType(anyType, null, downstreamOf, outgoingRelationships, service, downstreamOfDatasetOneRelatedEntity); + + service.addEdge(new Edge(datasetOneUrn, nullUrn, downstreamOf)); + syncAfterWrite(); + doTestFindRelatedEntitiesEntityType(anyType, "null", downstreamOf, outgoingRelationships, service, nullRelatedEntity); + doTestFindRelatedEntitiesEntityType(anyType, null, downstreamOf, outgoingRelationships, service, nullRelatedEntity, downstreamOfDatasetOneRelatedEntity); + } + + @Test + public void testFindRelatedEntitiesRelationshipTypes() throws Exception { + GraphService service = getPopulatedGraphService(); + + RelatedEntitiesResult allOutgoingRelatedEntities = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100 + ); + assertEqualsAnyOrder( + allOutgoingRelatedEntities, + Arrays.asList( + downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, + hasOwnerUserOneRelatedEntity, hasOwnerUserTwoRelatedEntity, + knowsUserOneRelatedEntity, knowsUserTwoRelatedEntity + ) + ); + + RelatedEntitiesResult allIncomingRelatedEntities = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), incomingRelationships, + 0, 100 + ); + assertEqualsAnyOrder( + allIncomingRelatedEntities, + Arrays.asList( + downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity, + hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, hasOwnerDatasetThreeRelatedEntity, hasOwnerDatasetFourRelatedEntity, + knowsUserOneRelatedEntity, knowsUserTwoRelatedEntity + ) + ); + + RelatedEntitiesResult allUnknownRelationshipTypeRelatedEntities = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList("unknownRelationshipType", "unseenRelationshipType"), outgoingRelationships, + 0, 100 + ); + assertEqualsAnyOrder( + allUnknownRelationshipTypeRelatedEntities, + Collections.emptyList() + ); + + RelatedEntitiesResult someUnknownRelationshipTypeRelatedEntities = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList("unknownRelationshipType", downstreamOf), outgoingRelationships, + 0, 100 + ); + assertEqualsAnyOrder( + someUnknownRelationshipTypeRelatedEntities, + Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity) + ); + } + + @Test + public void testFindRelatedEntitiesNoRelationshipTypes() throws Exception { + GraphService service = getPopulatedGraphService(); + + RelatedEntitiesResult relatedEntities = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Collections.emptyList(), outgoingRelationships, + 0, 10 + ); + + assertEquals(relatedEntities.entities, Collections.emptyList()); + + // does the test actually test something? is the Collections.emptyList() the only reason why we did not get any related urns? + RelatedEntitiesResult relatedEntitiesAll = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 10 + ); + + assertNotEquals(relatedEntitiesAll.entities, Collections.emptyList()); + } + + @Test + public void testFindRelatedEntitiesAllFilters() throws Exception { + GraphService service = getPopulatedGraphService(); + + RelatedEntitiesResult relatedEntities = service.findRelatedEntities( + datasetType, newFilter("urn", datasetOneUrnString), + userType, newFilter("urn", userOneUrnString), + Arrays.asList(hasOwner), outgoingRelationships, + 0, 10 + ); + + assertEquals(relatedEntities.entities, Arrays.asList(hasOwnerUserOneRelatedEntity)); + + relatedEntities = service.findRelatedEntities( + datasetType, newFilter("urn", datasetOneUrnString), + userType, newFilter("urn", userTwoUrnString), + Arrays.asList(hasOwner), incomingRelationships, + 0, 10 + ); + + assertEquals(relatedEntities.entities, Collections.emptyList()); + } + + @Test + public void testFindRelatedEntitiesOffsetAndCount() throws Exception { + GraphService service = getPopulatedGraphService(); + + // populated graph asserted in testPopulatedGraphService + RelatedEntitiesResult allRelatedEntities = service.findRelatedEntities( + datasetType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100 + ); + + List individualRelatedEntities = new ArrayList<>(); + IntStream.range(0, allRelatedEntities.entities.size()) + .forEach(idx -> individualRelatedEntities.addAll( + service.findRelatedEntities( + datasetType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + idx, 1 + ).entities + )); + Assert.assertEquals(individualRelatedEntities, allRelatedEntities.entities); + } + + @DataProvider(name = "RemoveEdgesFromNodeTests") + public Object[][] getRemoveEdgesFromNodeTests() { + return new Object[][] { + new Object[] { + datasetTwoUrn, + Arrays.asList(downstreamOf), + outgoingRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity), + Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity), + Arrays.asList(), + Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) + }, + new Object[] { + datasetTwoUrn, + Arrays.asList(downstreamOf), + incomingRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity), + Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity), + Arrays.asList(downstreamOfDatasetOneRelatedEntity), + Arrays.asList(), + }, + new Object[] { + datasetTwoUrn, + Arrays.asList(downstreamOf), + undirectedRelationships, + Arrays.asList(downstreamOfDatasetOneRelatedEntity), + Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity), + Arrays.asList(), + Arrays.asList() + }, + + new Object[] { + userOneUrn, + Arrays.asList(hasOwner, knowsUser), + outgoingRelationships, + Arrays.asList(knowsUserTwoRelatedEntity), + Arrays.asList(hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, knowsUserTwoRelatedEntity), + Arrays.asList(), + Arrays.asList(hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, knowsUserTwoRelatedEntity) + }, + new Object[] { + userOneUrn, + Arrays.asList(hasOwner, knowsUser), + incomingRelationships, + Arrays.asList(knowsUserTwoRelatedEntity), + Arrays.asList(hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, knowsUserTwoRelatedEntity), + Arrays.asList(knowsUserTwoRelatedEntity), + Arrays.asList() + }, + new Object[] { + userOneUrn, + Arrays.asList(hasOwner, knowsUser), + undirectedRelationships, + Arrays.asList(knowsUserTwoRelatedEntity), + Arrays.asList(hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, knowsUserTwoRelatedEntity), + Arrays.asList(), + Arrays.asList() + } + }; + } + + @Test(dataProvider = "RemoveEdgesFromNodeTests") + public void testRemoveEdgesFromNode(@Nonnull Urn nodeToRemoveFrom, + @Nonnull List relationTypes, + @Nonnull RelationshipFilter relationshipFilter, + List expectedOutgoingRelatedUrnsBeforeRemove, + List expectedIncomingRelatedUrnsBeforeRemove, + List expectedOutgoingRelatedUrnsAfterRemove, + List expectedIncomingRelatedUrnsAfterRemove) throws Exception { + GraphService service = getPopulatedGraphService(); + + List allOtherRelationTypes = + allRelationshipTypes.stream() + .filter(relation -> !relationTypes.contains(relation)) + .collect(Collectors.toList()); + assertTrue(allOtherRelationTypes.size() > 0); + + RelatedEntitiesResult actualOutgoingRelatedUrnsBeforeRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + relationTypes, outgoingRelationships, + 0, 100); + RelatedEntitiesResult actualIncomingRelatedUrnsBeforeRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + relationTypes, incomingRelationships, + 0, 100); + assertEqualsAnyOrder(actualOutgoingRelatedUrnsBeforeRemove, expectedOutgoingRelatedUrnsBeforeRemove); + assertEqualsAnyOrder(actualIncomingRelatedUrnsBeforeRemove, expectedIncomingRelatedUrnsBeforeRemove); + + // we expect these do not change + RelatedEntitiesResult relatedEntitiesOfOtherOutgoingRelationTypesBeforeRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + allOtherRelationTypes, outgoingRelationships, + 0, 100); + RelatedEntitiesResult relatedEntitiesOfOtherIncomingRelationTypesBeforeRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + allOtherRelationTypes, incomingRelationships, + 0, 100); + + service.removeEdgesFromNode( + nodeToRemoveFrom, + relationTypes, + relationshipFilter + ); + syncAfterWrite(); + + RelatedEntitiesResult actualOutgoingRelatedUrnsAfterRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + relationTypes, outgoingRelationships, + 0, 100); + RelatedEntitiesResult actualIncomingRelatedUrnsAfterRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + relationTypes, incomingRelationships, + 0, 100); + assertEqualsAnyOrder(actualOutgoingRelatedUrnsAfterRemove, expectedOutgoingRelatedUrnsAfterRemove); + assertEqualsAnyOrder(actualIncomingRelatedUrnsAfterRemove, expectedIncomingRelatedUrnsAfterRemove); + + // assert these did not change + RelatedEntitiesResult relatedEntitiesOfOtherOutgoingRelationTypesAfterRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + allOtherRelationTypes, outgoingRelationships, + 0, 100); + RelatedEntitiesResult relatedEntitiesOfOtherIncomingRelationTypesAfterRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + allOtherRelationTypes, incomingRelationships, + 0, 100); + assertEqualsAnyOrder(relatedEntitiesOfOtherOutgoingRelationTypesAfterRemove, relatedEntitiesOfOtherOutgoingRelationTypesBeforeRemove); + assertEqualsAnyOrder(relatedEntitiesOfOtherIncomingRelationTypesAfterRemove, relatedEntitiesOfOtherIncomingRelationTypesBeforeRemove); + } + + @Test + public void testRemoveEdgesFromNodeNoRelationshipTypes() throws Exception { + GraphService service = getPopulatedGraphService(); + Urn nodeToRemoveFrom = datasetOneUrn; + + // populated graph asserted in testPopulatedGraphService + RelatedEntitiesResult relatedOutgoingEntitiesBeforeRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100); + + // can be replaced with a single removeEdgesFromNode and undirectedRelationships once supported by all implementations + service.removeEdgesFromNode( + nodeToRemoveFrom, + Collections.emptyList(), + outgoingRelationships + ); + service.removeEdgesFromNode( + nodeToRemoveFrom, + Collections.emptyList(), + incomingRelationships + ); + syncAfterWrite(); + + RelatedEntitiesResult relatedOutgoingEntitiesAfterRemove = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100); + assertEqualsAnyOrder(relatedOutgoingEntitiesAfterRemove, relatedOutgoingEntitiesBeforeRemove); + + // does the test actually test something? is the Collections.emptyList() the only reason why we did not see changes? + service.removeEdgesFromNode( + nodeToRemoveFrom, + Arrays.asList(downstreamOf, hasOwner, knowsUser), + outgoingRelationships + ); + service.removeEdgesFromNode( + nodeToRemoveFrom, + Arrays.asList(downstreamOf, hasOwner, knowsUser), + incomingRelationships + ); + syncAfterWrite(); + + RelatedEntitiesResult relatedOutgoingEntitiesAfterRemoveAll = service.findRelatedEntities( + anyType, newFilter("urn", nodeToRemoveFrom.toString()), + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100); + assertEqualsAnyOrder(relatedOutgoingEntitiesAfterRemoveAll, Collections.emptyList()); + } + + @Test + public void testRemoveEdgesFromUnknownNode() throws Exception { + GraphService service = getPopulatedGraphService(); + Urn nodeToRemoveFrom = unknownUrn; + + // populated graph asserted in testPopulatedGraphService + RelatedEntitiesResult relatedOutgoingEntitiesBeforeRemove = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100); + + // can be replaced with a single removeEdgesFromNode and undirectedRelationships once supported by all implementations + service.removeEdgesFromNode( + nodeToRemoveFrom, + Arrays.asList(downstreamOf, hasOwner, knowsUser), + outgoingRelationships + ); + service.removeEdgesFromNode( + nodeToRemoveFrom, + Arrays.asList(downstreamOf, hasOwner, knowsUser), + incomingRelationships + ); + syncAfterWrite(); + + RelatedEntitiesResult relatedOutgoingEntitiesAfterRemove = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100); + assertEqualsAnyOrder(relatedOutgoingEntitiesAfterRemove, relatedOutgoingEntitiesBeforeRemove); + } + + @Test + public void testRemoveNode() throws Exception { + GraphService service = getPopulatedGraphService(); + + service.removeNode(datasetTwoUrn); + syncAfterWrite(); + + // assert the modified graph + assertEqualsAnyOrder( + service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100 + ), + Arrays.asList( + hasOwnerUserOneRelatedEntity, hasOwnerUserTwoRelatedEntity, + knowsUserOneRelatedEntity, knowsUserTwoRelatedEntity + ) + ); + } + + @Test + public void testRemoveUnknownNode() throws Exception { + GraphService service = getPopulatedGraphService(); + + // populated graph asserted in testPopulatedGraphService + RelatedEntitiesResult entitiesBeforeRemove = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100); + + service.removeNode(unknownUrn); + syncAfterWrite(); + + RelatedEntitiesResult entitiesAfterRemove = service.findRelatedEntities( + anyType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf, hasOwner, knowsUser), outgoingRelationships, + 0, 100); + assertEqualsAnyOrder(entitiesBeforeRemove, entitiesAfterRemove); + } + + @Test + public void testClear() throws Exception { + GraphService service = getPopulatedGraphService(); + + // populated graph asserted in testPopulatedGraphService + + service.clear(); + syncAfterWrite(); + + // assert the modified graph: check all nodes related to upstreamOf and nextVersionOf edges again + assertEqualsAnyOrder( + service.findRelatedEntities( + datasetType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(downstreamOf), outgoingRelationships, + 0, 100 + ), + Collections.emptyList() + ); + assertEqualsAnyOrder( + service.findRelatedEntities( + userType, EMPTY_FILTER, + anyType, EMPTY_FILTER, + Arrays.asList(hasOwner), outgoingRelationships, + 0, 100 + ), + Collections.emptyList() + ); + assertEqualsAnyOrder( + service.findRelatedEntities( + anyType, EMPTY_FILTER, + userType, EMPTY_FILTER, + Arrays.asList(knowsUser), outgoingRelationships, + 0, 100 + ), + Collections.emptyList() + ); + } + + private List getFullyConnectedGraph(int nodes, List relationshipTypes) { + List edges = new ArrayList<>(); + + for (int sourceNode = 1; sourceNode <= nodes; sourceNode++) { + for (int destinationNode = 1; destinationNode <= nodes; destinationNode++) { + for (String relationship : relationshipTypes) { + int sourceType = sourceNode % 3; + Urn source = createFromString("urn:li:type" + sourceType + ":(urn:li:node" + sourceNode + ")"); + int destinationType = destinationNode % 3; + Urn destination = createFromString("urn:li:type" + destinationType + ":(urn:li:node" + destinationNode + ")"); + + edges.add(new Edge(source, destination, relationship)); + } + } + } + + return edges; + } + + @Test + public void testConcurrentAddEdge() throws Exception { + final GraphService service = getGraphService(); + + // too many edges may cause too many threads throwing + // java.util.concurrent.RejectedExecutionException: Thread limit exceeded replacing blocked worker + int nodes = 5; + int relationshipTypes = 5; + List allRelationships = IntStream.range(1, relationshipTypes + 1).mapToObj(id -> "relationship" + id).collect(Collectors.toList()); + List edges = getFullyConnectedGraph(nodes, allRelationships); + + List operations = edges.stream().map(edge -> new Runnable() { + @Override + public void run() { + service.addEdge(edge); + } + }).collect(Collectors.toList()); + + doTestConcurrentOp(operations); + syncAfterWrite(); + + RelatedEntitiesResult relatedEntities = service.findRelatedEntities( + null, EMPTY_FILTER, + null, EMPTY_FILTER, + allRelationships, outgoingRelationships, + 0, nodes * relationshipTypes * 2 + ); + + Set expectedRelatedEntities = edges.stream() + .map(edge -> new RelatedEntity(edge.getRelationshipType(), edge.getDestination().toString())) + .collect(Collectors.toSet()); + assertEquals(new HashSet<>(relatedEntities.entities), expectedRelatedEntities); + } + + @Test + public void testConcurrentRemoveEdgesFromNode() throws Exception { + final GraphService service = getGraphService(); + + int nodes = 10; + int relationshipTypes = 5; + List allRelationships = IntStream.range(1, relationshipTypes + 1).mapToObj(id -> "relationship" + id).collect(Collectors.toList()); + List edges = getFullyConnectedGraph(nodes, allRelationships); + + // add fully connected graph + edges.forEach(service::addEdge); + syncAfterWrite(); + + // assert the graph is there + RelatedEntitiesResult relatedEntities = service.findRelatedEntities( + null, EMPTY_FILTER, + null, EMPTY_FILTER, + allRelationships, outgoingRelationships, + 0, nodes * relationshipTypes * 2 + ); + assertEquals(relatedEntities.entities.size(), nodes * relationshipTypes); + + // delete all edges concurrently + List operations = edges.stream().map(edge -> new Runnable() { + @Override + public void run() { + service.removeEdgesFromNode(edge.getSource(), Arrays.asList(edge.getRelationshipType()), outgoingRelationships); + } + }).collect(Collectors.toList()); + doTestConcurrentOp(operations); + syncAfterWrite(); + + // assert the graph is gone + RelatedEntitiesResult relatedEntitiesAfterDeletion = service.findRelatedEntities( + null, EMPTY_FILTER, + null, EMPTY_FILTER, + allRelationships, outgoingRelationships, + 0, nodes * relationshipTypes * 2 + ); + assertEquals(relatedEntitiesAfterDeletion.entities.size(), 0); + } + + @Test + public void testConcurrentRemoveNodes() throws Exception { + final GraphService service = getGraphService(); + + // too many edges may cause too many threads throwing + // java.util.concurrent.RejectedExecutionException: Thread limit exceeded replacing blocked worker + int nodes = 10; + int relationshipTypes = 5; + List allRelationships = IntStream.range(1, relationshipTypes + 1).mapToObj(id -> "relationship" + id).collect(Collectors.toList()); + List edges = getFullyConnectedGraph(nodes, allRelationships); + + // add fully connected graph + edges.forEach(service::addEdge); + syncAfterWrite(); + + // assert the graph is there + RelatedEntitiesResult relatedEntities = service.findRelatedEntities( + null, EMPTY_FILTER, + null, EMPTY_FILTER, + allRelationships, outgoingRelationships, + 0, nodes * relationshipTypes * 2 + ); + assertEquals(relatedEntities.entities.size(), nodes * relationshipTypes); + + // remove all nodes concurrently + // nodes will be removed multiple times + List operations = edges.stream().map(edge -> new Runnable() { + @Override + public void run() { + service.removeNode(edge.getSource()); + } + }).collect(Collectors.toList()); + doTestConcurrentOp(operations); + syncAfterWrite(); + + // assert the graph is gone + RelatedEntitiesResult relatedEntitiesAfterDeletion = service.findRelatedEntities( + null, EMPTY_FILTER, + null, EMPTY_FILTER, + allRelationships, outgoingRelationships, + 0, nodes * relationshipTypes * 2 + ); + assertEquals(relatedEntitiesAfterDeletion.entities.size(), 0); + } + + private void doTestConcurrentOp(List operations) throws Exception { + final Queue throwables = new ConcurrentLinkedQueue<>(); + final CountDownLatch started = new CountDownLatch(operations.size()); + final CountDownLatch finished = new CountDownLatch(operations.size()); + operations.forEach(operation -> new Thread(new Runnable() { + @Override + public void run() { + try { + started.countDown(); + + try { + if (!started.await(10, TimeUnit.SECONDS)) { + fail("Timed out waiting for all threads to start"); + } + } catch (InterruptedException e) { + fail("Got interrupted waiting for all threads to start"); + } + + operation.run(); + finished.countDown(); + } catch (Throwable t) { + t.printStackTrace(); + throwables.add(t); + } + } + }).start()); + + assertTrue(finished.await(10, TimeUnit.SECONDS)); + assertEquals(throwables.size(), 0); + } + } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/Neo4jGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/Neo4jGraphServiceTest.java index c4c4152c8a..0b9e779ef5 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/Neo4jGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/Neo4jGraphServiceTest.java @@ -1,11 +1,19 @@ package com.linkedin.metadata.graph; +import com.linkedin.metadata.query.RelationshipFilter; import org.neo4j.driver.Driver; import org.neo4j.driver.GraphDatabase; +import org.testng.SkipException; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; import javax.annotation.Nonnull; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; + +import static org.testng.Assert.assertEquals; public class Neo4jGraphServiceTest extends GraphServiceTestBase { @@ -28,11 +36,115 @@ public class Neo4jGraphServiceTest extends GraphServiceTestBase { } @Override - protected @Nonnull GraphService getGraphService() { + protected @Nonnull + GraphService getGraphService() { return _client; } @Override - protected void syncAfterWrite() { } + protected void syncAfterWrite() { + } + + @Override + protected void assertEqualsAnyOrder(RelatedEntitiesResult actual, RelatedEntitiesResult expected) { + // https://github.com/linkedin/datahub/issues/3118 + // Neo4jGraphService produces duplicates, which is here ignored until fixed + // actual.count and actual.total not tested due to duplicates + assertEquals(actual.start, expected.start); + assertEqualsAnyOrder(actual.entities, expected.entities, RELATED_ENTITY_COMPARATOR); + } + + @Override + protected void assertEqualsAnyOrder(List actual, List expected, Comparator comparator) { + // https://github.com/linkedin/datahub/issues/3118 + // Neo4jGraphService produces duplicates, which is here ignored until fixed + assertEquals( + new HashSet<>(actual), + new HashSet<>(expected) + ); + } + + @Override + public void testFindRelatedEntitiesSourceType(String datasetType, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + if (datasetType != null && datasetType.isEmpty()) { + // https://github.com/linkedin/datahub/issues/3119 + throw new SkipException("Neo4jGraphService does not support empty source type"); + } + if (datasetType != null && datasetType.equals(GraphServiceTestBase.userType)) { + // https://github.com/linkedin/datahub/issues/3123 + // only test cases with "user" type fail due to this bug + throw new SkipException("Neo4jGraphService does not apply source / destination types"); + } + super.testFindRelatedEntitiesSourceType(datasetType, relationshipTypes, relationships, expectedRelatedEntities); + } + + @Override + public void testFindRelatedEntitiesDestinationType(String datasetType, + List relationshipTypes, + RelationshipFilter relationships, + List expectedRelatedEntities) throws Exception { + if (datasetType != null && datasetType.isEmpty()) { + // https://github.com/linkedin/datahub/issues/3119 + throw new SkipException("Neo4jGraphService does not support empty destination type"); + } + if (relationshipTypes.contains(hasOwner)) { + // https://github.com/linkedin/datahub/issues/3123 + // only test cases with "HasOwner" relatioship fail due to this bug + throw new SkipException("Neo4jGraphService does not apply source / destination types"); + } + super.testFindRelatedEntitiesDestinationType(datasetType, relationshipTypes, relationships, expectedRelatedEntities); + } + + @Test + @Override + public void testFindRelatedEntitiesNullSourceType() throws Exception { + // https://github.com/linkedin/datahub/issues/3121 + throw new SkipException("Neo4jGraphService does not support 'null' entity type string"); + } + + @Test + @Override + public void testFindRelatedEntitiesNullDestinationType() throws Exception { + // https://github.com/linkedin/datahub/issues/3121 + throw new SkipException("Neo4jGraphService does not support 'null' entity type string"); + } + + @Test + @Override + public void testFindRelatedEntitiesNoRelationshipTypes() { + // https://github.com/linkedin/datahub/issues/3120 + throw new SkipException("Neo4jGraphService does not support empty list of relationship types"); + } + + @Test + @Override + public void testRemoveEdgesFromNodeNoRelationshipTypes() { + // https://github.com/linkedin/datahub/issues/3120 + throw new SkipException("Neo4jGraphService does not support empty list of relationship types"); + } + + @Test + @Override + public void testConcurrentAddEdge() { + // https://github.com/linkedin/datahub/issues/3141 + throw new SkipException("Neo4jGraphService does not manage to add all edges added concurrently"); + } + + @Test + @Override + public void testConcurrentRemoveEdgesFromNode() { + // https://github.com/linkedin/datahub/issues/3118 + throw new SkipException("Neo4jGraphService produces duplicates"); + } + + @Test + @Override + public void testConcurrentRemoveNodes() { + // https://github.com/linkedin/datahub/issues/3118 + throw new SkipException("Neo4jGraphService produces duplicates"); + } }