From 4fc9db8847a5c2c5bb7cb3d8bac98eafd29ea1c9 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 11 Jan 2022 13:45:39 -0800 Subject: [PATCH] Issue2147 - Add support for deleting edge in lineage (#2164) * Restore deleted entity on PATCH requests * Add support for deleting edge in lineage --- .idea/runConfigurations.xml | 10 --- .../catalog/jdbi3/CollectionDAO.java | 10 +-- .../catalog/jdbi3/LineageRepository.java | 19 +++++ .../resources/lineage/LineageResource.java | 46 ++++++++++-- .../json/schema/api/lineage/addLineage.json | 2 +- .../lineage/LineageResourceTest.java | 72 ++++++++++++++++++- 6 files changed, 137 insertions(+), 22 deletions(-) delete mode 100644 .idea/runConfigurations.xml diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml deleted file mode 100644 index 797acea53eb..00000000000 --- a/.idea/runConfigurations.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index d4d03027f3a..7950dec109f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -408,7 +408,7 @@ public interface CollectionDAO { "DELETE from entity_relationship WHERE fromId = :fromId " + "AND fromEntity = :fromEntity AND toId = :toId AND toEntity = :toEntity " + "AND relation = :relation") - void delete( + int delete( @Bind("fromId") String fromId, @Bind("fromEntity") String fromEntity, @Bind("toId") String toId, @@ -419,7 +419,7 @@ public interface CollectionDAO { @SqlUpdate( "DELETE from entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " + "AND relation = :relation AND toEntity = :toEntity") - void deleteFrom( + int deleteFrom( @Bind("fromId") String fromId, @Bind("fromEntity") String fromEntity, @Bind("relation") int relation, @@ -429,14 +429,14 @@ public interface CollectionDAO { @SqlUpdate( "DELETE from entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " + "AND relation = :relation") - void deleteFrom( + int deleteFrom( @Bind("fromId") String fromId, @Bind("fromEntity") String fromEntity, @Bind("relation") int relation); // Delete all the entity relationship toId <-- relation -- entity of type fromEntity @SqlUpdate( "DELETE from entity_relationship WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation " + "AND fromEntity = :fromEntity") - void deleteTo( + int deleteTo( @Bind("toId") String toId, @Bind("toEntity") String toEntity, @Bind("relation") int relation, @@ -445,7 +445,7 @@ public interface CollectionDAO { @SqlUpdate( "DELETE from entity_relationship WHERE (toId = :id AND toEntity = :entity) OR " + "(fromId = :id AND toEntity = :entity)") - void deleteAll(@Bind("id") String id, @Bind("entity") String entity); + int deleteAll(@Bind("id") String id, @Bind("entity") String entity); @SqlUpdate( "UPDATE entity_relationship SET deleted = true WHERE (toId = :id AND toEntity = :entity) " diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java index 9e28fe65d31..3a5638a8f0d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LineageRepository.java @@ -65,6 +65,25 @@ public class LineageRepository { Relationship.UPSTREAM.ordinal()); } + @Transaction + public boolean deleteLineage(String fromEntity, String fromId, String toEntity, String toId) throws IOException { + // Validate from entity + EntityReference from = Entity.getEntityReference(fromEntity, UUID.fromString(fromId)); + + // Validate to entity + EntityReference to = Entity.getEntityReference(toEntity, UUID.fromString(toId)); + + // Finally, delete lineage relationship + return dao.relationshipDAO() + .delete( + from.getId().toString(), + from.getType(), + to.getId().toString(), + to.getType(), + Relationship.UPSTREAM.ordinal()) + > 0; + } + private EntityLineage getLineage(EntityReference primary, int upstreamDepth, int downstreamDepth) throws IOException { List entities = new ArrayList<>(); EntityLineage lineage = diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/lineage/LineageResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/lineage/LineageResource.java index 8424837c85b..aa3e0057efc 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/lineage/LineageResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/lineage/LineageResource.java @@ -13,6 +13,9 @@ package org.openmetadata.catalog.resources.lineage; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; + +import io.dropwizard.jersey.errors.ErrorMessage; import io.swagger.annotations.Api; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -25,6 +28,7 @@ import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.PUT; @@ -42,11 +46,8 @@ import org.openmetadata.catalog.api.lineage.AddLineage; import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.jdbi3.LineageRepository; import org.openmetadata.catalog.resources.Collection; -import org.openmetadata.catalog.resources.teams.UserResource; import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.type.EntityLineage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Path("/v1/lineage") @Api(value = "Lineage resource", tags = "Lineage resource") @@ -54,7 +55,6 @@ import org.slf4j.LoggerFactory; @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "lineage") public class LineageResource { - private static final Logger LOG = LoggerFactory.getLogger(UserResource.class); private final LineageRepository dao; public LineageResource(CollectionDAO dao, Authorizer authorizer) { @@ -154,6 +154,44 @@ public class LineageResource { return Response.status(Status.OK).build(); } + @DELETE + @Path("/{fromEntity}/{fromId}/{toEntity}/{toId}") + @Operation( + summary = "Delete a lineage edge", + tags = "lineage", + description = "Delete a lineage edge with from entity as upstream node and to entity as downstream node.", + responses = { + @ApiResponse(responseCode = "200"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deleteLineage( + @Context UriInfo uriInfo, + @Parameter( + description = "Entity type of upstream entity of the edge", + required = true, + schema = @Schema(type = "string", example = "table, report, metrics, or dashboard")) + @PathParam("fromEntity") + String fromEntity, + @Parameter(description = "Entity id", required = true, schema = @Schema(type = "string")) @PathParam("fromId") + String fromId, + @Parameter( + description = "Entity type for downstream entity of the edge", + required = true, + schema = @Schema(type = "string", example = "table, report, metrics, or dashboard")) + @PathParam("toEntity") + String toEntity, + @Parameter(description = "Entity id", required = true, schema = @Schema(type = "string")) @PathParam("toId") + String toId) + throws IOException { + boolean deleted = dao.deleteLineage(fromEntity, fromId, toEntity, toId); + if (!deleted) { + return Response.status(NOT_FOUND) + .entity(new ErrorMessage(NOT_FOUND.getStatusCode(), "Lineage edge not " + "found")) + .build(); + } + return Response.status(Status.OK).build(); + } + private EntityLineage addHref(UriInfo uriInfo, EntityLineage lineage) { Entity.withHref(uriInfo, lineage.getEntity()); Entity.withHref(uriInfo, lineage.getNodes()); diff --git a/catalog-rest-service/src/main/resources/json/schema/api/lineage/addLineage.json b/catalog-rest-service/src/main/resources/json/schema/api/lineage/addLineage.json index ea598f32941..337bccae197 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/lineage/addLineage.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/lineage/addLineage.json @@ -2,7 +2,7 @@ "$id": "https://open-metadata.org/schema/api/lineage/addLineage.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "addLineage", - "description": "Add lineage details between two entities", + "description": "Add lineage edge between two entities", "type": "object", "properties" : { "description": { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/lineage/LineageResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/lineage/LineageResourceTest.java index 68ffb4b5456..6e306bb971c 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/lineage/LineageResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/lineage/LineageResourceTest.java @@ -14,6 +14,7 @@ package org.openmetadata.catalog.resources.lineage; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders; @@ -62,7 +63,7 @@ public class LineageResourceTest extends CatalogApplicationTest { } @Test - void put_addLineageForInvalidEntities() throws HttpResponseException { + void put_delete_lineage_200() throws HttpResponseException { // Add lineage table4-->table5 addEdge(TABLES.get(4), TABLES.get(5)); @@ -118,6 +119,24 @@ public class LineageResourceTest extends CatalogApplicationTest { lineage = getLineage(Entity.TABLE, TABLES.get(4).getId(), 2, 2, adminAuthHeaders()); assertEdges( lineage, Arrays.copyOfRange(expectedUpstreamEdges, 0, 4), Arrays.copyOfRange(expectedDownstreamEdges, 0, 4)); + + // + // Delete all the lineage edges + // table2--> -->table9 + // table0-->table3-->table4-->table5->table6->table7 + // table1--> -->table8 + deleteEdge(TABLES.get(0), TABLES.get(3)); + deleteEdge(TABLES.get(3), TABLES.get(4)); + deleteEdge(TABLES.get(2), TABLES.get(4)); + deleteEdge(TABLES.get(1), TABLES.get(4)); + deleteEdge(TABLES.get(4), TABLES.get(9)); + deleteEdge(TABLES.get(4), TABLES.get(5)); + deleteEdge(TABLES.get(4), TABLES.get(8)); + deleteEdge(TABLES.get(5), TABLES.get(6)); + deleteEdge(TABLES.get(6), TABLES.get(7)); + lineage = getLineage(Entity.TABLE, TABLES.get(4).getId(), 2, 2, adminAuthHeaders()); + assertTrue(lineage.getUpstreamEdges().isEmpty()); + assertTrue(lineage.getDownstreamEdges().isEmpty()); } public Edge getEdge(Table from, Table to) { @@ -137,14 +156,40 @@ public class LineageResourceTest extends CatalogApplicationTest { addLineageAndCheck(addLineage, adminAuthHeaders()); } + public void deleteEdge(Table from, Table to) throws HttpResponseException { + EntitiesEdge edge = + new EntitiesEdge() + .withFromEntity(new TableEntityInterface(from).getEntityReference()) + .withToEntity(new TableEntityInterface(to).getEntityReference()); + deleteLineageAndCheck(edge, adminAuthHeaders()); + } + public static void addLineageAndCheck(AddLineage addLineage, Map authHeaders) throws HttpResponseException { addLineage(addLineage, authHeaders); validateLineage(addLineage, authHeaders); } + public static void deleteLineageAndCheck(EntitiesEdge deleteEdge, Map authHeaders) + throws HttpResponseException { + deleteLineage(deleteEdge, authHeaders); + validateLineageDeleted(deleteEdge, authHeaders); + } + public static void addLineage(AddLineage addLineage, Map authHeaders) throws HttpResponseException { - TestUtils.put(CatalogApplicationTest.getResource("lineage"), addLineage, Status.OK, authHeaders); + TestUtils.put(getResource("lineage"), addLineage, Status.OK, authHeaders); + } + + public static void deleteLineage(EntitiesEdge edge, Map authHeaders) throws HttpResponseException { + WebTarget target = + getResource( + String.format( + "lineage/%s/%s/%s/%s", + edge.getFromEntity().getType(), + edge.getFromEntity().getId(), + edge.getToEntity().getType(), + edge.getToEntity().getId())); + TestUtils.delete(target, authHeaders); } private static void validateLineage(AddLineage addLineage, Map authHeaders) @@ -162,6 +207,21 @@ public class LineageResourceTest extends CatalogApplicationTest { assertEdge(lineage, expectedEdge, false); } + private static void validateLineageDeleted(EntitiesEdge deletedEdge, Map authHeaders) + throws HttpResponseException { + EntityReference from = deletedEdge.getFromEntity(); + EntityReference to = deletedEdge.getToEntity(); + Edge expectedEdge = getEdge(from.getId(), to.getId()); + + // Check fromEntity ---> toEntity downstream edge is returned + EntityLineage lineage = getLineage(from.getType(), from.getId(), 0, 1, authHeaders); + assertDeleted(lineage, expectedEdge, true); + + // Check fromEntity ---> toEntity upstream edge is returned + lineage = getLineage(to.getType(), to.getId(), 1, 0, authHeaders); + assertDeleted(lineage, expectedEdge, false); + } + private static void validateLineage(EntityLineage lineage) { TestUtils.validateEntityReference(lineage.getEntity()); lineage.getNodes().forEach(TestUtils::validateEntityReference); @@ -217,6 +277,14 @@ public class LineageResourceTest extends CatalogApplicationTest { } } + public static void assertDeleted(EntityLineage lineage, Edge expectedEdge, boolean downstream) { + if (downstream) { + assertFalse(lineage.getDownstreamEdges().contains(expectedEdge)); + } else { + assertFalse(lineage.getUpstreamEdges().contains(expectedEdge)); + } + } + public static void assertEdges(EntityLineage lineage, Edge[] expectedUpstreamEdges, Edge[] expectedDownstreamEdges) { assertEquals(lineage.getUpstreamEdges().size(), expectedUpstreamEdges.length); for (Edge expectedUpstreamEdge : expectedUpstreamEdges) {