diff --git a/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java b/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java index 2d2e0fa4fc9..c4e5c691a9a 100644 --- a/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java +++ b/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java @@ -17,6 +17,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.io.IOException; +import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -207,4 +208,24 @@ public final class CommonUtil { } return null; } + + public static boolean findChildren(List list, String methodName, String fqn) { + if (list == null || list.isEmpty()) return false; + try { + Method getChildren = list.get(0).getClass().getMethod(methodName); + Method getFQN = list.get(0).getClass().getMethod("getFullyQualifiedName"); + return list.stream() + .anyMatch( + o -> { + try { + return getFQN.invoke(o).equals(fqn) + || findChildren((List) getChildren.invoke(o), methodName, fqn); + } catch (Exception e) { + return false; + } + }); + } catch (Exception e) { + return false; + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java index e3994394ffb..ed8d2783182 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java @@ -158,6 +158,10 @@ public final class CatalogExceptionMessage { return String.format("Invalid %s name %s", fieldType, fieldName); } + public static String invalidFieldFQN(String fqn) { + return String.format("Invalid fully qualified field name %s", fqn); + } + public static String entityVersionNotFound(String entityType, UUID id, Double version) { return String.format("%s instance for %s and version %s not found", entityType, id, version); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java index b1654d6b47f..e57f30f73bb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java @@ -1,5 +1,6 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.common.utils.CommonUtil.findChildren; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; @@ -9,6 +10,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.Field; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.util.FullyQualifiedName; @@ -56,18 +58,21 @@ public final class ColumnUtil { // Validate if a given column exists in the table public static void validateColumnFQN(List columns, String columnFQN) { - boolean validColumn = false; - for (Column column : columns) { - if (column.getFullyQualifiedName().equals(columnFQN)) { - validColumn = true; - break; - } - } - if (!validColumn) { + boolean exists = findChildren(columns, "getChildren", columnFQN); + if (!exists) { throw new IllegalArgumentException(CatalogExceptionMessage.invalidColumnFQN(columnFQN)); } } + // validate if a given field exists in the topic + public static void validateFieldFQN(List fields, String fieldFQN) { + boolean exists = findChildren(fields, "getChildren", fieldFQN); + if (!exists) { + throw new IllegalArgumentException( + CatalogExceptionMessage.invalidFieldName("field", fieldFQN)); + } + } + public static Set getAllTags(Column column) { Set tags = new HashSet<>(); if (!listOrEmpty(column.getTags()).isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index dbf68dd939b..09defe6a272 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -13,6 +13,12 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.service.Entity.CONTAINER; +import static org.openmetadata.service.Entity.DASHBOARD; +import static org.openmetadata.service.Entity.DASHBOARD_DATA_MODEL; +import static org.openmetadata.service.Entity.MLMODEL; +import static org.openmetadata.service.Entity.TABLE; +import static org.openmetadata.service.Entity.TOPIC; import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT; @@ -21,14 +27,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.common.utils.CommonUtil; -import org.openmetadata.schema.ColumnsEntityInterface; import org.openmetadata.schema.api.lineage.AddLineage; +import org.openmetadata.schema.entity.data.Container; +import org.openmetadata.schema.entity.data.Dashboard; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.entity.data.MlModel; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.type.ColumnLineage; import org.openmetadata.schema.type.Edge; import org.openmetadata.schema.type.EntityLineage; @@ -37,17 +46,17 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.LineageDetails; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.models.IndexMapping; -import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @Repository public class LineageRepository { private final CollectionDAO dao; - public SearchClient searchClient = Entity.getSearchRepository().getSearchClient(); + private static final SearchClient searchClient = Entity.getSearchRepository().getSearchClient(); public LineageRepository() { this.dao = Entity.getCollectionDAO(); @@ -173,41 +182,86 @@ public class LineageRepository { if (details == null) { return null; } - List columnsLineage = details.getColumnsLineage(); if (columnsLineage != null && !columnsLineage.isEmpty()) { - if (areValidEntities(from, to)) { - throw new IllegalArgumentException( - "Column level lineage is only allowed between two tables or from table to dashboard."); - } - Table fromTable = dao.tableDAO().findEntityById(from.getId()); - ColumnsEntityInterface toTable = getToEntity(to); for (ColumnLineage columnLineage : columnsLineage) { for (String fromColumn : columnLineage.getFromColumns()) { - // From column belongs to the fromNode - if (fromColumn.startsWith(fromTable.getFullyQualifiedName())) { - ColumnUtil.validateColumnFQN(fromTable.getColumns(), fromColumn); - } else { - Table otherTable = - dao.tableDAO().findEntityByName(FullyQualifiedName.getTableFQN(fromColumn)); - ColumnUtil.validateColumnFQN(otherTable.getColumns(), fromColumn); - } + validateChildren(fromColumn, from); } - ColumnUtil.validateColumnFQN(toTable.getColumns(), columnLineage.getToColumn()); + validateChildren(columnLineage.getToColumn(), to); } } return JsonUtils.pojoToJson(details); } - private ColumnsEntityInterface getToEntity(EntityReference from) { - return from.getType().equals(Entity.TABLE) - ? dao.tableDAO().findEntityById(from.getId()) - : dao.dashboardDataModelDAO().findEntityById(from.getId()); + private void validateChildren(String columnFQN, EntityReference entityReference) { + switch (entityReference.getType()) { + case TABLE -> { + Table table = + Entity.getEntity(TABLE, entityReference.getId(), "columns", Include.NON_DELETED); + ColumnUtil.validateColumnFQN(table.getColumns(), columnFQN); + } + case TOPIC -> { + Topic topic = + Entity.getEntity(TOPIC, entityReference.getId(), "messageSchema", Include.NON_DELETED); + ColumnUtil.validateFieldFQN(topic.getMessageSchema().getSchemaFields(), columnFQN); + } + case CONTAINER -> { + Container container = + Entity.getEntity(CONTAINER, entityReference.getId(), "dataModel", Include.NON_DELETED); + ColumnUtil.validateColumnFQN(container.getDataModel().getColumns(), columnFQN); + } + case DASHBOARD_DATA_MODEL -> { + DashboardDataModel dashboardDataModel = + Entity.getEntity( + DASHBOARD_DATA_MODEL, entityReference.getId(), "columns", Include.NON_DELETED); + ColumnUtil.validateColumnFQN(dashboardDataModel.getColumns(), columnFQN); + } + case DASHBOARD -> { + Dashboard dashboard = + Entity.getEntity(DASHBOARD, entityReference.getId(), "charts", Include.NON_DELETED); + dashboard.getCharts().stream() + .filter(c -> c.getFullyQualifiedName().equals(columnFQN)) + .findAny() + .orElseThrow( + () -> + new IllegalArgumentException( + CatalogExceptionMessage.invalidFieldName("chart", columnFQN))); + } + case MLMODEL -> { + MlModel mlModel = + Entity.getEntity(MLMODEL, entityReference.getId(), "", Include.NON_DELETED); + mlModel.getMlFeatures().stream() + .filter(f -> f.getFullyQualifiedName().equals(columnFQN)) + .findAny() + .orElseThrow( + () -> + new IllegalArgumentException( + CatalogExceptionMessage.invalidFieldName("feature", columnFQN))); + } + default -> throw new IllegalArgumentException( + String.format("Unsupported Entity Type %s for lineage", entityReference.getType())); + } } - private boolean areValidEntities(EntityReference from, EntityReference to) { - return !from.getType().equals(Entity.TABLE) - || !(to.getType().equals(Entity.TABLE) || to.getType().equals(Entity.DASHBOARD_DATA_MODEL)); + @Transaction + public boolean deleteLineageByFQN( + String fromEntity, String fromFQN, String toEntity, String toFQN) { + EntityReference from = + Entity.getEntityReferenceByName(fromEntity, fromFQN, Include.NON_DELETED); + EntityReference to = Entity.getEntityReferenceByName(toEntity, toFQN, Include.NON_DELETED); + // Finally, delete lineage relationship + boolean result = + dao.relationshipDAO() + .delete( + from.getId(), + from.getType(), + to.getId(), + to.getType(), + Relationship.UPSTREAM.ordinal()) + > 0; + deleteLineageFromSearch(from, to); + return result; } @Transaction @@ -260,7 +314,7 @@ public class LineageRepository { getDownstreamLineage(primary.getId(), primary.getType(), lineage, downstreamDepth); // Remove duplicate nodes - lineage.withNodes(lineage.getNodes().stream().distinct().collect(Collectors.toList())); + lineage.withNodes(lineage.getNodes().stream().distinct().toList()); return lineage; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java index 5472de2d710..5d81fc92435 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java @@ -273,6 +273,53 @@ public class LineageResource { return Response.status(Status.OK).build(); } + @DELETE + @Path("/{fromEntity}/name/{fromFQN}/{toEntity}/name/{toFQN}") + @Operation( + operationId = "deleteLineageEdgeByName", + summary = "Delete a lineage edge by FQNs", + description = + "Delete a lineage edge with from entity as upstream node and to entity as downstream node.", + responses = { + @ApiResponse(responseCode = "200"), + @ApiResponse( + responseCode = "404", + description = "Entity for instance {fromFQN} is not found") + }) + public Response deleteLineageByName( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Entity type of upstream entity of the edge", + required = true, + schema = @Schema(type = "string", example = "table, report, metrics, or dashboard")) + @PathParam("fromEntity") + String fromEntity, + @Parameter(description = "Entity FQN", required = true, schema = @Schema(type = "string")) + @PathParam("fromFQN") + String fromFQN, + @Parameter( + description = "Entity type for downstream entity of the edge", + required = true, + schema = @Schema(type = "string", example = "table, report, metrics, or dashboard")) + @PathParam("toEntity") + String toEntity, + @Parameter(description = "Entity FQN", required = true, schema = @Schema(type = "string")) + @PathParam("toFQN") + String toFQN) { + authorizer.authorize( + securityContext, + new OperationContext(LINEAGE_FIELD, MetadataOperation.EDIT_LINEAGE), + new LineageResourceContext()); + boolean deleted = dao.deleteLineageByFQN(fromEntity, fromFQN, toEntity, toFQN); + if (!deleted) { + return Response.status(NOT_FOUND) + .entity(new ErrorMessage(NOT_FOUND.getStatusCode(), "Lineage edge not found")) + .build(); + } + return Response.status(Status.OK).build(); + } + private EntityLineage addHref(UriInfo uriInfo, EntityLineage lineage) { Entity.withHref(uriInfo, lineage.getEntity()); Entity.withHref(uriInfo, lineage.getNodes()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java index e515eace435..f402f39a7ce 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java @@ -21,6 +21,7 @@ import io.dropwizard.jersey.jackson.JacksonFeature; import io.dropwizard.testing.ConfigOverride; import io.dropwizard.testing.ResourceHelpers; import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.net.URI; import java.util.HashSet; import java.util.Set; import javax.ws.rs.client.Client; @@ -216,6 +217,11 @@ public abstract class OpenMetadataApplicationTest { return client.target(format("http://localhost:%s/api/v1/%s", APP.getLocalPort(), collection)); } + public static WebTarget getResourceAsURI(String collection) { + return client.target( + URI.create((format("http://localhost:%s/api/v1/%s", APP.getLocalPort(), collection)))); + } + public static WebTarget getConfigResource(String resource) { return client.target( format("http://localhost:%s/api/v1/system/config/%s", APP.getLocalPort(), resource)); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java index 2f7d8299eac..be87e695b7a 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java @@ -26,6 +26,7 @@ import static org.openmetadata.service.util.TestUtils.assertResponse; import java.io.IOException; import java.net.URISyntaxException; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -42,14 +43,23 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.data.CreateContainer; +import org.openmetadata.schema.api.data.CreateDashboard; import org.openmetadata.schema.api.data.CreateDashboardDataModel; +import org.openmetadata.schema.api.data.CreateMlModel; import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.data.CreateTopic; import org.openmetadata.schema.api.lineage.AddLineage; +import org.openmetadata.schema.entity.data.Container; +import org.openmetadata.schema.entity.data.Dashboard; import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.entity.data.MlModel; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.entity.teams.Role; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.ColumnLineage; +import org.openmetadata.schema.type.ContainerDataModel; import org.openmetadata.schema.type.Edge; import org.openmetadata.schema.type.EntitiesEdge; import org.openmetadata.schema.type.EntityLineage; @@ -58,11 +68,15 @@ import org.openmetadata.schema.type.LineageDetails; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.resources.dashboards.DashboardResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.datamodels.DashboardDataModelResourceTest; +import org.openmetadata.service.resources.mlmodels.MlModelResourceTest; +import org.openmetadata.service.resources.storages.ContainerResourceTest; import org.openmetadata.service.resources.teams.RoleResource; import org.openmetadata.service.resources.teams.RoleResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; +import org.openmetadata.service.resources.topics.TopicResourceTest; import org.openmetadata.service.util.TestUtils; @Slf4j @@ -71,10 +85,13 @@ public class LineageResourceTest extends OpenMetadataApplicationTest { public static final List TABLES = new ArrayList<>(); public static final int TABLE_COUNT = 10; private static final String DATA_STEWARD_ROLE_NAME = "DataSteward"; - private static DashboardDataModel DATA_MODEL; - private static Table TABLE_DATA_MODEL_LINEAGE; + private static Topic TOPIC; + private static Container CONTAINER; + private static MlModel ML_MODEL; + + private static Dashboard DASHBOARD; @BeforeAll public static void setup(TestInfo test) throws IOException, URISyntaxException { @@ -93,6 +110,25 @@ public class LineageResourceTest extends OpenMetadataApplicationTest { CreateTable createTable = tableResourceTest.createRequest(test, TABLE_COUNT); createTable.setColumns(createDashboardDataModel.getColumns()); TABLE_DATA_MODEL_LINEAGE = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + TopicResourceTest topicResourceTest = new TopicResourceTest(); + CreateTopic topicRequest = + topicResourceTest + .createRequest(test) + .withMessageSchema(TopicResourceTest.SCHEMA.withSchemaFields(TopicResourceTest.fields)); + TOPIC = topicResourceTest.createEntity(topicRequest, ADMIN_AUTH_HEADERS); + ContainerResourceTest containerResourceTest = new ContainerResourceTest(); + ContainerDataModel dataModel = + new ContainerDataModel().withColumns(ContainerResourceTest.dataModelColumns); + CreateContainer containerRequest = + containerResourceTest.createRequest(test).withDataModel(dataModel); + CONTAINER = containerResourceTest.createEntity(containerRequest, ADMIN_AUTH_HEADERS); + MlModelResourceTest mlModelResourceTest = new MlModelResourceTest(); + CreateMlModel createMlModel = + mlModelResourceTest.createRequest(test).withMlFeatures(MlModelResourceTest.ML_FEATURES); + ML_MODEL = mlModelResourceTest.createEntity(createMlModel, ADMIN_AUTH_HEADERS); + DashboardResourceTest dashboardResourceTest1 = new DashboardResourceTest(); + CreateDashboard createDashboard = dashboardResourceTest1.createRequest(test); + DASHBOARD = dashboardResourceTest1.createEntity(createDashboard, ADMIN_AUTH_HEADERS); } @Order(1) @@ -300,26 +336,22 @@ public class LineageResourceTest extends OpenMetadataApplicationTest { details.getColumnsLineage().clear(); details .getColumnsLineage() - .add(new ColumnLineage().withFromColumns(List.of(t1c1FQN, t3c1FQN)).withToColumn(t2c1FQN)); + .add(new ColumnLineage().withFromColumns(List.of(t1c1FQN, t1c3FQN)).withToColumn(t2c1FQN)); addEdge(TABLES.get(0), TABLES.get(1), details, ADMIN_AUTH_HEADERS); // Finally, add detailed column level lineage details.getColumnsLineage().clear(); List lineage = details.getColumnsLineage(); - lineage.add( - new ColumnLineage().withFromColumns(List.of(t1c1FQN, t3c1FQN)).withToColumn(t2c1FQN)); - lineage.add( - new ColumnLineage().withFromColumns(List.of(t1c2FQN, t3c2FQN)).withToColumn(t2c2FQN)); - lineage.add( - new ColumnLineage().withFromColumns(List.of(t1c3FQN, t3c3FQN)).withToColumn(t2c3FQN)); + lineage.add(new ColumnLineage().withFromColumns(List.of(t1c1FQN)).withToColumn(t2c1FQN)); + lineage.add(new ColumnLineage().withFromColumns(List.of(t1c2FQN)).withToColumn(t2c2FQN)); + lineage.add(new ColumnLineage().withFromColumns(List.of(t1c3FQN)).withToColumn(t2c3FQN)); addEdge(TABLES.get(0), TABLES.get(1), details, ADMIN_AUTH_HEADERS); } @Order(4) @Test - void putLineageFromDashboardDataModelToTable() throws HttpResponseException { - + void putLineageFromEntityToEntity() throws HttpResponseException { // Add column lineage dashboard.d1 -> table.c1 LineageDetails details = new LineageDetails(); String d1c1FQN = DATA_MODEL.getColumns().get(0).getFullyQualifiedName(); @@ -333,13 +365,83 @@ public class LineageResourceTest extends OpenMetadataApplicationTest { lineage.add(new ColumnLineage().withFromColumns(List.of(c1c1FQN)).withToColumn(d1c1FQN)); lineage.add(new ColumnLineage().withFromColumns(List.of(c1c2FQN)).withToColumn(d1c2FQN)); lineage.add(new ColumnLineage().withFromColumns(List.of(c1c3FQN)).withToColumn(d1c3FQN)); - addEdge(TABLE_DATA_MODEL_LINEAGE, DATA_MODEL, details, ADMIN_AUTH_HEADERS); + LineageDetails topicToTable = new LineageDetails(); + String f1FQN = TOPIC.getMessageSchema().getSchemaFields().get(0).getFullyQualifiedName(); + String f2FQN = TOPIC.getMessageSchema().getSchemaFields().get(0).getFullyQualifiedName(); + String f1t1 = TABLE_DATA_MODEL_LINEAGE.getColumns().get(0).getFullyQualifiedName(); + String f2t2 = TABLE_DATA_MODEL_LINEAGE.getColumns().get(1).getFullyQualifiedName(); + List topicToTableLineage = topicToTable.getColumnsLineage(); + topicToTableLineage.add(new ColumnLineage().withFromColumns(List.of(f1FQN)).withToColumn(f1t1)); + topicToTableLineage.add(new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2t2)); + addEdge(TOPIC, TABLE_DATA_MODEL_LINEAGE, topicToTable, ADMIN_AUTH_HEADERS); + String f3FQN = "test_non_existent_filed"; + topicToTableLineage.add( + new ColumnLineage().withFromColumns(List.of(f3FQN)).withToColumn(d1c1FQN)); assertResponse( - () -> addEdge(DATA_MODEL, TABLE_DATA_MODEL_LINEAGE, details, ADMIN_AUTH_HEADERS), + () -> addEdge(TOPIC, TABLE_DATA_MODEL_LINEAGE, topicToTable, ADMIN_AUTH_HEADERS), BAD_REQUEST, - "Column level lineage is only allowed between two tables or from table to dashboard."); + String.format("Invalid field name %s", f3FQN)); + + LineageDetails topicToContainer = new LineageDetails(); + String f1c1 = CONTAINER.getDataModel().getColumns().get(0).getFullyQualifiedName(); + String f2c2 = CONTAINER.getDataModel().getColumns().get(1).getFullyQualifiedName(); + List topicToContainerLineage = topicToContainer.getColumnsLineage(); + topicToContainerLineage.add( + new ColumnLineage().withFromColumns(List.of(f1FQN)).withToColumn(f1c1)); + topicToContainerLineage.add( + new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2c2)); + addEdge(TOPIC, CONTAINER, topicToContainer, ADMIN_AUTH_HEADERS); + String f2c3FQN = "test_non_existent_container_column"; + topicToContainerLineage.add( + new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2c3FQN)); + assertResponse( + () -> addEdge(TOPIC, CONTAINER, topicToContainer, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + String.format("Invalid fully qualified column name %s", f2c3FQN)); + + LineageDetails containerToTable = new LineageDetails(); + List containerToTableLineage = containerToTable.getColumnsLineage(); + containerToTableLineage.add( + new ColumnLineage().withFromColumns(List.of(f1c1)).withToColumn(f1t1)); + containerToTableLineage.add( + new ColumnLineage().withFromColumns(List.of(f2c2)).withToColumn(f2t2)); + addEdge(CONTAINER, TABLE_DATA_MODEL_LINEAGE, containerToTable, ADMIN_AUTH_HEADERS); + + LineageDetails tableToMlModel = new LineageDetails(); + String m1f1 = ML_MODEL.getMlFeatures().get(0).getFullyQualifiedName(); + String m2f2 = ML_MODEL.getMlFeatures().get(1).getFullyQualifiedName(); + List tableToMlModelLineage = tableToMlModel.getColumnsLineage(); + tableToMlModelLineage.add( + new ColumnLineage().withFromColumns(List.of(f1t1)).withToColumn(m1f1)); + tableToMlModelLineage.add( + new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(m2f2)); + addEdge(TABLE_DATA_MODEL_LINEAGE, ML_MODEL, tableToMlModel, ADMIN_AUTH_HEADERS); + String m3f3 = "test_non_existent_feature"; + tableToMlModelLineage.add( + new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(m3f3)); + assertResponse( + () -> addEdge(TABLE_DATA_MODEL_LINEAGE, ML_MODEL, tableToMlModel, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + String.format("Invalid feature name %s", m3f3)); + + LineageDetails tableToDashboard = new LineageDetails(); + String c1d1 = DASHBOARD.getCharts().get(0).getFullyQualifiedName(); + String c2d1 = DASHBOARD.getCharts().get(1).getFullyQualifiedName(); + + List tableToDashboardLineage = tableToDashboard.getColumnsLineage(); + tableToDashboardLineage.add( + new ColumnLineage().withFromColumns(List.of(f1t1)).withToColumn(c1d1)); + tableToDashboardLineage.add( + new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(c2d1)); + addEdge(TABLE_DATA_MODEL_LINEAGE, DASHBOARD, tableToDashboard, ADMIN_AUTH_HEADERS); + + deleteEdgeByName( + TOPIC.getEntityReference().getType(), + TOPIC.getFullyQualifiedName(), + CONTAINER.getEntityReference().getType(), + CONTAINER.getFullyQualifiedName()); } @Order(5) @@ -386,6 +488,11 @@ public class LineageResourceTest extends OpenMetadataApplicationTest { deleteEdge(from, to, ADMIN_AUTH_HEADERS); } + public void deleteEdgeByName(String fromEntity, String fromFQN, String toEntity, String toFQN) + throws HttpResponseException { + deleteLineageByName(fromEntity, fromFQN, toEntity, toFQN, ADMIN_AUTH_HEADERS); + } + private void deleteEdge(Table from, Table to, Map authHeaders) throws HttpResponseException { EntitiesEdge edge = @@ -425,6 +532,21 @@ public class LineageResourceTest extends OpenMetadataApplicationTest { TestUtils.delete(target, authHeaders); } + public void deleteLineageByName( + String fromEntity, + String fromFQN, + String toEntity, + String toFQN, + Map authHeaders) + throws HttpResponseException { + WebTarget target = + getResourceAsURI( + String.format( + "lineage/%s/name/%s/%s/name/%s", + fromEntity, URLEncoder.encode(fromFQN), toEntity, URLEncoder.encode(toFQN))); + TestUtils.delete(target, authHeaders); + } + private void validateLineage(AddLineage addLineage, Map authHeaders) throws HttpResponseException { EntityReference from = addLineage.getEdge().getFromEntity(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/mlmodels/MlModelResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/mlmodels/MlModelResourceTest.java index 2ffffe4a87a..8fad246ee36 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/mlmodels/MlModelResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/mlmodels/MlModelResourceTest.java @@ -465,12 +465,12 @@ public class MlModelResourceTest extends EntityResourceTest { - private static final String SCHEMA_TEXT = + public static final String SCHEMA_TEXT = "{\"namespace\":\"org.open-metadata.kafka\",\"name\":\"Customer\",\"type\":\"record\"," + "\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"}," + "{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"address_line_1\",\"type\":\"string\"},{\"name\":\"address_line_2\",\"type\":\"string\"}," + "{\"name\":\"post_code\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"}]}"; - private static final MessageSchema schema = + public static final List fields = + Arrays.asList( + getField("id", FieldDataType.STRING, null), + getField("first_name", FieldDataType.STRING, null), + getField("last_name", FieldDataType.STRING, null), + getField("email", FieldDataType.STRING, null), + getField("address_line_1", FieldDataType.STRING, null), + getField("address_line_2", FieldDataType.STRING, null), + getField("post_code", FieldDataType.STRING, null), + getField("county", FieldDataType.STRING, PERSONAL_DATA_TAG_LABEL)); + public static final MessageSchema SCHEMA = new MessageSchema().withSchemaText(SCHEMA_TEXT).withSchemaType(SchemaType.Avro); public TopicResourceTest() { @@ -171,16 +181,6 @@ public class TopicResourceTest extends EntityResourceTest { @Test void put_topicSchemaFields_200_ok(TestInfo test) throws IOException { - List fields = - Arrays.asList( - getField("id", FieldDataType.STRING, null), - getField("first_name", FieldDataType.STRING, null), - getField("last_name", FieldDataType.STRING, null), - getField("email", FieldDataType.STRING, null), - getField("address_line_1", FieldDataType.STRING, null), - getField("address_line_2", FieldDataType.STRING, null), - getField("post_code", FieldDataType.STRING, null), - getField("county", FieldDataType.STRING, PERSONAL_DATA_TAG_LABEL)); CreateTopic createTopic = createRequest(test) @@ -191,7 +191,7 @@ public class TopicResourceTest extends EntityResourceTest { .withReplicationFactor(1) .withRetentionTime(1.0) .withRetentionSize(1.0) - .withMessageSchema(schema.withSchemaFields(fields)) + .withMessageSchema(SCHEMA.withSchemaFields(fields)) .withCleanupPolicies(List.of(CleanupPolicy.COMPACT)); // Patch and update the topic @@ -221,7 +221,7 @@ public class TopicResourceTest extends EntityResourceTest { .withReplicationFactor(1) .withRetentionTime(1.0) .withRetentionSize(1.0) - .withMessageSchema(schema.withSchemaFields(fields)) + .withMessageSchema(SCHEMA.withSchemaFields(fields)) .withCleanupPolicies(List.of(CleanupPolicy.COMPACT)); // Patch and update the topic @@ -236,7 +236,7 @@ public class TopicResourceTest extends EntityResourceTest { .withReplicationFactor(2) .withRetentionTime(2.0) .withRetentionSize(2.0) - .withMessageSchema(schema.withSchemaFields(fields)) + .withMessageSchema(SCHEMA.withSchemaFields(fields)) .withCleanupPolicies(List.of(CleanupPolicy.DELETE)); ChangeDescription change = getChangeDescription(topic, MINOR_UPDATE); @@ -284,7 +284,7 @@ public class TopicResourceTest extends EntityResourceTest { Field field = getField("first_name", FieldDataType.STRING, null) .withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL)); - create1.withMessageSchema(schema.withSchemaFields(List.of(field))); + create1.withMessageSchema(SCHEMA.withSchemaFields(List.of(field))); assertResponse( () -> createEntity(create1, ADMIN_AUTH_HEADERS), BAD_REQUEST, @@ -304,7 +304,7 @@ public class TopicResourceTest extends EntityResourceTest { getField("testNested", FieldDataType.STRING, null) .withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL)); Field field1 = getField("test", FieldDataType.RECORD, null).withChildren(List.of(nestedField)); - create2.setMessageSchema(schema.withSchemaFields(List.of(field1))); + create2.setMessageSchema(SCHEMA.withSchemaFields(List.of(field1))); assertResponse( () -> createEntity(create2, ADMIN_AUTH_HEADERS), BAD_REQUEST, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.ts b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.ts index 17331870b7d..3249fe7d39e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.ts @@ -44,6 +44,7 @@ export const LINEAGE_ITEMS = [ fqn: 'sample_kafka.shop_products', entityType: 'Topic', searchIndex: SEARCH_INDEX.topics, + columns: ['sample_kafka.shop_products.Shop.shop_id'], }, { term: 'forecast_sales', @@ -52,14 +53,16 @@ export const LINEAGE_ITEMS = [ entityType: 'ML Model', fqn: 'mlflow_svc.forecast_sales', searchIndex: SEARCH_INDEX.mlmodels, + columns: [], }, { - term: 'media', + term: 'transactions', entity: DATA_ASSETS.containers, serviceName: 's3_storage_sample', entityType: 'Container', - fqn: 's3_storage_sample.departments.media', + fqn: 's3_storage_sample.transactions', searchIndex: SEARCH_INDEX.containers, + columns: ['s3_storage_sample.transactions.transaction_id'], }, { term: 'customers', @@ -68,6 +71,7 @@ export const LINEAGE_ITEMS = [ entityType: 'Dashboard', fqn: 'sample_looker.customers', searchIndex: SEARCH_INDEX.dashboards, + columns: ['sample_looker.chart_1'], }, ]; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.ts index 8f67a47cd36..444ad1d4a9b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.ts @@ -13,6 +13,7 @@ import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { visitEntityDetailsPage } from '../../common/Utils/Entity'; +import { EntityType } from '../../constants/Entity.interface'; import { LINEAGE_ITEMS, PIPELINE_ITEMS, @@ -133,8 +134,8 @@ const verifyPipelineDataInDrawer = ( const addPipelineBetweenNodes = ( sourceEntity, targetEntity, - pipelineItem, - bVerifyPipeline + pipelineItem?, + bVerifyPipeline?: boolean ) => { visitEntityDetailsPage({ term: sourceEntity.term, @@ -171,16 +172,18 @@ const expandCols = (nodeFqn, hasShowMore) => { } }; -const addColumnLineage = (fromNode, toNode) => { +const addColumnLineage = (fromNode, toNode, exitEditMode = true) => { interceptURL('PUT', '/api/v1/lineage', 'lineageApi'); expandCols(fromNode.fqn, false); - expandCols(toNode.fqn, true); + expandCols(toNode.fqn, toNode.entityType === EntityType.Table); dragConnection( `column-${fromNode.columns[0]}`, `column-${toNode.columns[0]}` ); verifyResponseStatusCode('@lineageApi', 200); - cy.get('[data-testid="edit-lineage"]').click(); + if (exitEditMode) { + cy.get('[data-testid="edit-lineage"]').click(); + } cy.get( `[data-testid="column-edge-${fromNode.columns[0]}-${toNode.columns[0]}"]` ); @@ -196,7 +199,7 @@ describe('Lineage verification', { tags: 'DataAssets' }, () => { visitEntityDetailsPage({ term: entity.term, serviceName: entity.serviceName, - entity: entity.entity, + entity: entity.entity as EntityType, }); cy.get('[data-testid="lineage"]').click(); @@ -230,7 +233,7 @@ describe('Lineage verification', { tags: 'DataAssets' }, () => { visitEntityDetailsPage({ term: entity.term, serviceName: entity.serviceName, - entity: entity.entity, + entity: entity.entity as EntityType, }); cy.get('[data-testid="lineage"]').click(); @@ -280,11 +283,16 @@ describe('Lineage verification', { tags: 'DataAssets' }, () => { it('Add column lineage', () => { const sourceEntity = LINEAGE_ITEMS[0]; - const targetEntity = LINEAGE_ITEMS[1]; - addPipelineBetweenNodes(sourceEntity, targetEntity); - // Add column lineage - addColumnLineage(sourceEntity, targetEntity); - cy.get('[data-testid="edit-lineage"]').click(); - deleteNode(targetEntity); + for (let i = 1; i < LINEAGE_ITEMS.length; i++) { + const targetEntity = LINEAGE_ITEMS[i]; + if (targetEntity.columns.length > 0) { + addPipelineBetweenNodes(sourceEntity, targetEntity); + // Add column lineage + addColumnLineage(sourceEntity, targetEntity); + cy.get('[data-testid="edit-lineage"]').click(); + deleteNode(targetEntity); + cy.goToHomePage(); + } + } }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx index a2a607a5f95..73854e46d81 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx @@ -12,11 +12,16 @@ */ import { Button } from 'antd'; import classNames from 'classnames'; +import { isEmpty } from 'lodash'; import React, { Fragment } from 'react'; import { Handle, HandleProps, HandleType, Position } from 'reactflow'; import { ReactComponent as MinusIcon } from '../../../assets/svg/control-minus.svg'; import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-outlined.svg'; import { EntityLineageNodeType } from '../../../enums/entity.enum'; +import { Column, TestSuite } from '../../../generated/entity/data/table'; +import { formTwoDigitNumber } from '../../../utils/CommonUtils'; +import { getEntityName } from '../../../utils/EntityUtils'; +import { getConstraintIcon } from '../../../utils/TableUtils'; import { EdgeTypeEnum } from './EntityLineage.interface'; export const getHandleByType = ( @@ -111,3 +116,65 @@ export const getCollapseHandle = ( /> ); }; + +export const getTestSuiteSummary = (testSuite?: TestSuite) => { + if (isEmpty(testSuite)) { + return null; + } + + return ( +
+
+
+ {formTwoDigitNumber(testSuite?.summary?.success ?? 0)} +
+
+
+
+ {formTwoDigitNumber(testSuite?.summary?.aborted ?? 0)} +
+
+
+
+ {formTwoDigitNumber(testSuite?.summary?.failed ?? 0)} +
+
+
+ ); +}; + +export const getColumnContent = ( + column: Column, + isColumnTraced: boolean, + isConnectable: boolean, + onColumnClick: (column: string) => void +) => { + const { fullyQualifiedName } = column; + + return ( +
{ + e.stopPropagation(); + onColumnClick(fullyQualifiedName ?? ''); + }}> + {getColumnHandle( + EntityLineageNodeType.DEFAULT, + isConnectable, + 'lineage-column-node-handle', + fullyQualifiedName + )} + {getConstraintIcon({ + constraint: column.constraint, + })} +

{getEntityName(column)}

+
+ ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx index d45843e7474..339e4784ea7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx @@ -11,13 +11,10 @@ * limitations under the License. */ -import { DownOutlined, SearchOutlined, UpOutlined } from '@ant-design/icons'; import Icon from '@ant-design/icons/lib/components/Icon'; -import { Button, Input } from 'antd'; +import { Button } from 'antd'; import classNames from 'classnames'; -import { isEmpty } from 'lodash'; import React, { memo, useCallback, useEffect, useMemo, useState } from 'react'; -import { useTranslation } from 'react-i18next'; import { getIncomers, getOutgoers, @@ -26,37 +23,26 @@ import { Position, } from 'reactflow'; import { ReactComponent as IconTimesCircle } from '../../../assets/svg/ic-times-circle.svg'; -import { BORDER_COLOR } from '../../../constants/constants'; import { useLineageProvider } from '../../../context/LineageProvider/LineageProvider'; -import { EntityLineageNodeType, EntityType } from '../../../enums/entity.enum'; -import { formTwoDigitNumber } from '../../../utils/CommonUtils'; +import { EntityLineageNodeType } from '../../../enums/entity.enum'; import { checkUpstreamDownstream } from '../../../utils/EntityLineageUtils'; -import { getEntityName } from '../../../utils/EntityUtils'; -import { getConstraintIcon, getEntityIcon } from '../../../utils/TableUtils'; import './custom-node.less'; -import { - getCollapseHandle, - getColumnHandle, - getExpandHandle, -} from './CustomNode.utils'; +import { getCollapseHandle, getExpandHandle } from './CustomNode.utils'; import './entity-lineage.style.less'; -import { EdgeTypeEnum, ModifiedColumn } from './EntityLineage.interface'; +import { EdgeTypeEnum } from './EntityLineage.interface'; import LineageNodeLabelV1 from './LineageNodeLabelV1'; +import NodeChildren from './NodeChildren/NodeChildren.component'; const CustomNodeV1 = (props: NodeProps) => { - const { t } = useTranslation(); const { data, type, isConnectable } = props; const { isEditMode, - expandedNodes, tracedNodes, - tracedColumns, selectedNode, nodes, edges, upstreamDownstreamData, - onColumnClick, onNodeCollapse, removeNodeHandler, loadChildNodesHandler, @@ -65,11 +51,8 @@ const CustomNodeV1 = (props: NodeProps) => { const { label, isNewNode, node = {}, isRootNode } = data; const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type; const isSelected = selectedNode === node; - const { columns, id, testSuite, lineage, fullyQualifiedName } = node; - const [searchValue, setSearchValue] = useState(''); - const [filteredColumns, setFilteredColumns] = useState([]); - const [showAllColumns, setShowAllColumns] = useState(false); - const [isExpanded, setIsExpanded] = useState(false); + const { id, lineage, fullyQualifiedName } = node; + const [isTraced, setIsTraced] = useState(false); const getActiveNode = useCallback( @@ -105,14 +88,6 @@ const CustomNodeV1 = (props: NodeProps) => { }; }, [id, nodes, edges, hasUpstream, hasDownstream]); - const supportsColumns = useMemo(() => { - if (node && node.entityType === EntityType.TABLE) { - return true; - } - - return false; - }, [node.id]); - const { isUpstreamNode, isDownstreamNode } = useMemo(() => { return { isUpstreamNode: upstreamDownstreamData.upstreamNodes.some( @@ -124,40 +99,6 @@ const CustomNodeV1 = (props: NodeProps) => { }; }, [fullyQualifiedName, upstreamDownstreamData]); - const handleSearchChange = useCallback( - (e: React.ChangeEvent) => { - e.stopPropagation(); - const value = e.target.value; - setSearchValue(value); - - if (value.trim() === '') { - // If search value is empty, show all columns or the default number of columns - const filterColumns = Object.values(columns || {}) as ModifiedColumn[]; - setFilteredColumns( - showAllColumns ? filterColumns : filterColumns.slice(0, 5) - ); - } else { - // Filter columns based on search value - const filtered = ( - Object.values(columns || {}) as ModifiedColumn[] - ).filter((column) => - getEntityName(column).toLowerCase().includes(value.toLowerCase()) - ); - setFilteredColumns(filtered); - } - }, - [columns] - ); - - const handleShowMoreClick = useCallback( - (e: React.MouseEvent) => { - e.stopPropagation(); - setShowAllColumns(true); - setFilteredColumns(Object.values(columns ?? [])); - }, - [] - ); - const onExpand = useCallback( (direction: EdgeTypeEnum) => { loadChildNodesHandler(node, direction); @@ -307,30 +248,10 @@ const CustomNodeV1 = (props: NodeProps) => { loadChildNodesHandler, ]); - useEffect(() => { - setIsExpanded(expandedNodes.includes(id)); - }, [expandedNodes, id]); - useEffect(() => { setIsTraced(tracedNodes.includes(id)); }, [tracedNodes, id]); - useEffect(() => { - if (!isExpanded) { - setShowAllColumns(false); - } else if (!isEmpty(columns) && Object.values(columns).length < 5) { - setShowAllColumns(true); - } - }, [isEditMode, isExpanded, columns]); - - useEffect(() => { - if (!isEmpty(columns)) { - setFilteredColumns( - Object.values(columns).slice(0, 5) as ModifiedColumn[] - ); - } - }, [columns]); - return (
{ {getHandle()}
{nodeLabel}
- {supportsColumns && ( -
-
-
- } - type="text" - onClick={(e) => { - e.stopPropagation(); - setIsExpanded((prevIsExpanded: boolean) => !prevIsExpanded); - }}> - {t('label.column-plural')} - {isExpanded ? ( - - ) : ( - - )} - - {node.entityType === EntityType.TABLE && testSuite && ( -
-
-
- {formTwoDigitNumber(testSuite?.summary?.success ?? 0)} -
-
-
-
- {formTwoDigitNumber(testSuite?.summary?.aborted ?? 0)} -
-
-
-
- {formTwoDigitNumber(testSuite?.summary?.failed ?? 0)} -
-
-
- )} -
- - {isExpanded && ( -
-
- } - value={searchValue} - onChange={handleSearchChange} - /> -
- -
-
- {filteredColumns.map((column) => { - const isColumnTraced = tracedColumns.includes( - column.fullyQualifiedName ?? '' - ); - - return ( -
{ - e.stopPropagation(); - onColumnClick(column.fullyQualifiedName ?? ''); - }}> - {getColumnHandle( - column.type, - isConnectable, - 'lineage-column-node-handle', - column.fullyQualifiedName - )} - {getConstraintIcon({ constraint: column.constraint })} -

{getEntityName(column)}

-
- ); - })} -
-
- - {!showAllColumns && ( - - )} -
- )} -
- )} +
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.component.tsx new file mode 100644 index 00000000000..b818c1ca9e6 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.component.tsx @@ -0,0 +1,263 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DownOutlined, SearchOutlined, UpOutlined } from '@ant-design/icons'; +import { Button, Collapse, Input } from 'antd'; +import { isEmpty } from 'lodash'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { BORDER_COLOR } from '../../../../constants/constants'; +import { LINEAGE_COLUMN_NODE_SUPPORTED } from '../../../../constants/Lineage.constants'; +import { useLineageProvider } from '../../../../context/LineageProvider/LineageProvider'; +import { EntityType } from '../../../../enums/entity.enum'; +import { Container } from '../../../../generated/entity/data/container'; +import { Dashboard } from '../../../../generated/entity/data/dashboard'; +import { Mlmodel } from '../../../../generated/entity/data/mlmodel'; +import { Column, Table } from '../../../../generated/entity/data/table'; +import { Topic } from '../../../../generated/entity/data/topic'; +import { getEntityName } from '../../../../utils/EntityUtils'; +import { getEntityIcon } from '../../../../utils/TableUtils'; +import { getColumnContent, getTestSuiteSummary } from '../CustomNode.utils'; +import { EntityChildren, NodeChildrenProps } from './NodeChildren.interface'; + +const NodeChildren = ({ node, isConnectable }: NodeChildrenProps) => { + const { t } = useTranslation(); + const { Panel } = Collapse; + const { isEditMode, tracedColumns, expandedNodes, onColumnClick } = + useLineageProvider(); + const { entityType, id } = node; + const [searchValue, setSearchValue] = useState(''); + const [filteredColumns, setFilteredColumns] = useState([]); + const [showAllColumns, setShowAllColumns] = useState(false); + const [isExpanded, setIsExpanded] = useState(false); + + const supportsColumns = useMemo(() => { + return ( + node && + LINEAGE_COLUMN_NODE_SUPPORTED.includes(node.entityType as EntityType) + ); + }, [node.id]); + + const { children, childrenHeading } = useMemo(() => { + const entityMappings: Record< + string, + { data: EntityChildren; label: string } + > = { + [EntityType.TABLE]: { + data: (node as Table).columns ?? [], + label: t('label.column-plural'), + }, + [EntityType.DASHBOARD]: { + data: (node as Dashboard).charts ?? [], + label: t('label.chart-plural'), + }, + [EntityType.MLMODEL]: { + data: (node as Mlmodel).mlFeatures ?? [], + label: t('label.feature-plural'), + }, + [EntityType.DASHBOARD_DATA_MODEL]: { + data: (node as Table).columns ?? [], + label: t('label.column-plural'), + }, + [EntityType.CONTAINER]: { + data: (node as Container).dataModel?.columns ?? [], + label: t('label.column-plural'), + }, + [EntityType.TOPIC]: { + data: (node as Topic).messageSchema?.schemaFields ?? [], + label: t('label.field-plural'), + }, + }; + + const { data, label } = entityMappings[node.entityType as EntityType] || { + data: [], + label: '', + }; + + return { + children: data, + childrenHeading: label, + }; + }, [node.id]); + + const handleSearchChange = useCallback( + (e: React.ChangeEvent) => { + e.stopPropagation(); + const value = e.target.value; + setSearchValue(value); + + if (value.trim() === '') { + // If search value is empty, show all columns or the default number of columns + const filterColumns = Object.values(children ?? {}); + setFilteredColumns( + showAllColumns ? filterColumns : filterColumns.slice(0, 5) + ); + } else { + // Filter columns based on search value + const filtered = Object.values(children ?? {}).filter((column) => + getEntityName(column).toLowerCase().includes(value.toLowerCase()) + ); + setFilteredColumns(filtered); + } + }, + [children] + ); + + useEffect(() => { + setIsExpanded(expandedNodes.includes(id ?? '')); + }, [expandedNodes, id]); + + useEffect(() => { + if (!isEmpty(children)) { + setFilteredColumns(children.slice(0, 5)); + } + }, [children]); + + useEffect(() => { + if (!isExpanded) { + setShowAllColumns(false); + } else if (!isEmpty(children) && Object.values(children).length < 5) { + setShowAllColumns(true); + } + }, [isEditMode, isExpanded, children]); + + const handleShowMoreClick = useCallback( + (e: React.MouseEvent) => { + e.stopPropagation(); + setShowAllColumns(true); + setFilteredColumns(children ?? []); + }, + [children] + ); + + const renderRecord = useCallback( + (record: Column) => { + return ( + + + {record?.children?.map((child) => { + const { fullyQualifiedName, dataType } = child; + if (['RECORD', 'STRUCT'].includes(dataType)) { + return renderRecord(child); + } else { + const isColumnTraced = tracedColumns.includes( + fullyQualifiedName ?? '' + ); + + return getColumnContent( + child, + isColumnTraced, + isConnectable, + onColumnClick + ); + } + })} + + + ); + }, + [isConnectable, tracedColumns] + ); + + const renderColumnsData = useCallback( + (column: Column) => { + const { fullyQualifiedName, dataType } = column; + if (['RECORD', 'STRUCT'].includes(dataType)) { + return renderRecord(column); + } else { + const isColumnTraced = tracedColumns.includes(fullyQualifiedName ?? ''); + + return getColumnContent( + column, + isColumnTraced, + isConnectable, + onColumnClick + ); + } + }, + [isConnectable, tracedColumns] + ); + + if (supportsColumns) { + return ( +
+
+
+ } + type="text" + onClick={(e) => { + e.stopPropagation(); + setIsExpanded((prevIsExpanded: boolean) => !prevIsExpanded); + }}> + {childrenHeading} + {isExpanded ? ( + + ) : ( + + )} + + {entityType === EntityType.TABLE && + getTestSuiteSummary((node as Table).testSuite)} +
+ + {isExpanded && ( +
+
+ } + value={searchValue} + onChange={handleSearchChange} + /> +
+ +
+
+ {filteredColumns.map((column) => + renderColumnsData(column as Column) + )} +
+
+ + {!showAllColumns && ( + + )} +
+ )} + + ); + } else { + return null; + } +}; + +export default NodeChildren; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.interface.ts new file mode 100644 index 00000000000..db2aa15ee81 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeChildren/NodeChildren.interface.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { MlFeature } from '../../../../generated/entity/data/mlmodel'; +import { Column } from '../../../../generated/entity/data/table'; +import { Field } from '../../../../generated/entity/data/topic'; +import { EntityReference } from '../../../../generated/entity/type'; +import { SearchedDataProps } from '../../../SearchedData/SearchedData.interface'; + +export interface NodeChildrenProps { + node: SearchedDataProps['data'][number]['_source']; + isConnectable: boolean; +} + +export type EntityChildren = + | Column[] + | EntityReference[] + | MlFeature[] + | Field[]; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx index 99e7bdcbe4f..1615f805616 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/NodeSuggestions.component.tsx @@ -14,7 +14,6 @@ import { Select } from 'antd'; import { AxiosError } from 'axios'; import { capitalize, debounce } from 'lodash'; -import { FormattedTableData } from 'Models'; import React, { FC, HTMLAttributes, @@ -28,15 +27,14 @@ import { PAGE_SIZE } from '../../../constants/constants'; import { EntityType, FqnPart } from '../../../enums/entity.enum'; import { SearchIndex } from '../../../enums/search.enum'; import { EntityReference } from '../../../generated/entity/type'; -import { SearchSourceAlias } from '../../../interface/search.interface'; import { searchData } from '../../../rest/miscAPI'; -import { formatDataResponse } from '../../../utils/APIUtils'; import { getPartialNameFromTableFQN } from '../../../utils/CommonUtils'; import { getEntityNodeIcon } from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; import serviceUtilClassBase from '../../../utils/ServiceUtilClassBase'; import { showErrorToast } from '../../../utils/ToastUtils'; import { ExploreSearchIndex } from '../../Explore/ExplorePage.interface'; +import { SourceType } from '../../SearchedData/SearchedData.interface'; import './node-suggestion.less'; interface EntitySuggestionProps extends HTMLAttributes { @@ -50,7 +48,7 @@ const NodeSuggestions: FC = ({ }) => { const { t } = useTranslation(); - const [data, setData] = useState>([]); + const [data, setData] = useState>([]); const [searchValue, setSearchValue] = useState(''); @@ -78,7 +76,8 @@ const NodeSuggestions: FC = ({ '', (entityType as ExploreSearchIndex) ?? SearchIndex.TABLE ); - setData(formatDataResponse(data.data.hits.hits)); + const sources = data.data.hits.hits.map((hit) => hit._source); + setData(sources); } catch (error) { showErrorToast( error as AxiosError, @@ -133,16 +132,14 @@ const NodeSuggestions: FC = ({ alt={entity.serviceType} className="m-r-xs" height="16px" - src={serviceUtilClassBase.getServiceTypeLogo( - entity as SearchSourceAlias - )} + src={serviceUtilClassBase.getServiceTypeLogo(entity)} width="16px" />
{entity.entityType === EntityType.TABLE && (

{getSuggestionLabelHeading( - entity.fullyQualifiedName, + entity.fullyQualifiedName ?? '', entity.entityType as string )}

diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less index 66225141039..633b3fdabc9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/custom-node.less @@ -59,6 +59,25 @@ border: 1px solid @red-5; } } + .column-container { + .ant-collapse-arrow { + margin-right: 0 !important; + } + .ant-collapse { + border-radius: 0; + } + .ant-collapse-header { + padding: 4px 12px !important; + font-size: 12px; + } + .ant-collapse-content-box { + padding: 4px !important; + } + .ant-collapse-item { + border: none !important; + border-radius: 0; + } + } } .react-flow__node-default, @@ -151,13 +170,13 @@ } .react-flow .lineage-node-handle { - width: 35px; - min-width: 35px; - height: 35px; - border-radius: 50%; - border-color: @lineage-border; - background: @white; - top: 43px; // Need to show handles on top half + width: 35px !important; + min-width: 35px !important; + height: 35px !important; + border-radius: 50% !important; + border-color: @lineage-border !important; + background: @white !important; + top: 43px !important; // Need to show handles on top half svg { color: @text-grey-muted; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage-sidebar.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage-sidebar.less index 769bebb5e31..bb123b55f56 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage-sidebar.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage-sidebar.less @@ -20,7 +20,7 @@ } width: 46px; - box-sizing: content-box; + box-sizing: content-box !important; border: @global-border; display: flex; align-items: center; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less index a4f5a01c736..17e0c3f7064 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/entity-lineage.style.less @@ -68,11 +68,15 @@ } } .custom-node-column-lineage-normal { - border-bottom: 1px solid @border-color; + border-top: 1px solid @border-color; display: flex; justify-content: center; align-items: center; position: relative; + font-size: 12px; + &:first-child { + border-top: none; + } } .custom-node { @@ -134,6 +138,12 @@ &.active { background-color: @primary-color; } + &.active:hover { + background-color: @primary-color; + } + &.active:focus { + background-color: @primary-color; + } } .custom-lineage-heading { @@ -185,12 +195,6 @@ } // lineage -.lineage-card { - > .ant-card-body { - padding: 0; - } -} - .lineage-card { height: calc(100vh - 240px); @@ -223,7 +227,7 @@ } .lineage-node-remove-btn { - position: absolute; + position: absolute !important; top: -20px; right: -20px; cursor: pointer; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx index f03afe70400..42e71b72c89 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx @@ -118,7 +118,7 @@ const Lineage = ({ // considerably. So added an init state for showing loader. return ( {isFullScreen && ( diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts index 031368c16a6..2c34530d9ec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Lineage.constants.ts @@ -13,6 +13,7 @@ import { t } from 'i18next'; import { ElementLoadingState } from '../components/Entity/EntityLineage/EntityLineage.interface'; +import { EntityType } from '../enums/entity.enum'; import { SearchIndex } from '../enums/search.enum'; import { Source } from '../generated/type/entityLineage'; @@ -86,3 +87,12 @@ export const LINEAGE_SOURCE: { [key in Source]: string } = { [Source.OpenLineage]: 'OpenLineage', [Source.ExternalTableLineage]: 'External Table Lineage', }; + +export const LINEAGE_COLUMN_NODE_SUPPORTED = [ + EntityType.TABLE, + EntityType.DASHBOARD, + EntityType.MLMODEL, + EntityType.DASHBOARD_DATA_MODEL, + EntityType.CONTAINER, + EntityType.TOPIC, +]; diff --git a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx index 22c3d3c4af3..e5f2836f2ae 100644 --- a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx @@ -67,6 +67,7 @@ import { import { AddLineage } from '../../generated/api/lineage/addLineage'; import { PipelineStatus } from '../../generated/entity/data/pipeline'; import { + ColumnLineage, EntityReference, LineageDetails, } from '../../generated/type/entityLineage'; @@ -404,7 +405,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }; const removeNodeHandler = useCallback( - (node: Node | NodeProps) => { + async (node: Node | NodeProps) => { if (!entityLineage) { return; } @@ -414,18 +415,57 @@ const LineageProvider = ({ children }: LineageProviderProps) => { (item) => item?.data?.isColumnLineage === false ); - // Get edges connected to selected node const edgesToRemove = getConnectedEdges([node as Node], nodeEdges); - edgesToRemove.forEach((edge) => { - removeEdgeHandler(edge, true); + + const filteredEdges: EdgeDetails[] = []; + + await Promise.all( + edgesToRemove.map(async (edge) => { + const { data } = edge; + const edgeData: EdgeData = { + fromEntity: data.edge.fromEntity.type, + fromId: data.edge.fromEntity.id, + toEntity: data.edge.toEntity.type, + toId: data.edge.toEntity.id, + }; + + await removeLineageHandler(edgeData); + + filteredEdges.push( + ...(entityLineage.edges ?? []).filter( + (item) => + !( + item.fromEntity.id === edgeData.fromId && + item.toEntity.id === edgeData.toId + ) + ) + ); + + setEdges((prev) => { + return prev.filter( + (item) => + !( + item.source === edgeData.fromId && + item.target === edgeData.toId + ) + ); + }); + }) + ); + + const updatedNodes = (entityLineage.nodes ?? []).filter( + (previousNode) => previousNode.id !== node.id + ); + + setNodes((prev) => { + return prev.filter((previousNode) => previousNode.id !== node.id); }); - setEntityLineage((prev) => { + setUpdatedEntityLineage(() => { return { - ...prev, - nodes: (prev.nodes ?? []).filter( - (previousNode) => previousNode.id !== node.id - ), + ...entityLineage, + edges: filteredEdges, + nodes: updatedNodes, }; }); @@ -636,8 +676,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => { targetNode.data.node ); + let updatedColumns: ColumnLineage[] = []; + if (columnConnection && currentEdge) { - const updatedColumns = getUpdatedColumnsFromEdge(params, currentEdge); + updatedColumns = getUpdatedColumnsFromEdge(params, currentEdge); const lineageDetails: LineageDetails = { pipeline: currentEdge.pipeline, @@ -647,7 +689,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }; lineageDetails.columnsLineage = updatedColumns; newEdgeWithoutFqn.edge.lineageDetails = lineageDetails; - currentEdge.columns = updatedColumns; // update current edge with new columns } addLineageHandler(newEdgeWithoutFqn) @@ -668,6 +709,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => { ? [...(entityLineage.edges ?? []), newEdgeWithFqn.edge] : entityLineage.edges ?? []; + if (currentEdge && columnConnection) { + currentEdge.columns = updatedColumns; // update current edge with new columns + } + setEntityLineage((pre) => { const newData = { ...pre,