mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-01 11:52:12 +00:00
Show fields for other lineage entities (#15513)
* show fields for other entities * fix minor delete issue * add support for topics and dashboards * minor fixes * Remove constraints of connecting column level lineae * Enable Column Level lineage for all the entities * Add delete edge by name apis * fix fields display on drop * Fix finding a nested column while adding column level lineage * Fix finding a nested column while adding column level lineage * add support for collapsible record and struct fields * add column level lineage cypress * minor fixes * Fix dashboard chart lineage * Fix dashboard chart lineage --------- Co-authored-by: Sriharsha Chintalapani <harsha@getcollate.io>
This commit is contained in:
Karan Hotchandani
GitHub
parent
0eb18ad891
commit
f9f7cde43e
@ -17,6 +17,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@ -207,4 +208,24 @@ public final class CommonUtil {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static <T> boolean findChildren(List<?> list, String methodName, String fqn) {
|
||||
if (list == null || list.isEmpty()) return false;
|
||||
try {
|
||||
Method getChildren = list.get(0).getClass().getMethod(methodName);
|
||||
Method getFQN = list.get(0).getClass().getMethod("getFullyQualifiedName");
|
||||
return list.stream()
|
||||
.anyMatch(
|
||||
o -> {
|
||||
try {
|
||||
return getFQN.invoke(o).equals(fqn)
|
||||
|| findChildren((List<?>) getChildren.invoke(o), methodName, fqn);
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -158,6 +158,10 @@ public final class CatalogExceptionMessage {
|
||||
return String.format("Invalid %s name %s", fieldType, fieldName);
|
||||
}
|
||||
|
||||
public static String invalidFieldFQN(String fqn) {
|
||||
return String.format("Invalid fully qualified field name %s", fqn);
|
||||
}
|
||||
|
||||
public static String entityVersionNotFound(String entityType, UUID id, Double version) {
|
||||
return String.format("%s instance for %s and version %s not found", entityType, id, version);
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package org.openmetadata.service.jdbi3;
|
||||
|
||||
import static org.openmetadata.common.utils.CommonUtil.findChildren;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
|
||||
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
|
||||
|
||||
@ -9,6 +10,7 @@ import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import org.openmetadata.schema.type.Column;
|
||||
import org.openmetadata.schema.type.Field;
|
||||
import org.openmetadata.schema.type.TagLabel;
|
||||
import org.openmetadata.service.exception.CatalogExceptionMessage;
|
||||
import org.openmetadata.service.util.FullyQualifiedName;
|
||||
@ -56,18 +58,21 @@ public final class ColumnUtil {
|
||||
|
||||
// Validate if a given column exists in the table
|
||||
public static void validateColumnFQN(List<Column> columns, String columnFQN) {
|
||||
boolean validColumn = false;
|
||||
for (Column column : columns) {
|
||||
if (column.getFullyQualifiedName().equals(columnFQN)) {
|
||||
validColumn = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!validColumn) {
|
||||
boolean exists = findChildren(columns, "getChildren", columnFQN);
|
||||
if (!exists) {
|
||||
throw new IllegalArgumentException(CatalogExceptionMessage.invalidColumnFQN(columnFQN));
|
||||
}
|
||||
}
|
||||
|
||||
// validate if a given field exists in the topic
|
||||
public static void validateFieldFQN(List<Field> fields, String fieldFQN) {
|
||||
boolean exists = findChildren(fields, "getChildren", fieldFQN);
|
||||
if (!exists) {
|
||||
throw new IllegalArgumentException(
|
||||
CatalogExceptionMessage.invalidFieldName("field", fieldFQN));
|
||||
}
|
||||
}
|
||||
|
||||
public static Set<String> getAllTags(Column column) {
|
||||
Set<String> tags = new HashSet<>();
|
||||
if (!listOrEmpty(column.getTags()).isEmpty()) {
|
||||
|
||||
@ -13,6 +13,12 @@
|
||||
|
||||
package org.openmetadata.service.jdbi3;
|
||||
|
||||
import static org.openmetadata.service.Entity.CONTAINER;
|
||||
import static org.openmetadata.service.Entity.DASHBOARD;
|
||||
import static org.openmetadata.service.Entity.DASHBOARD_DATA_MODEL;
|
||||
import static org.openmetadata.service.Entity.MLMODEL;
|
||||
import static org.openmetadata.service.Entity.TABLE;
|
||||
import static org.openmetadata.service.Entity.TOPIC;
|
||||
import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS;
|
||||
import static org.openmetadata.service.search.SearchClient.REMOVE_LINEAGE_SCRIPT;
|
||||
|
||||
@ -21,14 +27,17 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.jdbi.v3.sqlobject.transaction.Transaction;
|
||||
import org.openmetadata.common.utils.CommonUtil;
|
||||
import org.openmetadata.schema.ColumnsEntityInterface;
|
||||
import org.openmetadata.schema.api.lineage.AddLineage;
|
||||
import org.openmetadata.schema.entity.data.Container;
|
||||
import org.openmetadata.schema.entity.data.Dashboard;
|
||||
import org.openmetadata.schema.entity.data.DashboardDataModel;
|
||||
import org.openmetadata.schema.entity.data.MlModel;
|
||||
import org.openmetadata.schema.entity.data.Table;
|
||||
import org.openmetadata.schema.entity.data.Topic;
|
||||
import org.openmetadata.schema.type.ColumnLineage;
|
||||
import org.openmetadata.schema.type.Edge;
|
||||
import org.openmetadata.schema.type.EntityLineage;
|
||||
@ -37,17 +46,17 @@ import org.openmetadata.schema.type.Include;
|
||||
import org.openmetadata.schema.type.LineageDetails;
|
||||
import org.openmetadata.schema.type.Relationship;
|
||||
import org.openmetadata.service.Entity;
|
||||
import org.openmetadata.service.exception.CatalogExceptionMessage;
|
||||
import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord;
|
||||
import org.openmetadata.service.search.SearchClient;
|
||||
import org.openmetadata.service.search.models.IndexMapping;
|
||||
import org.openmetadata.service.util.FullyQualifiedName;
|
||||
import org.openmetadata.service.util.JsonUtils;
|
||||
|
||||
@Repository
|
||||
public class LineageRepository {
|
||||
private final CollectionDAO dao;
|
||||
|
||||
public SearchClient searchClient = Entity.getSearchRepository().getSearchClient();
|
||||
private static final SearchClient searchClient = Entity.getSearchRepository().getSearchClient();
|
||||
|
||||
public LineageRepository() {
|
||||
this.dao = Entity.getCollectionDAO();
|
||||
@ -173,41 +182,86 @@ public class LineageRepository {
|
||||
if (details == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<ColumnLineage> columnsLineage = details.getColumnsLineage();
|
||||
if (columnsLineage != null && !columnsLineage.isEmpty()) {
|
||||
if (areValidEntities(from, to)) {
|
||||
throw new IllegalArgumentException(
|
||||
"Column level lineage is only allowed between two tables or from table to dashboard.");
|
||||
}
|
||||
Table fromTable = dao.tableDAO().findEntityById(from.getId());
|
||||
ColumnsEntityInterface toTable = getToEntity(to);
|
||||
for (ColumnLineage columnLineage : columnsLineage) {
|
||||
for (String fromColumn : columnLineage.getFromColumns()) {
|
||||
// From column belongs to the fromNode
|
||||
if (fromColumn.startsWith(fromTable.getFullyQualifiedName())) {
|
||||
ColumnUtil.validateColumnFQN(fromTable.getColumns(), fromColumn);
|
||||
} else {
|
||||
Table otherTable =
|
||||
dao.tableDAO().findEntityByName(FullyQualifiedName.getTableFQN(fromColumn));
|
||||
ColumnUtil.validateColumnFQN(otherTable.getColumns(), fromColumn);
|
||||
}
|
||||
validateChildren(fromColumn, from);
|
||||
}
|
||||
ColumnUtil.validateColumnFQN(toTable.getColumns(), columnLineage.getToColumn());
|
||||
validateChildren(columnLineage.getToColumn(), to);
|
||||
}
|
||||
}
|
||||
return JsonUtils.pojoToJson(details);
|
||||
}
|
||||
|
||||
private ColumnsEntityInterface getToEntity(EntityReference from) {
|
||||
return from.getType().equals(Entity.TABLE)
|
||||
? dao.tableDAO().findEntityById(from.getId())
|
||||
: dao.dashboardDataModelDAO().findEntityById(from.getId());
|
||||
private void validateChildren(String columnFQN, EntityReference entityReference) {
|
||||
switch (entityReference.getType()) {
|
||||
case TABLE -> {
|
||||
Table table =
|
||||
Entity.getEntity(TABLE, entityReference.getId(), "columns", Include.NON_DELETED);
|
||||
ColumnUtil.validateColumnFQN(table.getColumns(), columnFQN);
|
||||
}
|
||||
case TOPIC -> {
|
||||
Topic topic =
|
||||
Entity.getEntity(TOPIC, entityReference.getId(), "messageSchema", Include.NON_DELETED);
|
||||
ColumnUtil.validateFieldFQN(topic.getMessageSchema().getSchemaFields(), columnFQN);
|
||||
}
|
||||
case CONTAINER -> {
|
||||
Container container =
|
||||
Entity.getEntity(CONTAINER, entityReference.getId(), "dataModel", Include.NON_DELETED);
|
||||
ColumnUtil.validateColumnFQN(container.getDataModel().getColumns(), columnFQN);
|
||||
}
|
||||
case DASHBOARD_DATA_MODEL -> {
|
||||
DashboardDataModel dashboardDataModel =
|
||||
Entity.getEntity(
|
||||
DASHBOARD_DATA_MODEL, entityReference.getId(), "columns", Include.NON_DELETED);
|
||||
ColumnUtil.validateColumnFQN(dashboardDataModel.getColumns(), columnFQN);
|
||||
}
|
||||
case DASHBOARD -> {
|
||||
Dashboard dashboard =
|
||||
Entity.getEntity(DASHBOARD, entityReference.getId(), "charts", Include.NON_DELETED);
|
||||
dashboard.getCharts().stream()
|
||||
.filter(c -> c.getFullyQualifiedName().equals(columnFQN))
|
||||
.findAny()
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalArgumentException(
|
||||
CatalogExceptionMessage.invalidFieldName("chart", columnFQN)));
|
||||
}
|
||||
case MLMODEL -> {
|
||||
MlModel mlModel =
|
||||
Entity.getEntity(MLMODEL, entityReference.getId(), "", Include.NON_DELETED);
|
||||
mlModel.getMlFeatures().stream()
|
||||
.filter(f -> f.getFullyQualifiedName().equals(columnFQN))
|
||||
.findAny()
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalArgumentException(
|
||||
CatalogExceptionMessage.invalidFieldName("feature", columnFQN)));
|
||||
}
|
||||
default -> throw new IllegalArgumentException(
|
||||
String.format("Unsupported Entity Type %s for lineage", entityReference.getType()));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean areValidEntities(EntityReference from, EntityReference to) {
|
||||
return !from.getType().equals(Entity.TABLE)
|
||||
|| !(to.getType().equals(Entity.TABLE) || to.getType().equals(Entity.DASHBOARD_DATA_MODEL));
|
||||
@Transaction
|
||||
public boolean deleteLineageByFQN(
|
||||
String fromEntity, String fromFQN, String toEntity, String toFQN) {
|
||||
EntityReference from =
|
||||
Entity.getEntityReferenceByName(fromEntity, fromFQN, Include.NON_DELETED);
|
||||
EntityReference to = Entity.getEntityReferenceByName(toEntity, toFQN, Include.NON_DELETED);
|
||||
// Finally, delete lineage relationship
|
||||
boolean result =
|
||||
dao.relationshipDAO()
|
||||
.delete(
|
||||
from.getId(),
|
||||
from.getType(),
|
||||
to.getId(),
|
||||
to.getType(),
|
||||
Relationship.UPSTREAM.ordinal())
|
||||
> 0;
|
||||
deleteLineageFromSearch(from, to);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Transaction
|
||||
@ -260,7 +314,7 @@ public class LineageRepository {
|
||||
getDownstreamLineage(primary.getId(), primary.getType(), lineage, downstreamDepth);
|
||||
|
||||
// Remove duplicate nodes
|
||||
lineage.withNodes(lineage.getNodes().stream().distinct().collect(Collectors.toList()));
|
||||
lineage.withNodes(lineage.getNodes().stream().distinct().toList());
|
||||
return lineage;
|
||||
}
|
||||
|
||||
|
||||
@ -273,6 +273,53 @@ public class LineageResource {
|
||||
return Response.status(Status.OK).build();
|
||||
}
|
||||
|
||||
@DELETE
|
||||
@Path("/{fromEntity}/name/{fromFQN}/{toEntity}/name/{toFQN}")
|
||||
@Operation(
|
||||
operationId = "deleteLineageEdgeByName",
|
||||
summary = "Delete a lineage edge by FQNs",
|
||||
description =
|
||||
"Delete a lineage edge with from entity as upstream node and to entity as downstream node.",
|
||||
responses = {
|
||||
@ApiResponse(responseCode = "200"),
|
||||
@ApiResponse(
|
||||
responseCode = "404",
|
||||
description = "Entity for instance {fromFQN} is not found")
|
||||
})
|
||||
public Response deleteLineageByName(
|
||||
@Context UriInfo uriInfo,
|
||||
@Context SecurityContext securityContext,
|
||||
@Parameter(
|
||||
description = "Entity type of upstream entity of the edge",
|
||||
required = true,
|
||||
schema = @Schema(type = "string", example = "table, report, metrics, or dashboard"))
|
||||
@PathParam("fromEntity")
|
||||
String fromEntity,
|
||||
@Parameter(description = "Entity FQN", required = true, schema = @Schema(type = "string"))
|
||||
@PathParam("fromFQN")
|
||||
String fromFQN,
|
||||
@Parameter(
|
||||
description = "Entity type for downstream entity of the edge",
|
||||
required = true,
|
||||
schema = @Schema(type = "string", example = "table, report, metrics, or dashboard"))
|
||||
@PathParam("toEntity")
|
||||
String toEntity,
|
||||
@Parameter(description = "Entity FQN", required = true, schema = @Schema(type = "string"))
|
||||
@PathParam("toFQN")
|
||||
String toFQN) {
|
||||
authorizer.authorize(
|
||||
securityContext,
|
||||
new OperationContext(LINEAGE_FIELD, MetadataOperation.EDIT_LINEAGE),
|
||||
new LineageResourceContext());
|
||||
boolean deleted = dao.deleteLineageByFQN(fromEntity, fromFQN, toEntity, toFQN);
|
||||
if (!deleted) {
|
||||
return Response.status(NOT_FOUND)
|
||||
.entity(new ErrorMessage(NOT_FOUND.getStatusCode(), "Lineage edge not found"))
|
||||
.build();
|
||||
}
|
||||
return Response.status(Status.OK).build();
|
||||
}
|
||||
|
||||
private EntityLineage addHref(UriInfo uriInfo, EntityLineage lineage) {
|
||||
Entity.withHref(uriInfo, lineage.getEntity());
|
||||
Entity.withHref(uriInfo, lineage.getNodes());
|
||||
|
||||
@ -21,6 +21,7 @@ import io.dropwizard.jersey.jackson.JacksonFeature;
|
||||
import io.dropwizard.testing.ConfigOverride;
|
||||
import io.dropwizard.testing.ResourceHelpers;
|
||||
import io.dropwizard.testing.junit5.DropwizardAppExtension;
|
||||
import java.net.URI;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import javax.ws.rs.client.Client;
|
||||
@ -216,6 +217,11 @@ public abstract class OpenMetadataApplicationTest {
|
||||
return client.target(format("http://localhost:%s/api/v1/%s", APP.getLocalPort(), collection));
|
||||
}
|
||||
|
||||
public static WebTarget getResourceAsURI(String collection) {
|
||||
return client.target(
|
||||
URI.create((format("http://localhost:%s/api/v1/%s", APP.getLocalPort(), collection))));
|
||||
}
|
||||
|
||||
public static WebTarget getConfigResource(String resource) {
|
||||
return client.target(
|
||||
format("http://localhost:%s/api/v1/system/config/%s", APP.getLocalPort(), resource));
|
||||
|
||||
@ -26,6 +26,7 @@ import static org.openmetadata.service.util.TestUtils.assertResponse;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
@ -42,14 +43,23 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.openmetadata.schema.EntityInterface;
|
||||
import org.openmetadata.schema.api.data.CreateContainer;
|
||||
import org.openmetadata.schema.api.data.CreateDashboard;
|
||||
import org.openmetadata.schema.api.data.CreateDashboardDataModel;
|
||||
import org.openmetadata.schema.api.data.CreateMlModel;
|
||||
import org.openmetadata.schema.api.data.CreateTable;
|
||||
import org.openmetadata.schema.api.data.CreateTopic;
|
||||
import org.openmetadata.schema.api.lineage.AddLineage;
|
||||
import org.openmetadata.schema.entity.data.Container;
|
||||
import org.openmetadata.schema.entity.data.Dashboard;
|
||||
import org.openmetadata.schema.entity.data.DashboardDataModel;
|
||||
import org.openmetadata.schema.entity.data.MlModel;
|
||||
import org.openmetadata.schema.entity.data.Table;
|
||||
import org.openmetadata.schema.entity.data.Topic;
|
||||
import org.openmetadata.schema.entity.teams.Role;
|
||||
import org.openmetadata.schema.entity.teams.User;
|
||||
import org.openmetadata.schema.type.ColumnLineage;
|
||||
import org.openmetadata.schema.type.ContainerDataModel;
|
||||
import org.openmetadata.schema.type.Edge;
|
||||
import org.openmetadata.schema.type.EntitiesEdge;
|
||||
import org.openmetadata.schema.type.EntityLineage;
|
||||
@ -58,11 +68,15 @@ import org.openmetadata.schema.type.LineageDetails;
|
||||
import org.openmetadata.schema.type.MetadataOperation;
|
||||
import org.openmetadata.service.Entity;
|
||||
import org.openmetadata.service.OpenMetadataApplicationTest;
|
||||
import org.openmetadata.service.resources.dashboards.DashboardResourceTest;
|
||||
import org.openmetadata.service.resources.databases.TableResourceTest;
|
||||
import org.openmetadata.service.resources.datamodels.DashboardDataModelResourceTest;
|
||||
import org.openmetadata.service.resources.mlmodels.MlModelResourceTest;
|
||||
import org.openmetadata.service.resources.storages.ContainerResourceTest;
|
||||
import org.openmetadata.service.resources.teams.RoleResource;
|
||||
import org.openmetadata.service.resources.teams.RoleResourceTest;
|
||||
import org.openmetadata.service.resources.teams.UserResourceTest;
|
||||
import org.openmetadata.service.resources.topics.TopicResourceTest;
|
||||
import org.openmetadata.service.util.TestUtils;
|
||||
|
||||
@Slf4j
|
||||
@ -71,10 +85,13 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
|
||||
public static final List<Table> TABLES = new ArrayList<>();
|
||||
public static final int TABLE_COUNT = 10;
|
||||
private static final String DATA_STEWARD_ROLE_NAME = "DataSteward";
|
||||
|
||||
private static DashboardDataModel DATA_MODEL;
|
||||
|
||||
private static Table TABLE_DATA_MODEL_LINEAGE;
|
||||
private static Topic TOPIC;
|
||||
private static Container CONTAINER;
|
||||
private static MlModel ML_MODEL;
|
||||
|
||||
private static Dashboard DASHBOARD;
|
||||
|
||||
@BeforeAll
|
||||
public static void setup(TestInfo test) throws IOException, URISyntaxException {
|
||||
@ -93,6 +110,25 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
|
||||
CreateTable createTable = tableResourceTest.createRequest(test, TABLE_COUNT);
|
||||
createTable.setColumns(createDashboardDataModel.getColumns());
|
||||
TABLE_DATA_MODEL_LINEAGE = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS);
|
||||
TopicResourceTest topicResourceTest = new TopicResourceTest();
|
||||
CreateTopic topicRequest =
|
||||
topicResourceTest
|
||||
.createRequest(test)
|
||||
.withMessageSchema(TopicResourceTest.SCHEMA.withSchemaFields(TopicResourceTest.fields));
|
||||
TOPIC = topicResourceTest.createEntity(topicRequest, ADMIN_AUTH_HEADERS);
|
||||
ContainerResourceTest containerResourceTest = new ContainerResourceTest();
|
||||
ContainerDataModel dataModel =
|
||||
new ContainerDataModel().withColumns(ContainerResourceTest.dataModelColumns);
|
||||
CreateContainer containerRequest =
|
||||
containerResourceTest.createRequest(test).withDataModel(dataModel);
|
||||
CONTAINER = containerResourceTest.createEntity(containerRequest, ADMIN_AUTH_HEADERS);
|
||||
MlModelResourceTest mlModelResourceTest = new MlModelResourceTest();
|
||||
CreateMlModel createMlModel =
|
||||
mlModelResourceTest.createRequest(test).withMlFeatures(MlModelResourceTest.ML_FEATURES);
|
||||
ML_MODEL = mlModelResourceTest.createEntity(createMlModel, ADMIN_AUTH_HEADERS);
|
||||
DashboardResourceTest dashboardResourceTest1 = new DashboardResourceTest();
|
||||
CreateDashboard createDashboard = dashboardResourceTest1.createRequest(test);
|
||||
DASHBOARD = dashboardResourceTest1.createEntity(createDashboard, ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
@Order(1)
|
||||
@ -300,26 +336,22 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
|
||||
details.getColumnsLineage().clear();
|
||||
details
|
||||
.getColumnsLineage()
|
||||
.add(new ColumnLineage().withFromColumns(List.of(t1c1FQN, t3c1FQN)).withToColumn(t2c1FQN));
|
||||
.add(new ColumnLineage().withFromColumns(List.of(t1c1FQN, t1c3FQN)).withToColumn(t2c1FQN));
|
||||
addEdge(TABLES.get(0), TABLES.get(1), details, ADMIN_AUTH_HEADERS);
|
||||
|
||||
// Finally, add detailed column level lineage
|
||||
details.getColumnsLineage().clear();
|
||||
List<ColumnLineage> lineage = details.getColumnsLineage();
|
||||
lineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(t1c1FQN, t3c1FQN)).withToColumn(t2c1FQN));
|
||||
lineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(t1c2FQN, t3c2FQN)).withToColumn(t2c2FQN));
|
||||
lineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(t1c3FQN, t3c3FQN)).withToColumn(t2c3FQN));
|
||||
lineage.add(new ColumnLineage().withFromColumns(List.of(t1c1FQN)).withToColumn(t2c1FQN));
|
||||
lineage.add(new ColumnLineage().withFromColumns(List.of(t1c2FQN)).withToColumn(t2c2FQN));
|
||||
lineage.add(new ColumnLineage().withFromColumns(List.of(t1c3FQN)).withToColumn(t2c3FQN));
|
||||
|
||||
addEdge(TABLES.get(0), TABLES.get(1), details, ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
@Order(4)
|
||||
@Test
|
||||
void putLineageFromDashboardDataModelToTable() throws HttpResponseException {
|
||||
|
||||
void putLineageFromEntityToEntity() throws HttpResponseException {
|
||||
// Add column lineage dashboard.d1 -> table.c1
|
||||
LineageDetails details = new LineageDetails();
|
||||
String d1c1FQN = DATA_MODEL.getColumns().get(0).getFullyQualifiedName();
|
||||
@ -333,13 +365,83 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
|
||||
lineage.add(new ColumnLineage().withFromColumns(List.of(c1c1FQN)).withToColumn(d1c1FQN));
|
||||
lineage.add(new ColumnLineage().withFromColumns(List.of(c1c2FQN)).withToColumn(d1c2FQN));
|
||||
lineage.add(new ColumnLineage().withFromColumns(List.of(c1c3FQN)).withToColumn(d1c3FQN));
|
||||
|
||||
addEdge(TABLE_DATA_MODEL_LINEAGE, DATA_MODEL, details, ADMIN_AUTH_HEADERS);
|
||||
|
||||
LineageDetails topicToTable = new LineageDetails();
|
||||
String f1FQN = TOPIC.getMessageSchema().getSchemaFields().get(0).getFullyQualifiedName();
|
||||
String f2FQN = TOPIC.getMessageSchema().getSchemaFields().get(0).getFullyQualifiedName();
|
||||
String f1t1 = TABLE_DATA_MODEL_LINEAGE.getColumns().get(0).getFullyQualifiedName();
|
||||
String f2t2 = TABLE_DATA_MODEL_LINEAGE.getColumns().get(1).getFullyQualifiedName();
|
||||
List<ColumnLineage> topicToTableLineage = topicToTable.getColumnsLineage();
|
||||
topicToTableLineage.add(new ColumnLineage().withFromColumns(List.of(f1FQN)).withToColumn(f1t1));
|
||||
topicToTableLineage.add(new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2t2));
|
||||
addEdge(TOPIC, TABLE_DATA_MODEL_LINEAGE, topicToTable, ADMIN_AUTH_HEADERS);
|
||||
String f3FQN = "test_non_existent_filed";
|
||||
topicToTableLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f3FQN)).withToColumn(d1c1FQN));
|
||||
assertResponse(
|
||||
() -> addEdge(DATA_MODEL, TABLE_DATA_MODEL_LINEAGE, details, ADMIN_AUTH_HEADERS),
|
||||
() -> addEdge(TOPIC, TABLE_DATA_MODEL_LINEAGE, topicToTable, ADMIN_AUTH_HEADERS),
|
||||
BAD_REQUEST,
|
||||
"Column level lineage is only allowed between two tables or from table to dashboard.");
|
||||
String.format("Invalid field name %s", f3FQN));
|
||||
|
||||
LineageDetails topicToContainer = new LineageDetails();
|
||||
String f1c1 = CONTAINER.getDataModel().getColumns().get(0).getFullyQualifiedName();
|
||||
String f2c2 = CONTAINER.getDataModel().getColumns().get(1).getFullyQualifiedName();
|
||||
List<ColumnLineage> topicToContainerLineage = topicToContainer.getColumnsLineage();
|
||||
topicToContainerLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f1FQN)).withToColumn(f1c1));
|
||||
topicToContainerLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2c2));
|
||||
addEdge(TOPIC, CONTAINER, topicToContainer, ADMIN_AUTH_HEADERS);
|
||||
String f2c3FQN = "test_non_existent_container_column";
|
||||
topicToContainerLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f2FQN)).withToColumn(f2c3FQN));
|
||||
assertResponse(
|
||||
() -> addEdge(TOPIC, CONTAINER, topicToContainer, ADMIN_AUTH_HEADERS),
|
||||
BAD_REQUEST,
|
||||
String.format("Invalid fully qualified column name %s", f2c3FQN));
|
||||
|
||||
LineageDetails containerToTable = new LineageDetails();
|
||||
List<ColumnLineage> containerToTableLineage = containerToTable.getColumnsLineage();
|
||||
containerToTableLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f1c1)).withToColumn(f1t1));
|
||||
containerToTableLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f2c2)).withToColumn(f2t2));
|
||||
addEdge(CONTAINER, TABLE_DATA_MODEL_LINEAGE, containerToTable, ADMIN_AUTH_HEADERS);
|
||||
|
||||
LineageDetails tableToMlModel = new LineageDetails();
|
||||
String m1f1 = ML_MODEL.getMlFeatures().get(0).getFullyQualifiedName();
|
||||
String m2f2 = ML_MODEL.getMlFeatures().get(1).getFullyQualifiedName();
|
||||
List<ColumnLineage> tableToMlModelLineage = tableToMlModel.getColumnsLineage();
|
||||
tableToMlModelLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f1t1)).withToColumn(m1f1));
|
||||
tableToMlModelLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(m2f2));
|
||||
addEdge(TABLE_DATA_MODEL_LINEAGE, ML_MODEL, tableToMlModel, ADMIN_AUTH_HEADERS);
|
||||
String m3f3 = "test_non_existent_feature";
|
||||
tableToMlModelLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(m3f3));
|
||||
assertResponse(
|
||||
() -> addEdge(TABLE_DATA_MODEL_LINEAGE, ML_MODEL, tableToMlModel, ADMIN_AUTH_HEADERS),
|
||||
BAD_REQUEST,
|
||||
String.format("Invalid feature name %s", m3f3));
|
||||
|
||||
LineageDetails tableToDashboard = new LineageDetails();
|
||||
String c1d1 = DASHBOARD.getCharts().get(0).getFullyQualifiedName();
|
||||
String c2d1 = DASHBOARD.getCharts().get(1).getFullyQualifiedName();
|
||||
|
||||
List<ColumnLineage> tableToDashboardLineage = tableToDashboard.getColumnsLineage();
|
||||
tableToDashboardLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f1t1)).withToColumn(c1d1));
|
||||
tableToDashboardLineage.add(
|
||||
new ColumnLineage().withFromColumns(List.of(f2t2)).withToColumn(c2d1));
|
||||
addEdge(TABLE_DATA_MODEL_LINEAGE, DASHBOARD, tableToDashboard, ADMIN_AUTH_HEADERS);
|
||||
|
||||
deleteEdgeByName(
|
||||
TOPIC.getEntityReference().getType(),
|
||||
TOPIC.getFullyQualifiedName(),
|
||||
CONTAINER.getEntityReference().getType(),
|
||||
CONTAINER.getFullyQualifiedName());
|
||||
}
|
||||
|
||||
@Order(5)
|
||||
@ -386,6 +488,11 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
|
||||
deleteEdge(from, to, ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
public void deleteEdgeByName(String fromEntity, String fromFQN, String toEntity, String toFQN)
|
||||
throws HttpResponseException {
|
||||
deleteLineageByName(fromEntity, fromFQN, toEntity, toFQN, ADMIN_AUTH_HEADERS);
|
||||
}
|
||||
|
||||
private void deleteEdge(Table from, Table to, Map<String, String> authHeaders)
|
||||
throws HttpResponseException {
|
||||
EntitiesEdge edge =
|
||||
@ -425,6 +532,21 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
|
||||
TestUtils.delete(target, authHeaders);
|
||||
}
|
||||
|
||||
public void deleteLineageByName(
|
||||
String fromEntity,
|
||||
String fromFQN,
|
||||
String toEntity,
|
||||
String toFQN,
|
||||
Map<String, String> authHeaders)
|
||||
throws HttpResponseException {
|
||||
WebTarget target =
|
||||
getResourceAsURI(
|
||||
String.format(
|
||||
"lineage/%s/name/%s/%s/name/%s",
|
||||
fromEntity, URLEncoder.encode(fromFQN), toEntity, URLEncoder.encode(toFQN)));
|
||||
TestUtils.delete(target, authHeaders);
|
||||
}
|
||||
|
||||
private void validateLineage(AddLineage addLineage, Map<String, String> authHeaders)
|
||||
throws HttpResponseException {
|
||||
EntityReference from = addLineage.getEdge().getFromEntity();
|
||||
|
||||
@ -465,12 +465,12 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel, CreateMlMod
|
||||
model.getUsageSummary());
|
||||
|
||||
// .../models?fields=mlFeatures,mlHyperParameters
|
||||
fields = "owner,dashboard,followers,tags,usageSummary";
|
||||
fields = "owner,followers,tags,usageSummary";
|
||||
model =
|
||||
byName
|
||||
? getEntityByName(model.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
|
||||
: getEntity(model.getId(), fields, ADMIN_AUTH_HEADERS);
|
||||
assertListNotNull(model.getDashboard(), model.getUsageSummary());
|
||||
assertListNotNull(model.getUsageSummary());
|
||||
// Checks for other owner, tags, and followers is done in the base class
|
||||
return model;
|
||||
}
|
||||
@ -482,7 +482,6 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel, CreateMlMod
|
||||
.withAlgorithm(ALGORITHM)
|
||||
.withMlFeatures(ML_FEATURES)
|
||||
.withMlHyperParameters(ML_HYPERPARAMS)
|
||||
.withDashboard(DASHBOARD.getFullyQualifiedName())
|
||||
.withService(MLFLOW_REFERENCE.getFullyQualifiedName());
|
||||
}
|
||||
|
||||
|
||||
@ -71,12 +71,22 @@ import org.openmetadata.service.util.TestUtils;
|
||||
|
||||
@Slf4j
|
||||
public class TopicResourceTest extends EntityResourceTest<Topic, CreateTopic> {
|
||||
private static final String SCHEMA_TEXT =
|
||||
public static final String SCHEMA_TEXT =
|
||||
"{\"namespace\":\"org.open-metadata.kafka\",\"name\":\"Customer\",\"type\":\"record\","
|
||||
+ "\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},"
|
||||
+ "{\"name\":\"email\",\"type\":\"string\"},{\"name\":\"address_line_1\",\"type\":\"string\"},{\"name\":\"address_line_2\",\"type\":\"string\"},"
|
||||
+ "{\"name\":\"post_code\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"}]}";
|
||||
private static final MessageSchema schema =
|
||||
public static final List<Field> fields =
|
||||
Arrays.asList(
|
||||
getField("id", FieldDataType.STRING, null),
|
||||
getField("first_name", FieldDataType.STRING, null),
|
||||
getField("last_name", FieldDataType.STRING, null),
|
||||
getField("email", FieldDataType.STRING, null),
|
||||
getField("address_line_1", FieldDataType.STRING, null),
|
||||
getField("address_line_2", FieldDataType.STRING, null),
|
||||
getField("post_code", FieldDataType.STRING, null),
|
||||
getField("county", FieldDataType.STRING, PERSONAL_DATA_TAG_LABEL));
|
||||
public static final MessageSchema SCHEMA =
|
||||
new MessageSchema().withSchemaText(SCHEMA_TEXT).withSchemaType(SchemaType.Avro);
|
||||
|
||||
public TopicResourceTest() {
|
||||
@ -171,16 +181,6 @@ public class TopicResourceTest extends EntityResourceTest<Topic, CreateTopic> {
|
||||
|
||||
@Test
|
||||
void put_topicSchemaFields_200_ok(TestInfo test) throws IOException {
|
||||
List<Field> fields =
|
||||
Arrays.asList(
|
||||
getField("id", FieldDataType.STRING, null),
|
||||
getField("first_name", FieldDataType.STRING, null),
|
||||
getField("last_name", FieldDataType.STRING, null),
|
||||
getField("email", FieldDataType.STRING, null),
|
||||
getField("address_line_1", FieldDataType.STRING, null),
|
||||
getField("address_line_2", FieldDataType.STRING, null),
|
||||
getField("post_code", FieldDataType.STRING, null),
|
||||
getField("county", FieldDataType.STRING, PERSONAL_DATA_TAG_LABEL));
|
||||
|
||||
CreateTopic createTopic =
|
||||
createRequest(test)
|
||||
@ -191,7 +191,7 @@ public class TopicResourceTest extends EntityResourceTest<Topic, CreateTopic> {
|
||||
.withReplicationFactor(1)
|
||||
.withRetentionTime(1.0)
|
||||
.withRetentionSize(1.0)
|
||||
.withMessageSchema(schema.withSchemaFields(fields))
|
||||
.withMessageSchema(SCHEMA.withSchemaFields(fields))
|
||||
.withCleanupPolicies(List.of(CleanupPolicy.COMPACT));
|
||||
|
||||
// Patch and update the topic
|
||||
@ -221,7 +221,7 @@ public class TopicResourceTest extends EntityResourceTest<Topic, CreateTopic> {
|
||||
.withReplicationFactor(1)
|
||||
.withRetentionTime(1.0)
|
||||
.withRetentionSize(1.0)
|
||||
.withMessageSchema(schema.withSchemaFields(fields))
|
||||
.withMessageSchema(SCHEMA.withSchemaFields(fields))
|
||||
.withCleanupPolicies(List.of(CleanupPolicy.COMPACT));
|
||||
|
||||
// Patch and update the topic
|
||||
@ -236,7 +236,7 @@ public class TopicResourceTest extends EntityResourceTest<Topic, CreateTopic> {
|
||||
.withReplicationFactor(2)
|
||||
.withRetentionTime(2.0)
|
||||
.withRetentionSize(2.0)
|
||||
.withMessageSchema(schema.withSchemaFields(fields))
|
||||
.withMessageSchema(SCHEMA.withSchemaFields(fields))
|
||||
.withCleanupPolicies(List.of(CleanupPolicy.DELETE));
|
||||
|
||||
ChangeDescription change = getChangeDescription(topic, MINOR_UPDATE);
|
||||
@ -284,7 +284,7 @@ public class TopicResourceTest extends EntityResourceTest<Topic, CreateTopic> {
|
||||
Field field =
|
||||
getField("first_name", FieldDataType.STRING, null)
|
||||
.withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL));
|
||||
create1.withMessageSchema(schema.withSchemaFields(List.of(field)));
|
||||
create1.withMessageSchema(SCHEMA.withSchemaFields(List.of(field)));
|
||||
assertResponse(
|
||||
() -> createEntity(create1, ADMIN_AUTH_HEADERS),
|
||||
BAD_REQUEST,
|
||||
@ -304,7 +304,7 @@ public class TopicResourceTest extends EntityResourceTest<Topic, CreateTopic> {
|
||||
getField("testNested", FieldDataType.STRING, null)
|
||||
.withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL));
|
||||
Field field1 = getField("test", FieldDataType.RECORD, null).withChildren(List.of(nestedField));
|
||||
create2.setMessageSchema(schema.withSchemaFields(List.of(field1)));
|
||||
create2.setMessageSchema(SCHEMA.withSchemaFields(List.of(field1)));
|
||||
assertResponse(
|
||||
() -> createEntity(create2, ADMIN_AUTH_HEADERS),
|
||||
BAD_REQUEST,
|
||||
|
||||
@ -44,6 +44,7 @@ export const LINEAGE_ITEMS = [
|
||||
fqn: 'sample_kafka.shop_products',
|
||||
entityType: 'Topic',
|
||||
searchIndex: SEARCH_INDEX.topics,
|
||||
columns: ['sample_kafka.shop_products.Shop.shop_id'],
|
||||
},
|
||||
{
|
||||
term: 'forecast_sales',
|
||||
@ -52,14 +53,16 @@ export const LINEAGE_ITEMS = [
|
||||
entityType: 'ML Model',
|
||||
fqn: 'mlflow_svc.forecast_sales',
|
||||
searchIndex: SEARCH_INDEX.mlmodels,
|
||||
columns: [],
|
||||
},
|
||||
{
|
||||
term: 'media',
|
||||
term: 'transactions',
|
||||
entity: DATA_ASSETS.containers,
|
||||
serviceName: 's3_storage_sample',
|
||||
entityType: 'Container',
|
||||
fqn: 's3_storage_sample.departments.media',
|
||||
fqn: 's3_storage_sample.transactions',
|
||||
searchIndex: SEARCH_INDEX.containers,
|
||||
columns: ['s3_storage_sample.transactions.transaction_id'],
|
||||
},
|
||||
{
|
||||
term: 'customers',
|
||||
@ -68,6 +71,7 @@ export const LINEAGE_ITEMS = [
|
||||
entityType: 'Dashboard',
|
||||
fqn: 'sample_looker.customers',
|
||||
searchIndex: SEARCH_INDEX.dashboards,
|
||||
columns: ['sample_looker.chart_1'],
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
|
||||
import { interceptURL, verifyResponseStatusCode } from '../../common/common';
|
||||
import { visitEntityDetailsPage } from '../../common/Utils/Entity';
|
||||
import { EntityType } from '../../constants/Entity.interface';
|
||||
import {
|
||||
LINEAGE_ITEMS,
|
||||
PIPELINE_ITEMS,
|
||||
@ -133,8 +134,8 @@ const verifyPipelineDataInDrawer = (
|
||||
const addPipelineBetweenNodes = (
|
||||
sourceEntity,
|
||||
targetEntity,
|
||||
pipelineItem,
|
||||
bVerifyPipeline
|
||||
pipelineItem?,
|
||||
bVerifyPipeline?: boolean
|
||||
) => {
|
||||
visitEntityDetailsPage({
|
||||
term: sourceEntity.term,
|
||||
@ -171,16 +172,18 @@ const expandCols = (nodeFqn, hasShowMore) => {
|
||||
}
|
||||
};
|
||||
|
||||
const addColumnLineage = (fromNode, toNode) => {
|
||||
const addColumnLineage = (fromNode, toNode, exitEditMode = true) => {
|
||||
interceptURL('PUT', '/api/v1/lineage', 'lineageApi');
|
||||
expandCols(fromNode.fqn, false);
|
||||
expandCols(toNode.fqn, true);
|
||||
expandCols(toNode.fqn, toNode.entityType === EntityType.Table);
|
||||
dragConnection(
|
||||
`column-${fromNode.columns[0]}`,
|
||||
`column-${toNode.columns[0]}`
|
||||
);
|
||||
verifyResponseStatusCode('@lineageApi', 200);
|
||||
cy.get('[data-testid="edit-lineage"]').click();
|
||||
if (exitEditMode) {
|
||||
cy.get('[data-testid="edit-lineage"]').click();
|
||||
}
|
||||
cy.get(
|
||||
`[data-testid="column-edge-${fromNode.columns[0]}-${toNode.columns[0]}"]`
|
||||
);
|
||||
@ -196,7 +199,7 @@ describe('Lineage verification', { tags: 'DataAssets' }, () => {
|
||||
visitEntityDetailsPage({
|
||||
term: entity.term,
|
||||
serviceName: entity.serviceName,
|
||||
entity: entity.entity,
|
||||
entity: entity.entity as EntityType,
|
||||
});
|
||||
|
||||
cy.get('[data-testid="lineage"]').click();
|
||||
@ -230,7 +233,7 @@ describe('Lineage verification', { tags: 'DataAssets' }, () => {
|
||||
visitEntityDetailsPage({
|
||||
term: entity.term,
|
||||
serviceName: entity.serviceName,
|
||||
entity: entity.entity,
|
||||
entity: entity.entity as EntityType,
|
||||
});
|
||||
|
||||
cy.get('[data-testid="lineage"]').click();
|
||||
@ -280,11 +283,16 @@ describe('Lineage verification', { tags: 'DataAssets' }, () => {
|
||||
|
||||
it('Add column lineage', () => {
|
||||
const sourceEntity = LINEAGE_ITEMS[0];
|
||||
const targetEntity = LINEAGE_ITEMS[1];
|
||||
addPipelineBetweenNodes(sourceEntity, targetEntity);
|
||||
// Add column lineage
|
||||
addColumnLineage(sourceEntity, targetEntity);
|
||||
cy.get('[data-testid="edit-lineage"]').click();
|
||||
deleteNode(targetEntity);
|
||||
for (let i = 1; i < LINEAGE_ITEMS.length; i++) {
|
||||
const targetEntity = LINEAGE_ITEMS[i];
|
||||
if (targetEntity.columns.length > 0) {
|
||||
addPipelineBetweenNodes(sourceEntity, targetEntity);
|
||||
// Add column lineage
|
||||
addColumnLineage(sourceEntity, targetEntity);
|
||||
cy.get('[data-testid="edit-lineage"]').click();
|
||||
deleteNode(targetEntity);
|
||||
cy.goToHomePage();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@ -12,11 +12,16 @@
|
||||
*/
|
||||
import { Button } from 'antd';
|
||||
import classNames from 'classnames';
|
||||
import { isEmpty } from 'lodash';
|
||||
import React, { Fragment } from 'react';
|
||||
import { Handle, HandleProps, HandleType, Position } from 'reactflow';
|
||||
import { ReactComponent as MinusIcon } from '../../../assets/svg/control-minus.svg';
|
||||
import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-outlined.svg';
|
||||
import { EntityLineageNodeType } from '../../../enums/entity.enum';
|
||||
import { Column, TestSuite } from '../../../generated/entity/data/table';
|
||||
import { formTwoDigitNumber } from '../../../utils/CommonUtils';
|
||||
import { getEntityName } from '../../../utils/EntityUtils';
|
||||
import { getConstraintIcon } from '../../../utils/TableUtils';
|
||||
import { EdgeTypeEnum } from './EntityLineage.interface';
|
||||
|
||||
export const getHandleByType = (
|
||||
@ -111,3 +116,65 @@ export const getCollapseHandle = (
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export const getTestSuiteSummary = (testSuite?: TestSuite) => {
|
||||
if (isEmpty(testSuite)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="d-flex justify-between">
|
||||
<div className="profiler-item green" data-testid="test-passed">
|
||||
<div className="font-medium" data-testid="test-passed-value">
|
||||
{formTwoDigitNumber(testSuite?.summary?.success ?? 0)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="profiler-item amber" data-testid="test-aborted">
|
||||
<div className="font-medium" data-testid="test-aborted-value">
|
||||
{formTwoDigitNumber(testSuite?.summary?.aborted ?? 0)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="profiler-item red" data-testid="test-failed">
|
||||
<div className="font-medium" data-testid="test-failed-value">
|
||||
{formTwoDigitNumber(testSuite?.summary?.failed ?? 0)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export const getColumnContent = (
|
||||
column: Column,
|
||||
isColumnTraced: boolean,
|
||||
isConnectable: boolean,
|
||||
onColumnClick: (column: string) => void
|
||||
) => {
|
||||
const { fullyQualifiedName } = column;
|
||||
|
||||
return (
|
||||
<div
|
||||
className={classNames(
|
||||
'custom-node-column-container',
|
||||
isColumnTraced
|
||||
? 'custom-node-header-tracing'
|
||||
: 'custom-node-column-lineage-normal bg-white'
|
||||
)}
|
||||
data-testid={`column-${fullyQualifiedName}`}
|
||||
key={fullyQualifiedName}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onColumnClick(fullyQualifiedName ?? '');
|
||||
}}>
|
||||
{getColumnHandle(
|
||||
EntityLineageNodeType.DEFAULT,
|
||||
isConnectable,
|
||||
'lineage-column-node-handle',
|
||||
fullyQualifiedName
|
||||
)}
|
||||
{getConstraintIcon({
|
||||
constraint: column.constraint,
|
||||
})}
|
||||
<p className="p-xss">{getEntityName(column)}</p>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@ -11,13 +11,10 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { DownOutlined, SearchOutlined, UpOutlined } from '@ant-design/icons';
|
||||
import Icon from '@ant-design/icons/lib/components/Icon';
|
||||
import { Button, Input } from 'antd';
|
||||
import { Button } from 'antd';
|
||||
import classNames from 'classnames';
|
||||
import { isEmpty } from 'lodash';
|
||||
import React, { memo, useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import {
|
||||
getIncomers,
|
||||
getOutgoers,
|
||||
@ -26,37 +23,26 @@ import {
|
||||
Position,
|
||||
} from 'reactflow';
|
||||
import { ReactComponent as IconTimesCircle } from '../../../assets/svg/ic-times-circle.svg';
|
||||
import { BORDER_COLOR } from '../../../constants/constants';
|
||||
import { useLineageProvider } from '../../../context/LineageProvider/LineageProvider';
|
||||
import { EntityLineageNodeType, EntityType } from '../../../enums/entity.enum';
|
||||
import { formTwoDigitNumber } from '../../../utils/CommonUtils';
|
||||
import { EntityLineageNodeType } from '../../../enums/entity.enum';
|
||||
import { checkUpstreamDownstream } from '../../../utils/EntityLineageUtils';
|
||||
import { getEntityName } from '../../../utils/EntityUtils';
|
||||
import { getConstraintIcon, getEntityIcon } from '../../../utils/TableUtils';
|
||||
import './custom-node.less';
|
||||
import {
|
||||
getCollapseHandle,
|
||||
getColumnHandle,
|
||||
getExpandHandle,
|
||||
} from './CustomNode.utils';
|
||||
import { getCollapseHandle, getExpandHandle } from './CustomNode.utils';
|
||||
import './entity-lineage.style.less';
|
||||
import { EdgeTypeEnum, ModifiedColumn } from './EntityLineage.interface';
|
||||
import { EdgeTypeEnum } from './EntityLineage.interface';
|
||||
import LineageNodeLabelV1 from './LineageNodeLabelV1';
|
||||
import NodeChildren from './NodeChildren/NodeChildren.component';
|
||||
|
||||
const CustomNodeV1 = (props: NodeProps) => {
|
||||
const { t } = useTranslation();
|
||||
const { data, type, isConnectable } = props;
|
||||
|
||||
const {
|
||||
isEditMode,
|
||||
expandedNodes,
|
||||
tracedNodes,
|
||||
tracedColumns,
|
||||
selectedNode,
|
||||
nodes,
|
||||
edges,
|
||||
upstreamDownstreamData,
|
||||
onColumnClick,
|
||||
onNodeCollapse,
|
||||
removeNodeHandler,
|
||||
loadChildNodesHandler,
|
||||
@ -65,11 +51,8 @@ const CustomNodeV1 = (props: NodeProps) => {
|
||||
const { label, isNewNode, node = {}, isRootNode } = data;
|
||||
const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type;
|
||||
const isSelected = selectedNode === node;
|
||||
const { columns, id, testSuite, lineage, fullyQualifiedName } = node;
|
||||
const [searchValue, setSearchValue] = useState('');
|
||||
const [filteredColumns, setFilteredColumns] = useState<ModifiedColumn[]>([]);
|
||||
const [showAllColumns, setShowAllColumns] = useState(false);
|
||||
const [isExpanded, setIsExpanded] = useState<boolean>(false);
|
||||
const { id, lineage, fullyQualifiedName } = node;
|
||||
|
||||
const [isTraced, setIsTraced] = useState<boolean>(false);
|
||||
|
||||
const getActiveNode = useCallback(
|
||||
@ -105,14 +88,6 @@ const CustomNodeV1 = (props: NodeProps) => {
|
||||
};
|
||||
}, [id, nodes, edges, hasUpstream, hasDownstream]);
|
||||
|
||||
const supportsColumns = useMemo(() => {
|
||||
if (node && node.entityType === EntityType.TABLE) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}, [node.id]);
|
||||
|
||||
const { isUpstreamNode, isDownstreamNode } = useMemo(() => {
|
||||
return {
|
||||
isUpstreamNode: upstreamDownstreamData.upstreamNodes.some(
|
||||
@ -124,40 +99,6 @@ const CustomNodeV1 = (props: NodeProps) => {
|
||||
};
|
||||
}, [fullyQualifiedName, upstreamDownstreamData]);
|
||||
|
||||
const handleSearchChange = useCallback(
|
||||
(e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
e.stopPropagation();
|
||||
const value = e.target.value;
|
||||
setSearchValue(value);
|
||||
|
||||
if (value.trim() === '') {
|
||||
// If search value is empty, show all columns or the default number of columns
|
||||
const filterColumns = Object.values(columns || {}) as ModifiedColumn[];
|
||||
setFilteredColumns(
|
||||
showAllColumns ? filterColumns : filterColumns.slice(0, 5)
|
||||
);
|
||||
} else {
|
||||
// Filter columns based on search value
|
||||
const filtered = (
|
||||
Object.values(columns || {}) as ModifiedColumn[]
|
||||
).filter((column) =>
|
||||
getEntityName(column).toLowerCase().includes(value.toLowerCase())
|
||||
);
|
||||
setFilteredColumns(filtered);
|
||||
}
|
||||
},
|
||||
[columns]
|
||||
);
|
||||
|
||||
const handleShowMoreClick = useCallback(
|
||||
(e: React.MouseEvent<HTMLButtonElement>) => {
|
||||
e.stopPropagation();
|
||||
setShowAllColumns(true);
|
||||
setFilteredColumns(Object.values(columns ?? []));
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
const onExpand = useCallback(
|
||||
(direction: EdgeTypeEnum) => {
|
||||
loadChildNodesHandler(node, direction);
|
||||
@ -307,30 +248,10 @@ const CustomNodeV1 = (props: NodeProps) => {
|
||||
loadChildNodesHandler,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
setIsExpanded(expandedNodes.includes(id));
|
||||
}, [expandedNodes, id]);
|
||||
|
||||
useEffect(() => {
|
||||
setIsTraced(tracedNodes.includes(id));
|
||||
}, [tracedNodes, id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isExpanded) {
|
||||
setShowAllColumns(false);
|
||||
} else if (!isEmpty(columns) && Object.values(columns).length < 5) {
|
||||
setShowAllColumns(true);
|
||||
}
|
||||
}, [isEditMode, isExpanded, columns]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isEmpty(columns)) {
|
||||
setFilteredColumns(
|
||||
Object.values(columns).slice(0, 5) as ModifiedColumn[]
|
||||
);
|
||||
}
|
||||
}, [columns]);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={classNames(
|
||||
@ -342,123 +263,7 @@ const CustomNodeV1 = (props: NodeProps) => {
|
||||
{getHandle()}
|
||||
<div className="lineage-node-content">
|
||||
<div className="label-container bg-white">{nodeLabel}</div>
|
||||
{supportsColumns && (
|
||||
<div className="column-container bg-grey-1 p-sm p-y-xs">
|
||||
<div className="d-flex justify-between items-center">
|
||||
<Button
|
||||
className="flex-center text-primary rounded-4 p-xss"
|
||||
data-testid="expand-cols-btn"
|
||||
icon={
|
||||
<div className="d-flex w-5 h-5 m-r-xs text-base-color">
|
||||
{getEntityIcon(node.entityType || '')}
|
||||
</div>
|
||||
}
|
||||
type="text"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setIsExpanded((prevIsExpanded: boolean) => !prevIsExpanded);
|
||||
}}>
|
||||
{t('label.column-plural')}
|
||||
{isExpanded ? (
|
||||
<UpOutlined style={{ fontSize: '12px' }} />
|
||||
) : (
|
||||
<DownOutlined style={{ fontSize: '12px' }} />
|
||||
)}
|
||||
</Button>
|
||||
{node.entityType === EntityType.TABLE && testSuite && (
|
||||
<div className="d-flex justify-between">
|
||||
<div
|
||||
className="profiler-item green"
|
||||
data-testid="test-passed">
|
||||
<div
|
||||
className="font-medium"
|
||||
data-testid="test-passed-value">
|
||||
{formTwoDigitNumber(testSuite?.summary?.success ?? 0)}
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
className="profiler-item amber"
|
||||
data-testid="test-aborted">
|
||||
<div
|
||||
className="font-medium"
|
||||
data-testid="test-aborted-value">
|
||||
{formTwoDigitNumber(testSuite?.summary?.aborted ?? 0)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="profiler-item red" data-testid="test-failed">
|
||||
<div
|
||||
className="font-medium"
|
||||
data-testid="test-failed-value">
|
||||
{formTwoDigitNumber(testSuite?.summary?.failed ?? 0)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isExpanded && (
|
||||
<div className="m-t-md">
|
||||
<div className="search-box">
|
||||
<Input
|
||||
placeholder={t('label.search-entity', {
|
||||
entity: t('label.column-plural'),
|
||||
})}
|
||||
suffix={<SearchOutlined color={BORDER_COLOR} />}
|
||||
value={searchValue}
|
||||
onChange={handleSearchChange}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<section className="m-t-md" id="table-columns">
|
||||
<div className="border rounded-4">
|
||||
{filteredColumns.map((column) => {
|
||||
const isColumnTraced = tracedColumns.includes(
|
||||
column.fullyQualifiedName ?? ''
|
||||
);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={classNames(
|
||||
'custom-node-column-container',
|
||||
isColumnTraced
|
||||
? 'custom-node-header-tracing'
|
||||
: 'custom-node-column-lineage-normal bg-white'
|
||||
)}
|
||||
data-testid={`column-${column.fullyQualifiedName}`}
|
||||
key={column.fullyQualifiedName}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onColumnClick(column.fullyQualifiedName ?? '');
|
||||
}}>
|
||||
{getColumnHandle(
|
||||
column.type,
|
||||
isConnectable,
|
||||
'lineage-column-node-handle',
|
||||
column.fullyQualifiedName
|
||||
)}
|
||||
{getConstraintIcon({ constraint: column.constraint })}
|
||||
<p className="p-xss">{getEntityName(column)}</p>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{!showAllColumns && (
|
||||
<Button
|
||||
className="m-t-xs text-primary"
|
||||
data-testid="show-more-cols-btn"
|
||||
type="text"
|
||||
onClick={handleShowMoreClick}>
|
||||
{t('label.show-more-entity', {
|
||||
entity: t('label.column-plural'),
|
||||
})}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<NodeChildren isConnectable={isConnectable} node={node} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@ -0,0 +1,263 @@
|
||||
/*
|
||||
* Copyright 2024 Collate.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { DownOutlined, SearchOutlined, UpOutlined } from '@ant-design/icons';
|
||||
import { Button, Collapse, Input } from 'antd';
|
||||
import { isEmpty } from 'lodash';
|
||||
import React, { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { BORDER_COLOR } from '../../../../constants/constants';
|
||||
import { LINEAGE_COLUMN_NODE_SUPPORTED } from '../../../../constants/Lineage.constants';
|
||||
import { useLineageProvider } from '../../../../context/LineageProvider/LineageProvider';
|
||||
import { EntityType } from '../../../../enums/entity.enum';
|
||||
import { Container } from '../../../../generated/entity/data/container';
|
||||
import { Dashboard } from '../../../../generated/entity/data/dashboard';
|
||||
import { Mlmodel } from '../../../../generated/entity/data/mlmodel';
|
||||
import { Column, Table } from '../../../../generated/entity/data/table';
|
||||
import { Topic } from '../../../../generated/entity/data/topic';
|
||||
import { getEntityName } from '../../../../utils/EntityUtils';
|
||||
import { getEntityIcon } from '../../../../utils/TableUtils';
|
||||
import { getColumnContent, getTestSuiteSummary } from '../CustomNode.utils';
|
||||
import { EntityChildren, NodeChildrenProps } from './NodeChildren.interface';
|
||||
|
||||
const NodeChildren = ({ node, isConnectable }: NodeChildrenProps) => {
|
||||
const { t } = useTranslation();
|
||||
const { Panel } = Collapse;
|
||||
const { isEditMode, tracedColumns, expandedNodes, onColumnClick } =
|
||||
useLineageProvider();
|
||||
const { entityType, id } = node;
|
||||
const [searchValue, setSearchValue] = useState('');
|
||||
const [filteredColumns, setFilteredColumns] = useState<EntityChildren>([]);
|
||||
const [showAllColumns, setShowAllColumns] = useState(false);
|
||||
const [isExpanded, setIsExpanded] = useState<boolean>(false);
|
||||
|
||||
const supportsColumns = useMemo(() => {
|
||||
return (
|
||||
node &&
|
||||
LINEAGE_COLUMN_NODE_SUPPORTED.includes(node.entityType as EntityType)
|
||||
);
|
||||
}, [node.id]);
|
||||
|
||||
const { children, childrenHeading } = useMemo(() => {
|
||||
const entityMappings: Record<
|
||||
string,
|
||||
{ data: EntityChildren; label: string }
|
||||
> = {
|
||||
[EntityType.TABLE]: {
|
||||
data: (node as Table).columns ?? [],
|
||||
label: t('label.column-plural'),
|
||||
},
|
||||
[EntityType.DASHBOARD]: {
|
||||
data: (node as Dashboard).charts ?? [],
|
||||
label: t('label.chart-plural'),
|
||||
},
|
||||
[EntityType.MLMODEL]: {
|
||||
data: (node as Mlmodel).mlFeatures ?? [],
|
||||
label: t('label.feature-plural'),
|
||||
},
|
||||
[EntityType.DASHBOARD_DATA_MODEL]: {
|
||||
data: (node as Table).columns ?? [],
|
||||
label: t('label.column-plural'),
|
||||
},
|
||||
[EntityType.CONTAINER]: {
|
||||
data: (node as Container).dataModel?.columns ?? [],
|
||||
label: t('label.column-plural'),
|
||||
},
|
||||
[EntityType.TOPIC]: {
|
||||
data: (node as Topic).messageSchema?.schemaFields ?? [],
|
||||
label: t('label.field-plural'),
|
||||
},
|
||||
};
|
||||
|
||||
const { data, label } = entityMappings[node.entityType as EntityType] || {
|
||||
data: [],
|
||||
label: '',
|
||||
};
|
||||
|
||||
return {
|
||||
children: data,
|
||||
childrenHeading: label,
|
||||
};
|
||||
}, [node.id]);
|
||||
|
||||
const handleSearchChange = useCallback(
|
||||
(e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
e.stopPropagation();
|
||||
const value = e.target.value;
|
||||
setSearchValue(value);
|
||||
|
||||
if (value.trim() === '') {
|
||||
// If search value is empty, show all columns or the default number of columns
|
||||
const filterColumns = Object.values(children ?? {});
|
||||
setFilteredColumns(
|
||||
showAllColumns ? filterColumns : filterColumns.slice(0, 5)
|
||||
);
|
||||
} else {
|
||||
// Filter columns based on search value
|
||||
const filtered = Object.values(children ?? {}).filter((column) =>
|
||||
getEntityName(column).toLowerCase().includes(value.toLowerCase())
|
||||
);
|
||||
setFilteredColumns(filtered);
|
||||
}
|
||||
},
|
||||
[children]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
setIsExpanded(expandedNodes.includes(id ?? ''));
|
||||
}, [expandedNodes, id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isEmpty(children)) {
|
||||
setFilteredColumns(children.slice(0, 5));
|
||||
}
|
||||
}, [children]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isExpanded) {
|
||||
setShowAllColumns(false);
|
||||
} else if (!isEmpty(children) && Object.values(children).length < 5) {
|
||||
setShowAllColumns(true);
|
||||
}
|
||||
}, [isEditMode, isExpanded, children]);
|
||||
|
||||
const handleShowMoreClick = useCallback(
|
||||
(e: React.MouseEvent<HTMLButtonElement>) => {
|
||||
e.stopPropagation();
|
||||
setShowAllColumns(true);
|
||||
setFilteredColumns(children ?? []);
|
||||
},
|
||||
[children]
|
||||
);
|
||||
|
||||
const renderRecord = useCallback(
|
||||
(record: Column) => {
|
||||
return (
|
||||
<Collapse
|
||||
destroyInactivePanel
|
||||
defaultActiveKey={record.fullyQualifiedName}>
|
||||
<Panel
|
||||
header={getEntityName(record)}
|
||||
key={record.fullyQualifiedName ?? ''}>
|
||||
{record?.children?.map((child) => {
|
||||
const { fullyQualifiedName, dataType } = child;
|
||||
if (['RECORD', 'STRUCT'].includes(dataType)) {
|
||||
return renderRecord(child);
|
||||
} else {
|
||||
const isColumnTraced = tracedColumns.includes(
|
||||
fullyQualifiedName ?? ''
|
||||
);
|
||||
|
||||
return getColumnContent(
|
||||
child,
|
||||
isColumnTraced,
|
||||
isConnectable,
|
||||
onColumnClick
|
||||
);
|
||||
}
|
||||
})}
|
||||
</Panel>
|
||||
</Collapse>
|
||||
);
|
||||
},
|
||||
[isConnectable, tracedColumns]
|
||||
);
|
||||
|
||||
const renderColumnsData = useCallback(
|
||||
(column: Column) => {
|
||||
const { fullyQualifiedName, dataType } = column;
|
||||
if (['RECORD', 'STRUCT'].includes(dataType)) {
|
||||
return renderRecord(column);
|
||||
} else {
|
||||
const isColumnTraced = tracedColumns.includes(fullyQualifiedName ?? '');
|
||||
|
||||
return getColumnContent(
|
||||
column,
|
||||
isColumnTraced,
|
||||
isConnectable,
|
||||
onColumnClick
|
||||
);
|
||||
}
|
||||
},
|
||||
[isConnectable, tracedColumns]
|
||||
);
|
||||
|
||||
if (supportsColumns) {
|
||||
return (
|
||||
<div className="column-container bg-grey-1 p-sm p-y-xs">
|
||||
<div className="d-flex justify-between items-center">
|
||||
<Button
|
||||
className="flex-center text-primary rounded-4 p-xss"
|
||||
data-testid="expand-cols-btn"
|
||||
icon={
|
||||
<div className="d-flex w-5 h-5 m-r-xs text-base-color">
|
||||
{getEntityIcon(node.entityType ?? '')}
|
||||
</div>
|
||||
}
|
||||
type="text"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setIsExpanded((prevIsExpanded: boolean) => !prevIsExpanded);
|
||||
}}>
|
||||
{childrenHeading}
|
||||
{isExpanded ? (
|
||||
<UpOutlined style={{ fontSize: '12px' }} />
|
||||
) : (
|
||||
<DownOutlined style={{ fontSize: '12px' }} />
|
||||
)}
|
||||
</Button>
|
||||
{entityType === EntityType.TABLE &&
|
||||
getTestSuiteSummary((node as Table).testSuite)}
|
||||
</div>
|
||||
|
||||
{isExpanded && (
|
||||
<div className="m-t-md">
|
||||
<div className="search-box">
|
||||
<Input
|
||||
placeholder={t('label.search-entity', {
|
||||
entity: childrenHeading,
|
||||
})}
|
||||
suffix={<SearchOutlined color={BORDER_COLOR} />}
|
||||
value={searchValue}
|
||||
onChange={handleSearchChange}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<section className="m-t-md" id="table-columns">
|
||||
<div className="border rounded-4 overflow-hidden">
|
||||
{filteredColumns.map((column) =>
|
||||
renderColumnsData(column as Column)
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{!showAllColumns && (
|
||||
<Button
|
||||
className="m-t-xs text-primary"
|
||||
data-testid="show-more-cols-btn"
|
||||
type="text"
|
||||
onClick={handleShowMoreClick}>
|
||||
{t('label.show-more-entity', {
|
||||
entity: childrenHeading,
|
||||
})}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export default NodeChildren;
|
||||
@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Copyright 2024 Collate.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { MlFeature } from '../../../../generated/entity/data/mlmodel';
|
||||
import { Column } from '../../../../generated/entity/data/table';
|
||||
import { Field } from '../../../../generated/entity/data/topic';
|
||||
import { EntityReference } from '../../../../generated/entity/type';
|
||||
import { SearchedDataProps } from '../../../SearchedData/SearchedData.interface';
|
||||
|
||||
export interface NodeChildrenProps {
|
||||
node: SearchedDataProps['data'][number]['_source'];
|
||||
isConnectable: boolean;
|
||||
}
|
||||
|
||||
export type EntityChildren =
|
||||
| Column[]
|
||||
| EntityReference[]
|
||||
| MlFeature[]
|
||||
| Field[];
|
||||
@ -14,7 +14,6 @@
|
||||
import { Select } from 'antd';
|
||||
import { AxiosError } from 'axios';
|
||||
import { capitalize, debounce } from 'lodash';
|
||||
import { FormattedTableData } from 'Models';
|
||||
import React, {
|
||||
FC,
|
||||
HTMLAttributes,
|
||||
@ -28,15 +27,14 @@ import { PAGE_SIZE } from '../../../constants/constants';
|
||||
import { EntityType, FqnPart } from '../../../enums/entity.enum';
|
||||
import { SearchIndex } from '../../../enums/search.enum';
|
||||
import { EntityReference } from '../../../generated/entity/type';
|
||||
import { SearchSourceAlias } from '../../../interface/search.interface';
|
||||
import { searchData } from '../../../rest/miscAPI';
|
||||
import { formatDataResponse } from '../../../utils/APIUtils';
|
||||
import { getPartialNameFromTableFQN } from '../../../utils/CommonUtils';
|
||||
import { getEntityNodeIcon } from '../../../utils/EntityLineageUtils';
|
||||
import { getEntityName } from '../../../utils/EntityUtils';
|
||||
import serviceUtilClassBase from '../../../utils/ServiceUtilClassBase';
|
||||
import { showErrorToast } from '../../../utils/ToastUtils';
|
||||
import { ExploreSearchIndex } from '../../Explore/ExplorePage.interface';
|
||||
import { SourceType } from '../../SearchedData/SearchedData.interface';
|
||||
import './node-suggestion.less';
|
||||
|
||||
interface EntitySuggestionProps extends HTMLAttributes<HTMLDivElement> {
|
||||
@ -50,7 +48,7 @@ const NodeSuggestions: FC<EntitySuggestionProps> = ({
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
const [data, setData] = useState<Array<FormattedTableData>>([]);
|
||||
const [data, setData] = useState<Array<SourceType>>([]);
|
||||
|
||||
const [searchValue, setSearchValue] = useState<string>('');
|
||||
|
||||
@ -78,7 +76,8 @@ const NodeSuggestions: FC<EntitySuggestionProps> = ({
|
||||
'',
|
||||
(entityType as ExploreSearchIndex) ?? SearchIndex.TABLE
|
||||
);
|
||||
setData(formatDataResponse(data.data.hits.hits));
|
||||
const sources = data.data.hits.hits.map((hit) => hit._source);
|
||||
setData(sources);
|
||||
} catch (error) {
|
||||
showErrorToast(
|
||||
error as AxiosError,
|
||||
@ -133,16 +132,14 @@ const NodeSuggestions: FC<EntitySuggestionProps> = ({
|
||||
alt={entity.serviceType}
|
||||
className="m-r-xs"
|
||||
height="16px"
|
||||
src={serviceUtilClassBase.getServiceTypeLogo(
|
||||
entity as SearchSourceAlias
|
||||
)}
|
||||
src={serviceUtilClassBase.getServiceTypeLogo(entity)}
|
||||
width="16px"
|
||||
/>
|
||||
<div className="flex-1 text-left">
|
||||
{entity.entityType === EntityType.TABLE && (
|
||||
<p className="d-block text-xs text-grey-muted w-max-400 truncate p-b-xss">
|
||||
{getSuggestionLabelHeading(
|
||||
entity.fullyQualifiedName,
|
||||
entity.fullyQualifiedName ?? '',
|
||||
entity.entityType as string
|
||||
)}
|
||||
</p>
|
||||
|
||||
@ -59,6 +59,25 @@
|
||||
border: 1px solid @red-5;
|
||||
}
|
||||
}
|
||||
.column-container {
|
||||
.ant-collapse-arrow {
|
||||
margin-right: 0 !important;
|
||||
}
|
||||
.ant-collapse {
|
||||
border-radius: 0;
|
||||
}
|
||||
.ant-collapse-header {
|
||||
padding: 4px 12px !important;
|
||||
font-size: 12px;
|
||||
}
|
||||
.ant-collapse-content-box {
|
||||
padding: 4px !important;
|
||||
}
|
||||
.ant-collapse-item {
|
||||
border: none !important;
|
||||
border-radius: 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.react-flow__node-default,
|
||||
@ -151,13 +170,13 @@
|
||||
}
|
||||
|
||||
.react-flow .lineage-node-handle {
|
||||
width: 35px;
|
||||
min-width: 35px;
|
||||
height: 35px;
|
||||
border-radius: 50%;
|
||||
border-color: @lineage-border;
|
||||
background: @white;
|
||||
top: 43px; // Need to show handles on top half
|
||||
width: 35px !important;
|
||||
min-width: 35px !important;
|
||||
height: 35px !important;
|
||||
border-radius: 50% !important;
|
||||
border-color: @lineage-border !important;
|
||||
background: @white !important;
|
||||
top: 43px !important; // Need to show handles on top half
|
||||
svg {
|
||||
color: @text-grey-muted;
|
||||
}
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
}
|
||||
|
||||
width: 46px;
|
||||
box-sizing: content-box;
|
||||
box-sizing: content-box !important;
|
||||
border: @global-border;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
@ -68,11 +68,15 @@
|
||||
}
|
||||
}
|
||||
.custom-node-column-lineage-normal {
|
||||
border-bottom: 1px solid @border-color;
|
||||
border-top: 1px solid @border-color;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
position: relative;
|
||||
font-size: 12px;
|
||||
&:first-child {
|
||||
border-top: none;
|
||||
}
|
||||
}
|
||||
|
||||
.custom-node {
|
||||
@ -134,6 +138,12 @@
|
||||
&.active {
|
||||
background-color: @primary-color;
|
||||
}
|
||||
&.active:hover {
|
||||
background-color: @primary-color;
|
||||
}
|
||||
&.active:focus {
|
||||
background-color: @primary-color;
|
||||
}
|
||||
}
|
||||
|
||||
.custom-lineage-heading {
|
||||
@ -185,12 +195,6 @@
|
||||
}
|
||||
|
||||
// lineage
|
||||
.lineage-card {
|
||||
> .ant-card-body {
|
||||
padding: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.lineage-card {
|
||||
height: calc(100vh - 240px);
|
||||
|
||||
@ -223,7 +227,7 @@
|
||||
}
|
||||
|
||||
.lineage-node-remove-btn {
|
||||
position: absolute;
|
||||
position: absolute !important;
|
||||
top: -20px;
|
||||
right: -20px;
|
||||
cursor: pointer;
|
||||
|
||||
@ -118,7 +118,7 @@ const Lineage = ({
|
||||
// considerably. So added an init state for showing loader.
|
||||
return (
|
||||
<Card
|
||||
className="lineage-card card-body-full w-auto border-none"
|
||||
className="lineage-card card-body-full w-auto border-none card-padding-0"
|
||||
data-testid="lineage-details">
|
||||
{isFullScreen && (
|
||||
<TitleBreadcrumb className="p-md" titleLinks={breadcrumbs} />
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
|
||||
import { t } from 'i18next';
|
||||
import { ElementLoadingState } from '../components/Entity/EntityLineage/EntityLineage.interface';
|
||||
import { EntityType } from '../enums/entity.enum';
|
||||
import { SearchIndex } from '../enums/search.enum';
|
||||
import { Source } from '../generated/type/entityLineage';
|
||||
|
||||
@ -86,3 +87,12 @@ export const LINEAGE_SOURCE: { [key in Source]: string } = {
|
||||
[Source.OpenLineage]: 'OpenLineage',
|
||||
[Source.ExternalTableLineage]: 'External Table Lineage',
|
||||
};
|
||||
|
||||
export const LINEAGE_COLUMN_NODE_SUPPORTED = [
|
||||
EntityType.TABLE,
|
||||
EntityType.DASHBOARD,
|
||||
EntityType.MLMODEL,
|
||||
EntityType.DASHBOARD_DATA_MODEL,
|
||||
EntityType.CONTAINER,
|
||||
EntityType.TOPIC,
|
||||
];
|
||||
|
||||
@ -67,6 +67,7 @@ import {
|
||||
import { AddLineage } from '../../generated/api/lineage/addLineage';
|
||||
import { PipelineStatus } from '../../generated/entity/data/pipeline';
|
||||
import {
|
||||
ColumnLineage,
|
||||
EntityReference,
|
||||
LineageDetails,
|
||||
} from '../../generated/type/entityLineage';
|
||||
@ -404,7 +405,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
|
||||
};
|
||||
|
||||
const removeNodeHandler = useCallback(
|
||||
(node: Node | NodeProps) => {
|
||||
async (node: Node | NodeProps) => {
|
||||
if (!entityLineage) {
|
||||
return;
|
||||
}
|
||||
@ -414,18 +415,57 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
|
||||
(item) => item?.data?.isColumnLineage === false
|
||||
);
|
||||
|
||||
// Get edges connected to selected node
|
||||
const edgesToRemove = getConnectedEdges([node as Node], nodeEdges);
|
||||
edgesToRemove.forEach((edge) => {
|
||||
removeEdgeHandler(edge, true);
|
||||
|
||||
const filteredEdges: EdgeDetails[] = [];
|
||||
|
||||
await Promise.all(
|
||||
edgesToRemove.map(async (edge) => {
|
||||
const { data } = edge;
|
||||
const edgeData: EdgeData = {
|
||||
fromEntity: data.edge.fromEntity.type,
|
||||
fromId: data.edge.fromEntity.id,
|
||||
toEntity: data.edge.toEntity.type,
|
||||
toId: data.edge.toEntity.id,
|
||||
};
|
||||
|
||||
await removeLineageHandler(edgeData);
|
||||
|
||||
filteredEdges.push(
|
||||
...(entityLineage.edges ?? []).filter(
|
||||
(item) =>
|
||||
!(
|
||||
item.fromEntity.id === edgeData.fromId &&
|
||||
item.toEntity.id === edgeData.toId
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
setEdges((prev) => {
|
||||
return prev.filter(
|
||||
(item) =>
|
||||
!(
|
||||
item.source === edgeData.fromId &&
|
||||
item.target === edgeData.toId
|
||||
)
|
||||
);
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
const updatedNodes = (entityLineage.nodes ?? []).filter(
|
||||
(previousNode) => previousNode.id !== node.id
|
||||
);
|
||||
|
||||
setNodes((prev) => {
|
||||
return prev.filter((previousNode) => previousNode.id !== node.id);
|
||||
});
|
||||
|
||||
setEntityLineage((prev) => {
|
||||
setUpdatedEntityLineage(() => {
|
||||
return {
|
||||
...prev,
|
||||
nodes: (prev.nodes ?? []).filter(
|
||||
(previousNode) => previousNode.id !== node.id
|
||||
),
|
||||
...entityLineage,
|
||||
edges: filteredEdges,
|
||||
nodes: updatedNodes,
|
||||
};
|
||||
});
|
||||
|
||||
@ -636,8 +676,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
|
||||
targetNode.data.node
|
||||
);
|
||||
|
||||
let updatedColumns: ColumnLineage[] = [];
|
||||
|
||||
if (columnConnection && currentEdge) {
|
||||
const updatedColumns = getUpdatedColumnsFromEdge(params, currentEdge);
|
||||
updatedColumns = getUpdatedColumnsFromEdge(params, currentEdge);
|
||||
|
||||
const lineageDetails: LineageDetails = {
|
||||
pipeline: currentEdge.pipeline,
|
||||
@ -647,7 +689,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
|
||||
};
|
||||
lineageDetails.columnsLineage = updatedColumns;
|
||||
newEdgeWithoutFqn.edge.lineageDetails = lineageDetails;
|
||||
currentEdge.columns = updatedColumns; // update current edge with new columns
|
||||
}
|
||||
|
||||
addLineageHandler(newEdgeWithoutFqn)
|
||||
@ -668,6 +709,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
|
||||
? [...(entityLineage.edges ?? []), newEdgeWithFqn.edge]
|
||||
: entityLineage.edges ?? [];
|
||||
|
||||
if (currentEdge && columnConnection) {
|
||||
currentEdge.columns = updatedColumns; // update current edge with new columns
|
||||
}
|
||||
|
||||
setEntityLineage((pre) => {
|
||||
const newData = {
|
||||
...pre,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user