diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index ace0e077048..ef0016589fc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -50,6 +50,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Function; import javax.json.JsonPatch; import javax.ws.rs.core.Response; import lombok.Getter; @@ -196,6 +197,7 @@ public class LineageRepository { addServiceLineage(fromEntity, toEntity, lineageDetails, childRelationExists); addDomainLineage(fromEntity, toEntity, lineageDetails, childRelationExists); + addDataProductsLineage(fromEntity, toEntity, lineageDetails, childRelationExists); } private void addServiceLineage( @@ -203,44 +205,17 @@ public class LineageRepository { EntityInterface toEntity, LineageDetails entityLineageDetails, boolean childRelationExists) { - boolean addService = - Entity.entityHasField(fromEntity.getEntityReference().getType(), FIELD_SERVICE) - && Entity.entityHasField(toEntity.getEntityReference().getType(), FIELD_SERVICE); + if (!shouldAddServiceLineage(fromEntity, toEntity)) { + return; + } // Add Service Level Lineage - if (addService && fromEntity.getService() != null && toEntity.getService() != null) { - EntityReference fromService = fromEntity.getService(); - EntityReference toService = toEntity.getService(); - if (Boolean.FALSE.equals(fromService.getId().equals(toService.getId()))) { - CollectionDAO.EntityRelationshipObject serviceRelation = - dao.relationshipDAO() - .getRecord(fromService.getId(), toService.getId(), Relationship.UPSTREAM.ordinal()); - LineageDetails serviceLineageDetails; - if (serviceRelation != null) { - serviceLineageDetails = - JsonUtils.readValue(serviceRelation.getJson(), LineageDetails.class); - if (!childRelationExists) { - serviceLineageDetails.withAssetEdges(serviceLineageDetails.getAssetEdges() + 1); - } - } else { - serviceLineageDetails = - new LineageDetails() - .withCreatedAt(entityLineageDetails.getCreatedAt()) - .withCreatedBy(entityLineageDetails.getCreatedBy()) - .withUpdatedAt(entityLineageDetails.getUpdatedAt()) - .withUpdatedBy(entityLineageDetails.getUpdatedBy()) - .withSource(LineageDetails.Source.CHILD_ASSETS) - .withAssetEdges(1); - } - dao.relationshipDAO() - .insert( - fromService.getId(), - toService.getId(), - fromService.getType(), - toService.getType(), - Relationship.UPSTREAM.ordinal(), - JsonUtils.pojoToJson(serviceLineageDetails)); - addLineageToSearch(fromService, toService, serviceLineageDetails); - } + EntityReference fromService = fromEntity.getService(); + EntityReference toService = toEntity.getService(); + if (Boolean.FALSE.equals(fromService.getId().equals(toService.getId()))) { + LineageDetails serviceLineageDetails = + getOrCreateLineageDetails( + fromService.getId(), toService.getId(), entityLineageDetails, childRelationExists); + insertLineage(fromService, toService, serviceLineageDetails); } } @@ -249,47 +224,106 @@ public class LineageRepository { EntityInterface toEntity, LineageDetails entityLineageDetails, boolean childRelationExists) { - boolean addDomain = - Entity.entityHasField(fromEntity.getEntityReference().getType(), FIELD_DOMAIN) - && Entity.entityHasField(toEntity.getEntityReference().getType(), FIELD_DOMAIN); - // Add Service Level Lineage - if (addDomain && fromEntity.getDomain() != null && toEntity.getDomain() != null) { - EntityReference fromDomain = fromEntity.getDomain(); - EntityReference toDomain = toEntity.getDomain(); - if (Boolean.FALSE.equals(fromDomain.getId().equals(toDomain.getId()))) { - CollectionDAO.EntityRelationshipObject serviceRelation = - dao.relationshipDAO() - .getRecord(fromDomain.getId(), toDomain.getId(), Relationship.UPSTREAM.ordinal()); - LineageDetails domainLineageDetails; - if (serviceRelation != null) { - domainLineageDetails = - JsonUtils.readValue(serviceRelation.getJson(), LineageDetails.class); - if (!childRelationExists) { - domainLineageDetails.withAssetEdges(domainLineageDetails.getAssetEdges() + 1); - } - } else { - domainLineageDetails = - new LineageDetails() - .withCreatedAt(entityLineageDetails.getCreatedAt()) - .withCreatedBy(entityLineageDetails.getCreatedBy()) - .withUpdatedAt(entityLineageDetails.getUpdatedAt()) - .withUpdatedBy(entityLineageDetails.getUpdatedBy()) - .withSource(LineageDetails.Source.CHILD_ASSETS) - .withAssetEdges(1); + + if (!shouldAddDomainsLineage(fromEntity, toEntity)) { + return; + } + + EntityReference fromDomain = fromEntity.getDomain(); + EntityReference toDomain = toEntity.getDomain(); + if (Boolean.FALSE.equals(fromDomain.getId().equals(toDomain.getId()))) { + LineageDetails domainLineageDetails = + getOrCreateLineageDetails( + fromDomain.getId(), toDomain.getId(), entityLineageDetails, childRelationExists); + insertLineage(fromDomain, toDomain, domainLineageDetails); + } + } + + private void addDataProductsLineage( + EntityInterface fromEntity, + EntityInterface toEntity, + LineageDetails entityLineageDetails, + boolean childRelationExists) { + + if (!shouldAddDataProductLineage(fromEntity, toEntity)) { + return; + } + + for (EntityReference fromEntityRef : fromEntity.getDataProducts()) { + for (EntityReference toEntityRef : toEntity.getDataProducts()) { + if (!fromEntityRef.getId().equals(toEntityRef.getId())) { + LineageDetails dataProductsLineageDetails = + getOrCreateLineageDetails( + fromEntityRef.getId(), + toEntityRef.getId(), + entityLineageDetails, + childRelationExists); + + insertLineage(fromEntityRef, toEntityRef, dataProductsLineageDetails); } - dao.relationshipDAO() - .insert( - fromDomain.getId(), - toDomain.getId(), - fromDomain.getType(), - toDomain.getType(), - Relationship.UPSTREAM.ordinal(), - JsonUtils.pojoToJson(domainLineageDetails)); - addLineageToSearch(fromDomain, toDomain, domainLineageDetails); } } } + private boolean shouldAddDataProductLineage( + EntityInterface fromEntity, EntityInterface toEntity) { + return Entity.entityHasField(fromEntity.getEntityReference().getType(), FIELD_DATA_PRODUCTS) + && Entity.entityHasField(toEntity.getEntityReference().getType(), FIELD_DATA_PRODUCTS) + && !nullOrEmpty(fromEntity.getDataProducts()) + && !nullOrEmpty(toEntity.getDataProducts()); + } + + private boolean shouldAddDomainsLineage(EntityInterface fromEntity, EntityInterface toEntity) { + return Entity.entityHasField(fromEntity.getEntityReference().getType(), FIELD_DOMAIN) + && Entity.entityHasField(toEntity.getEntityReference().getType(), FIELD_DOMAIN) + && fromEntity.getDomain() != null + && toEntity.getDomain() != null; + } + + private boolean shouldAddServiceLineage(EntityInterface fromEntity, EntityInterface toEntity) { + return Entity.entityHasField(fromEntity.getEntityReference().getType(), FIELD_SERVICE) + && Entity.entityHasField(toEntity.getEntityReference().getType(), FIELD_SERVICE) + && fromEntity.getService() != null + && toEntity.getService() != null; + } + + private LineageDetails getOrCreateLineageDetails( + UUID fromId, UUID toId, LineageDetails entityLineageDetails, boolean childRelationExists) { + + CollectionDAO.EntityRelationshipObject existingRelation = + dao.relationshipDAO().getRecord(fromId, toId, Relationship.UPSTREAM.ordinal()); + + if (existingRelation != null) { + LineageDetails lineageDetails = + JsonUtils.readValue(existingRelation.getJson(), LineageDetails.class); + if (!childRelationExists) { + lineageDetails.withAssetEdges(lineageDetails.getAssetEdges() + 1); + } + return lineageDetails; + } + + return new LineageDetails() + .withCreatedAt(entityLineageDetails.getCreatedAt()) + .withCreatedBy(entityLineageDetails.getCreatedBy()) + .withUpdatedAt(entityLineageDetails.getUpdatedAt()) + .withUpdatedBy(entityLineageDetails.getUpdatedBy()) + .withSource(LineageDetails.Source.CHILD_ASSETS) + .withAssetEdges(1); + } + + private void insertLineage( + EntityReference from, EntityReference to, LineageDetails lineageDetails) { + dao.relationshipDAO() + .insert( + from.getId(), + to.getId(), + from.getType(), + to.getType(), + Relationship.UPSTREAM.ordinal(), + JsonUtils.pojoToJson(lineageDetails)); + addLineageToSearch(from, to, lineageDetails); + } + private String getExtendedLineageFields(boolean service, boolean domain, boolean dataProducts) { StringBuilder fieldsBuilder = new StringBuilder(); @@ -535,6 +569,12 @@ public class LineageRepository { JsonNode fromEntity = entityMap.getOrDefault(fromEntityId, null); JsonNode toEntity = entityMap.getOrDefault(toEntityId, null); + if (fromEntity == null || toEntity == null) { + LOG.error( + "Entity not found for IDs: fromEntityId={}, toEntityId={}", fromEntityId, toEntityId); + return; + } + Map baseRow = new HashMap<>(); baseRow.put("fromEntityFQN", getText(fromEntity, FIELD_FULLY_QUALIFIED_NAME)); baseRow.put("fromServiceName", getText(fromEntity.path(FIELD_SERVICE), FIELD_NAME)); @@ -906,99 +946,88 @@ public class LineageRepository { } private void cleanUpExtendedLineage(EntityReference from, EntityReference to) { - boolean addService = - Entity.entityHasField(from.getType(), FIELD_SERVICE) - && Entity.entityHasField(to.getType(), FIELD_SERVICE); - boolean addDomain = - Entity.entityHasField(from.getType(), FIELD_DOMAIN) - && Entity.entityHasField(to.getType(), FIELD_DOMAIN); + boolean addService = hasField(from, FIELD_SERVICE) && hasField(to, FIELD_SERVICE); + boolean addDomain = hasField(from, FIELD_DOMAIN) && hasField(to, FIELD_DOMAIN); boolean addDataProduct = - Entity.entityHasField(from.getType(), FIELD_DATA_PRODUCTS) - && Entity.entityHasField(to.getType(), FIELD_DATA_PRODUCTS); + hasField(from, FIELD_DATA_PRODUCTS) && hasField(to, FIELD_DATA_PRODUCTS); String fields = getExtendedLineageFields(addService, addDomain, addDataProduct); EntityInterface fromEntity = Entity.getEntity(from.getType(), from.getId(), fields, Include.ALL); EntityInterface toEntity = Entity.getEntity(to.getType(), to.getId(), fields, Include.ALL); - cleanUpServiceLineage(fromEntity, toEntity); - cleanUpDomainLineage(fromEntity, toEntity); + cleanUpLineage(fromEntity, toEntity, FIELD_SERVICE, EntityInterface::getService); + cleanUpLineage(fromEntity, toEntity, FIELD_DOMAIN, EntityInterface::getDomain); + cleanUpLineageForDataProducts( + fromEntity, toEntity, FIELD_DATA_PRODUCTS, EntityInterface::getDataProducts); } - private void cleanUpServiceLineage(EntityInterface fromEntity, EntityInterface toEntity) { - boolean hasServiceField = - Entity.entityHasField(fromEntity.getEntityReference().getType(), FIELD_SERVICE) - && Entity.entityHasField(toEntity.getEntityReference().getType(), FIELD_SERVICE); - if (hasServiceField && fromEntity.getService() != null && toEntity.getService() != null) { - EntityReference fromService = fromEntity.getService(); - EntityReference toService = toEntity.getService(); - CollectionDAO.EntityRelationshipObject serviceRelation = - dao.relationshipDAO() - .getRecord(fromService.getId(), toService.getId(), Relationship.UPSTREAM.ordinal()); - LineageDetails serviceLineageDetails; - if (serviceRelation != null) { - serviceLineageDetails = - JsonUtils.readValue(serviceRelation.getJson(), LineageDetails.class); - if (serviceLineageDetails.getAssetEdges() - 1 < 1) { - dao.relationshipDAO() - .delete( - fromService.getId(), - fromService.getType(), - toService.getId(), - toService.getType(), - Relationship.UPSTREAM.ordinal()); - deleteLineageFromSearch(fromService, toService, serviceLineageDetails); - } else { - serviceLineageDetails.withAssetEdges(serviceLineageDetails.getAssetEdges() - 1); - dao.relationshipDAO() - .insert( - fromService.getId(), - toService.getId(), - fromService.getType(), - toService.getType(), - Relationship.UPSTREAM.ordinal(), - JsonUtils.pojoToJson(serviceLineageDetails)); - addLineageToSearch(fromService, toService, serviceLineageDetails); - } + private boolean hasField(EntityReference entity, String field) { + return Entity.entityHasField(entity.getType(), field); + } + + private void cleanUpLineage( + EntityInterface fromEntity, + EntityInterface toEntity, + String field, + Function getter) { + boolean hasField = + hasField(fromEntity.getEntityReference(), field) + && hasField(toEntity.getEntityReference(), field); + if (!hasField) return; + + EntityReference fromRef = getter.apply(fromEntity); + EntityReference toRef = getter.apply(toEntity); + processExtendedLineageCleanup(fromRef, toRef); + } + + private void cleanUpLineageForDataProducts( + EntityInterface fromEntity, + EntityInterface toEntity, + String field, + Function> getter) { + boolean hasField = + hasField(fromEntity.getEntityReference(), field) + && hasField(toEntity.getEntityReference(), field); + if (!hasField) return; + + for (EntityReference fromRef : getter.apply(fromEntity)) { + for (EntityReference toRef : getter.apply(toEntity)) { + processExtendedLineageCleanup(fromRef, toRef); } } } - private void cleanUpDomainLineage(EntityInterface fromEntity, EntityInterface toEntity) { - boolean hasDomainField = - Entity.entityHasField(fromEntity.getEntityReference().getType(), FIELD_DOMAIN) - && Entity.entityHasField(toEntity.getEntityReference().getType(), FIELD_DOMAIN); - if (hasDomainField && fromEntity.getDomain() != null && toEntity.getDomain() != null) { - EntityReference fromDomain = fromEntity.getDomain(); - EntityReference toDomain = toEntity.getDomain(); - CollectionDAO.EntityRelationshipObject domainRelation = - dao.relationshipDAO() - .getRecord(fromDomain.getId(), toDomain.getId(), Relationship.UPSTREAM.ordinal()); - LineageDetails domainLineageDetails; - if (domainRelation != null) { - domainLineageDetails = JsonUtils.readValue(domainRelation.getJson(), LineageDetails.class); - if (domainLineageDetails.getAssetEdges() - 1 < 1) { - dao.relationshipDAO() - .delete( - fromDomain.getId(), - fromDomain.getType(), - toDomain.getId(), - toDomain.getType(), - Relationship.UPSTREAM.ordinal()); - deleteLineageFromSearch(fromDomain, toDomain, domainLineageDetails); - } else { - domainLineageDetails.withAssetEdges(domainLineageDetails.getAssetEdges() - 1); - dao.relationshipDAO() - .insert( - fromDomain.getId(), - toDomain.getId(), - fromDomain.getType(), - toDomain.getType(), - Relationship.UPSTREAM.ordinal(), - JsonUtils.pojoToJson(domainLineageDetails)); - addLineageToSearch(fromDomain, toDomain, domainLineageDetails); - } - } + private void processExtendedLineageCleanup(EntityReference fromRef, EntityReference toRef) { + if (fromRef == null || toRef == null) return; + + CollectionDAO.EntityRelationshipObject relation = + dao.relationshipDAO() + .getRecord(fromRef.getId(), toRef.getId(), Relationship.UPSTREAM.ordinal()); + + if (relation == null) return; + + LineageDetails lineageDetails = JsonUtils.readValue(relation.getJson(), LineageDetails.class); + if (lineageDetails.getAssetEdges() - 1 < 1) { + dao.relationshipDAO() + .delete( + fromRef.getId(), + fromRef.getType(), + toRef.getId(), + toRef.getType(), + Relationship.UPSTREAM.ordinal()); + deleteLineageFromSearch(fromRef, toRef, lineageDetails); + } else { + lineageDetails.withAssetEdges(lineageDetails.getAssetEdges() - 1); + dao.relationshipDAO() + .insert( + fromRef.getId(), + toRef.getId(), + fromRef.getType(), + toRef.getType(), + Relationship.UPSTREAM.ordinal(), + JsonUtils.pojoToJson(lineageDetails)); + addLineageToSearch(fromRef, toRef, lineageDetails); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java index d074a8d8091..e78e78bc7b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java @@ -3,6 +3,7 @@ package org.openmetadata.service.migration.mysql.v170; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.createServiceCharts; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNonNullColumn; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNullColumn; +import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationForDataProductsLineage; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationForDomainLineage; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationServiceLineage; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateDataInsightsApplication; @@ -32,6 +33,7 @@ public class Migration extends MigrationProcessImpl { runLineageMigrationForNonNullColumn(handle); runMigrationServiceLineage(handle); runMigrationForDomainLineage(handle); + runMigrationForDataProductsLineage(handle); // DI createServiceCharts(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java index 76e5fe3ad2d..b1b91f7ac0f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java @@ -3,6 +3,7 @@ package org.openmetadata.service.migration.postgres.v170; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.createServiceCharts; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNonNullColumn; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNullColumn; +import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationForDataProductsLineage; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationForDomainLineage; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationServiceLineage; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateDataInsightsApplication; @@ -32,6 +33,7 @@ public class Migration extends MigrationProcessImpl { runLineageMigrationForNonNullColumn(handle); runMigrationServiceLineage(handle); runMigrationForDomainLineage(handle); + runMigrationForDataProductsLineage(handle); // DI createServiceCharts(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java index 8cf99b0e7a8..f9d03b94f75 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java @@ -19,12 +19,14 @@ import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.LineChart; import org.openmetadata.schema.dataInsight.custom.LineChartMetric; +import org.openmetadata.schema.entity.domains.DataProduct; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.policies.Policy; import org.openmetadata.schema.entity.policies.accessControl.Rule; import org.openmetadata.schema.governance.workflows.WorkflowConfiguration; import org.openmetadata.schema.governance.workflows.WorkflowDefinition; import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.LineageDetails; import org.openmetadata.schema.type.MetadataOperation; @@ -36,6 +38,7 @@ import org.openmetadata.service.governance.workflows.flowable.MainWorkflow; import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository; +import org.openmetadata.service.jdbi3.DataProductRepository; import org.openmetadata.service.jdbi3.DomainRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.PolicyRepository; @@ -63,7 +66,7 @@ public class MigrationUtil { private MigrationUtil() {} - public static final String DOMAIN_LINEAGE = + public static final String DOMAIN_AND_PRODUCTS_LINEAGE = "select count(*) from entity_relationship where fromId in (select toId from entity_relationship where fromId = '%s' and relation = 10) AND toId in (select toId from entity_relationship where fromId = '%s' and relation = 10) and relation = 13"; public static final String SERVICE_ENTITY_MIGRATION = @@ -383,36 +386,11 @@ public class MigrationUtil { public static void runMigrationForDomainLineage(Handle handle) { try { - LOG.info("MIGRATION 1.7.0 - STARTING MIGRATION FOR DOMAIN LINEAGE"); List allDomains = getAllDomains(); for (Domain fromDomain : allDomains) { for (Domain toDomain : allDomains) { - if (fromDomain.getId().equals(toDomain.getId())) { - continue; - } - String sql = - String.format( - DOMAIN_LINEAGE, fromDomain.getId().toString(), toDomain.getId().toString()); - int count = handle.createQuery(sql).mapTo(Integer.class).one(); - if (count > 0) { - LineageDetails domainLineageDetails = - new LineageDetails() - .withCreatedAt(System.currentTimeMillis()) - .withUpdatedAt(System.currentTimeMillis()) - .withCreatedBy(ADMIN_USER_NAME) - .withUpdatedBy(ADMIN_USER_NAME) - .withSource(LineageDetails.Source.CHILD_ASSETS) - .withAssetEdges(count); - Entity.getCollectionDAO() - .relationshipDAO() - .insert( - fromDomain.getId(), - toDomain.getId(), - fromDomain.getEntityReference().getType(), - toDomain.getEntityReference().getType(), - Relationship.UPSTREAM.ordinal(), - JsonUtils.pojoToJson(domainLineageDetails)); - } + insertDomainAndDataProductLineage( + handle, fromDomain.getEntityReference(), toDomain.getEntityReference()); } } @@ -423,6 +401,57 @@ public class MigrationUtil { } } + public static void runMigrationForDataProductsLineage(Handle handle) { + try { + List allDataProducts = getAllDataProducts(); + for (DataProduct fromDataProduct : allDataProducts) { + for (DataProduct toDataProduct : allDataProducts) { + insertDomainAndDataProductLineage( + handle, fromDataProduct.getEntityReference(), toDataProduct.getEntityReference()); + } + } + + } catch (Exception ex) { + LOG.error( + "Error while updating null json rows with createdAt, createdBy, updatedAt and updatedBy for lineage.", + ex); + } + } + + private static void insertDomainAndDataProductLineage( + Handle handle, EntityReference fromRef, EntityReference toRef) { + LOG.info( + "MIGRATION 1.7.0 - STARTING MIGRATION FOR DOMAIN/DATA_PRODUCT LINEAGE, FROM: {} TO: {}", + fromRef.getFullyQualifiedName(), + toRef.getFullyQualifiedName()); + if (fromRef.getId().equals(toRef.getId())) { + return; + } + String sql = + String.format( + DOMAIN_AND_PRODUCTS_LINEAGE, fromRef.getId().toString(), toRef.getId().toString()); + int count = handle.createQuery(sql).mapTo(Integer.class).one(); + if (count > 0) { + LineageDetails domainLineageDetails = + new LineageDetails() + .withCreatedAt(System.currentTimeMillis()) + .withUpdatedAt(System.currentTimeMillis()) + .withCreatedBy(ADMIN_USER_NAME) + .withUpdatedBy(ADMIN_USER_NAME) + .withSource(LineageDetails.Source.CHILD_ASSETS) + .withAssetEdges(count); + Entity.getCollectionDAO() + .relationshipDAO() + .insert( + fromRef.getId(), + toRef.getId(), + fromRef.getType(), + toRef.getType(), + Relationship.UPSTREAM.ordinal(), + JsonUtils.pojoToJson(domainLineageDetails)); + } + } + public static void runMigrationServiceLineage(Handle handle) { try { List allServices = getAllServicesForLineage(); @@ -441,7 +470,10 @@ public class MigrationUtil { private static void insertServiceLineageDetails( Handle handle, ServiceEntityInterface fromService, ServiceEntityInterface toService) { try { - LOG.info("MIGRATION 1.7.0 - STARTING MIGRATION FOR SERVICES LINEAGE"); + LOG.info( + "MIGRATION 1.7.0 - STARTING MIGRATION FOR SERVICES LINEAGE , FROM: {} TO: {}", + fromService.getFullyQualifiedName(), + toService.getFullyQualifiedName()); if (fromService.getId().equals(toService.getId()) && fromService @@ -510,6 +542,12 @@ public class MigrationUtil { return repository.listAll(repository.getFields("id"), new ListFilter(Include.ALL)); } + private static List getAllDataProducts() { + DataProductRepository repository = + (DataProductRepository) Entity.getEntityRepository(Entity.DATA_PRODUCT); + return repository.listAll(repository.getFields("id"), new ListFilter(Include.ALL)); + } + public static void updateLineageBotPolicy() { PolicyRepository policyRepository = (PolicyRepository) Entity.getEntityRepository(Entity.POLICY); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java index a5d7695c044..d5e482a0cf4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java @@ -261,7 +261,7 @@ public class LineageResource { @Parameter(description = "view (service or domain)") @QueryParam("view") @Pattern( - regexp = "service|domain|all", + regexp = "service|domain|dataProduct|all", message = "Invalid type. Allowed values: service, domain.") String view, @Parameter( @@ -273,6 +273,12 @@ public class LineageResource { @QueryParam("includeDeleted") boolean deleted) throws IOException { + if (Entity.getSearchRepository().getIndexMapping(view) != null) { + view = + Entity.getSearchRepository() + .getIndexMapping(view) + .getIndexName(Entity.getSearchRepository().getClusterAlias()); + } return Entity.getSearchRepository().searchPlatformLineage(view, queryFilter, deleted); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java index 58ef842fdb5..0e7f0dfc62e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java @@ -28,6 +28,7 @@ public record DataProductIndex(DataProduct dataProduct) implements SearchIndex { ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.DATA_PRODUCT, dataProduct)); doc.put("tags", parseTags.getTags()); doc.putAll(commonAttributes); + doc.put("upstreamLineage", SearchIndex.getLineageData(dataProduct.getEntityReference())); return doc; } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts index 24a46cd2e08..121e228ac67 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts @@ -44,7 +44,6 @@ export const SIDEBAR_LIST_ITEMS = { [SidebarItem.GLOSSARY]: [SidebarItem.GOVERNANCE, SidebarItem.GLOSSARY], [SidebarItem.TAGS]: [SidebarItem.GOVERNANCE, SidebarItem.TAGS], [SidebarItem.METRICS]: [SidebarItem.GOVERNANCE, SidebarItem.METRICS], - [SidebarItem.LINEAGE]: [SidebarItem.GOVERNANCE, SidebarItem.LINEAGE], // Profile Dropdown 'user-name': ['dropdown-profile', 'user-name'], diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PlatformLineage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PlatformLineage.spec.ts new file mode 100644 index 00000000000..4ec7b4442bf --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PlatformLineage.spec.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import test, { expect } from '@playwright/test'; +import { SidebarItem } from '../../constant/sidebar'; +import { redirectToHomePage } from '../../utils/common'; +import { sidebarClick } from '../../utils/sidebar'; + +test.use({ + storageState: 'playwright/.auth/admin.json', +}); + +test('Verify Platform Lineage View', async ({ page }) => { + await redirectToHomePage(page); + const lineageRes = page.waitForResponse( + '/api/v1/lineage/getPlatformLineage?view=service*' + ); + await sidebarClick(page, SidebarItem.LINEAGE); + await lineageRes; + + await expect(page.getByTestId('lineage-export')).not.toBeVisible(); + + await page.getByTestId('lineage-layer-btn').click(); + + await page.waitForSelector( + '[data-testid="lineage-layer-domain-btn"]:not(.active)' + ); + + const domainRes = page.waitForResponse( + '/api/v1/lineage/getPlatformLineage?view=domain*' + ); + await page.getByTestId('lineage-layer-domain-btn').click(); + await domainRes; + + await page.getByTestId('lineage-layer-btn').click(); + const dataProductRes = page.waitForResponse( + '/api/v1/lineage/getPlatformLineage?view=dataProduct*' + ); + await page.getByTestId('lineage-layer-data-product-btn').click(); + await dataProductRes; +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions-active.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions-active.svg new file mode 100644 index 00000000000..4f52c8deaee --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions-active.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx index d41186f7a66..65ec9339ec8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx @@ -354,7 +354,7 @@ const Suggestions = ({ className="m-b-md w-100 text-left d-flex items-center p-0" data-testid="nlp-suggestions-button" icon={ -
+
} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx index ffbe616d7d3..c710635cd3d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx @@ -135,19 +135,21 @@ const LineageControlButtons: FC = ({ /> )} -