From d4631d6d1cec29e6127452587fcbe3cf35ea3ae7 Mon Sep 17 00:00:00 2001 From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Date: Wed, 19 Mar 2025 12:07:05 +0530 Subject: [PATCH] Add Post Call for Aggregate (#20220) * Add Post Call for Aggregate * update aggregation calls to post for lineage * add platform lineage view * add unit tests * cleanup * update platform lineage page * add icon for other entities * cleanup * Add Lineage Migration * Move fetch all service to common class * Add view for service or domain * remove source field * add validation for inputs * add platform lineage page * fix tests * Checkstyle fix * fix tests * resolved conflicts * fix pytest --------- Co-authored-by: Karan Hotchandani <33024356+karanh37@users.noreply.github.com> Co-authored-by: karanh37 Co-authored-by: sonikashah --- .../java/org/openmetadata/service/Entity.java | 19 ++ .../service/jdbi3/CollectionDAO.java | 6 + .../service/jdbi3/GlossaryTermRepository.java | 26 +-- .../migration/mysql/v170/Migration.java | 4 + .../migration/postgres/v170/Migration.java | 4 + .../migration/utils/v170/MigrationUtil.java | 148 ++++++++++++ .../resources/lineage/LineageResource.java | 36 +++ .../resources/search/SearchResource.java | 137 ++++++------ .../service/search/SearchClient.java | 4 + .../service/search/SearchRepository.java | 29 ++- .../service/search/SearchRequest.java | 168 -------------- .../service/search/SearchUtils.java | 7 + .../elasticsearch/ESLineageGraphBuilder.java | 31 +++ .../elasticsearch/ElasticSearchClient.java | 28 ++- .../service/search/elasticsearch/EsUtils.java | 18 ++ .../service/search/nlq/NLQService.java | 2 +- .../service/search/nlq/NoOpNLQService.java | 2 +- .../opensearch/OSLineageGraphBuilder.java | 31 +++ .../search/opensearch/OpenSearchClient.java | 32 +-- .../service/search/opensearch/OsUtils.java | 18 ++ .../workflows/searchIndex/ReindexingUtil.java | 25 +-- .../resources/elasticsearch/indexMapping.json | 16 +- .../json/schema/search/searchRequest.json | 102 +++++++++ .../ui/playwright/constant/sidebar.ts | 2 + .../ui/playwright/e2e/Pages/Lineage.spec.ts | 40 ++++ .../ui/src/assets/svg/ic-platform-lineage.svg | 1 + .../AppRouter/AuthenticatedAppRouter.tsx | 6 + .../AssetSelectionModal.interface.ts | 2 + .../CustomControls.component.tsx | 21 +- .../EntitySuggestionOption.component.tsx | 97 ++++++++ .../EntitySuggestionOption.interface.ts | 21 ++ .../entity-suggestion-option.less | 24 ++ .../LineageControlButtons.interface.ts | 4 + .../LineageControlButtons.tsx | 40 ++-- .../LineageLayers/LineageLayers.interface.ts | 27 +++ .../LineageLayers/LineageLayers.test.tsx | 5 +- .../LineageLayers/LineageLayers.tsx | 170 ++++++++------ .../Explore/ExploreQuickFilters.interface.ts | 1 + .../Explore/ExploreQuickFilters.tsx | 16 +- .../components/Lineage/Lineage.component.tsx | 9 +- .../components/Lineage/Lineage.interface.ts | 1 + .../ui/src/constants/LeftSidebar.constants.ts | 8 + .../ui/src/constants/PageHeaders.constant.ts | 4 + .../resources/ui/src/constants/constants.ts | 2 + .../LineageProvider.interface.tsx | 7 +- .../LineageProvider/LineageProvider.test.tsx | 8 +- .../LineageProvider/LineageProvider.tsx | 111 +++++++-- .../resources/ui/src/enums/sidebar.enum.ts | 1 + .../ui/src/generated/search/searchRequest.ts | 97 ++++++++ .../ui/src/locale/languages/de-de.json | 2 + .../ui/src/locale/languages/en-us.json | 2 + .../ui/src/locale/languages/es-es.json | 2 + .../ui/src/locale/languages/fr-fr.json | 2 + .../ui/src/locale/languages/gl-es.json | 2 + .../ui/src/locale/languages/he-he.json | 2 + .../ui/src/locale/languages/ja-jp.json | 2 + .../ui/src/locale/languages/ko-kr.json | 2 + .../ui/src/locale/languages/mr-in.json | 2 + .../ui/src/locale/languages/nl-nl.json | 2 + .../ui/src/locale/languages/pr-pr.json | 2 + .../ui/src/locale/languages/pt-br.json | 2 + .../ui/src/locale/languages/pt-pt.json | 2 + .../ui/src/locale/languages/ru-ru.json | 2 + .../ui/src/locale/languages/th-th.json | 2 + .../ui/src/locale/languages/zh-cn.json | 2 + .../PlatformLineage.interface.ts | 16 ++ .../pages/PlatformLineage/PlatformLineage.tsx | 211 ++++++++++++++++++ .../PlatformLineage/platform-lineage.less | 23 ++ .../main/resources/ui/src/rest/lineageAPI.ts | 26 +++ .../src/main/resources/ui/src/rest/miscAPI.ts | 33 +++ .../ui/src/utils/Assets/AssetsUtils.ts | 5 + .../ui/src/utils/EntityLineageUtils.tsx | 26 +++ .../resources/ui/src/utils/ExploreUtils.ts | 17 ++ 73 files changed, 1585 insertions(+), 422 deletions(-) delete mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/search/searchRequest.json create mode 100644 openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-platform-lineage.svg create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.component.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/entity-suggestion-option.less create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageLayers/LineageLayers.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/search/searchRequest.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/platform-lineage.less diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index d93533fd376..5365d6c3e50 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -49,6 +49,7 @@ import org.jdbi.v3.core.Jdbi; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.EntityTimeSeriesInterface; import org.openmetadata.schema.FieldInterface; +import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; @@ -62,6 +63,7 @@ import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.EntityTimeSeriesRepository; import org.openmetadata.service.jdbi3.FeedRepository; import org.openmetadata.service.jdbi3.LineageRepository; +import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.PolicyRepository; import org.openmetadata.service.jdbi3.Repository; import org.openmetadata.service.jdbi3.RoleRepository; @@ -719,4 +721,21 @@ public final class Entity { EntityRepository entityRepository = Entity.getEntityRepository(entityType); return entityRepository.getAllowedFields().contains(field); } + + public static List getAllServicesForLineage() { + List allServices = new ArrayList<>(); + Set serviceTypes = new HashSet<>(List.of(ServiceType.values())); + serviceTypes.remove(ServiceType.METADATA); + + for (ServiceType serviceType : serviceTypes) { + EntityRepository repository = + Entity.getServiceEntityRepository(serviceType); + ListFilter filter = new ListFilter(Include.ALL); + List services = + (List) repository.listAll(repository.getFields("id"), filter); + allServices.addAll(services); + } + + return allServices; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index f7982b398de..d8851b692ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -1243,6 +1243,12 @@ public interface CollectionDAO { @BindUUID("toId") UUID toId, @Bind("relation") int relation); + @SqlQuery( + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship where relation = :relation ORDER BY fromId, toId LIMIT :limit OFFSET :offset") + @RegisterRowMapper(RelationshipObjectMapper.class) + List getRecordWithOffset( + @Bind("relation") int relation, @Bind("offset") long offset, @Bind("limit") int limit); + // // Delete Operations // diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 3fedf275921..8c05e20c48f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -72,6 +72,7 @@ import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.entity.data.GlossaryTerm.Status; import org.openmetadata.schema.entity.feed.Thread; import org.openmetadata.schema.entity.teams.Team; +import org.openmetadata.schema.search.SearchRequest; import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; @@ -94,7 +95,6 @@ import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.resources.glossary.GlossaryTermResource; -import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.security.AuthorizationException; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; @@ -491,20 +491,20 @@ public class GlossaryTermRepository extends EntityRepository { try { String key = "_source"; SearchRequest searchRequest = - new SearchRequest.ElasticSearchRequestBuilder( + new SearchRequest() + .withQuery( String.format( "** AND (tags.tagFQN:\"%s\")", - ReindexingUtil.escapeDoubleQuotes(glossaryFqn)), - size, - Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)) - .from(0) - .fetchSource(true) - .trackTotalHits(false) - .sortFieldParam("_score") - .deleted(false) - .sortOrder("desc") - .includeSourceFields(new ArrayList<>()) - .build(); + ReindexingUtil.escapeDoubleQuotes(glossaryFqn))) + .withSize(size) + .withIndex(Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)) + .withFrom(0) + .withFetchSource(true) + .withTrackTotalHits(false) + .withSortFieldParam("_score") + .withDeleted(false) + .withSortOrder("desc") + .withIncludeSourceFields(new ArrayList<>()); Response response = searchRepository.search(searchRequest, null); String json = (String) response.getEntity(); Set fqns = new TreeSet<>(compareEntityReferenceById); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java index 59652b19cca..d074a8d8091 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java @@ -3,6 +3,8 @@ package org.openmetadata.service.migration.mysql.v170; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.createServiceCharts; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNonNullColumn; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNullColumn; +import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationForDomainLineage; +import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationServiceLineage; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateDataInsightsApplication; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateGovernanceWorkflowDefinitions; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateLineageBotPolicy; @@ -28,6 +30,8 @@ public class Migration extends MigrationProcessImpl { // Lineage runLineageMigrationForNullColumn(handle); runLineageMigrationForNonNullColumn(handle); + runMigrationServiceLineage(handle); + runMigrationForDomainLineage(handle); // DI createServiceCharts(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java index 6990e6cb8d3..76e5fe3ad2d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java @@ -3,6 +3,8 @@ package org.openmetadata.service.migration.postgres.v170; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.createServiceCharts; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNonNullColumn; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNullColumn; +import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationForDomainLineage; +import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationServiceLineage; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateDataInsightsApplication; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateGovernanceWorkflowDefinitions; import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateLineageBotPolicy; @@ -28,6 +30,8 @@ public class Migration extends MigrationProcessImpl { // Lineage runLineageMigrationForNullColumn(handle); runLineageMigrationForNonNullColumn(handle); + runMigrationServiceLineage(handle); + runMigrationForDomainLineage(handle); // DI createServiceCharts(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java index c4c833f689e..e63d1132691 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java @@ -1,9 +1,13 @@ package org.openmetadata.service.migration.utils.v170; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.Entity.getAllServicesForLineage; import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; import java.lang.reflect.Field; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -11,33 +15,58 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.core.Handle; import org.jdbi.v3.core.statement.UnableToExecuteStatementException; +import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.LineChart; import org.openmetadata.schema.dataInsight.custom.LineChartMetric; +import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.policies.Policy; import org.openmetadata.schema.entity.policies.accessControl.Rule; import org.openmetadata.schema.governance.workflows.WorkflowConfiguration; import org.openmetadata.schema.governance.workflows.WorkflowDefinition; import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.LineageDetails; import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.governance.workflows.flowable.MainWorkflow; import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository; +import org.openmetadata.service.jdbi3.DomainRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.PolicyRepository; import org.openmetadata.service.jdbi3.WorkflowDefinitionRepository; import org.openmetadata.service.resources.databases.DatasourceConfig; import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @Slf4j public class MigrationUtil { + static final Map> SERVICE_TYPE_ENTITY_MAP = new HashMap<>(); + + static { + SERVICE_TYPE_ENTITY_MAP.put(Entity.DATABASE_SERVICE, List.of("table_entity")); + SERVICE_TYPE_ENTITY_MAP.put(Entity.MESSAGING_SERVICE, List.of("topic_entity")); + SERVICE_TYPE_ENTITY_MAP.put( + Entity.DASHBOARD_SERVICE, List.of("dashboard_entity", "dashboard_data_model_entity")); + SERVICE_TYPE_ENTITY_MAP.put(Entity.PIPELINE_SERVICE, List.of("pipeline_entity")); + SERVICE_TYPE_ENTITY_MAP.put(Entity.MLMODEL_SERVICE, List.of("ml_model_entity")); + SERVICE_TYPE_ENTITY_MAP.put(Entity.STORAGE_SERVICE, List.of("storage_container_entity")); + SERVICE_TYPE_ENTITY_MAP.put(Entity.SEARCH_SERVICE, List.of("search_index_entity")); + SERVICE_TYPE_ENTITY_MAP.put(Entity.API_SERVICE, List.of("api_endpoint_entity")); + } + private MigrationUtil() {} + public static final String DOMAIN_LINEAGE = + "select count(*) from entity_relationship where fromId in (select toId from entity_relationship where fromId = '%s' and relation = 10) AND toId in (select toId from entity_relationship where fromId = '%s' and relation = 10) and relation = 13"; + + public static final String SERVICE_ENTITY_MIGRATION = + "SELECT COUNT(*) FROM entity_relationship er JOIN %s f ON er.fromID = f.id JOIN %s t ON er.toID = t.id WHERE er.relation = 13 AND f.fqnHash LIKE '%s.%%' AND t.fqnHash LIKE '%s.%%'"; private static final String UPDATE_NULL_JSON = "UPDATE entity_relationship SET json = :json WHERE json IS NULL AND relation = 13"; @@ -341,6 +370,125 @@ public class MigrationUtil { .withName("ai")))); } + public static void runMigrationForDomainLineage(Handle handle) { + try { + List allDomains = getAllDomains(); + for (Domain fromDomain : allDomains) { + for (Domain toDomain : allDomains) { + if (fromDomain.getId().equals(toDomain.getId())) { + continue; + } + String sql = + String.format( + DOMAIN_LINEAGE, fromDomain.getId().toString(), toDomain.getId().toString()); + int count = handle.createQuery(sql).mapTo(Integer.class).one(); + if (count > 0) { + LineageDetails domainLineageDetails = + new LineageDetails() + .withCreatedAt(System.currentTimeMillis()) + .withUpdatedAt(System.currentTimeMillis()) + .withCreatedBy(ADMIN_USER_NAME) + .withUpdatedBy(ADMIN_USER_NAME) + .withSource(LineageDetails.Source.CHILD_ASSETS) + .withAssetEdges(count); + Entity.getCollectionDAO() + .relationshipDAO() + .insert( + fromDomain.getId(), + toDomain.getId(), + fromDomain.getEntityReference().getType(), + toDomain.getEntityReference().getType(), + Relationship.UPSTREAM.ordinal(), + JsonUtils.pojoToJson(domainLineageDetails)); + } + } + } + + } catch (Exception ex) { + LOG.error( + "Error while updating null json rows with createdAt, createdBy, updatedAt and updatedBy for lineage.", + ex); + } + } + + public static void runMigrationServiceLineage(Handle handle) { + try { + List allServices = getAllServicesForLineage(); + for (ServiceEntityInterface fromService : allServices) { + for (ServiceEntityInterface toService : allServices) { + insertServiceLineageDetails(handle, fromService, toService); + } + } + } catch (Exception ex) { + LOG.error( + "Error while updating null json rows with createdAt, createdBy, updatedAt and updatedBy for lineage.", + ex); + } + } + + private static void insertServiceLineageDetails( + Handle handle, ServiceEntityInterface fromService, ServiceEntityInterface toService) { + try { + if (fromService.getId().equals(toService.getId()) + && fromService.getServiceType().equals(toService.getServiceType())) { + return; + } + + String fromServiceHash = FullyQualifiedName.buildHash(fromService.getFullyQualifiedName()); + String toServiceHash = FullyQualifiedName.buildHash(toService.getFullyQualifiedName()); + List fromTableNames = + listOrEmpty(SERVICE_TYPE_ENTITY_MAP.get(fromService.getEntityReference().getType())); + List toTableNames = + listOrEmpty(SERVICE_TYPE_ENTITY_MAP.get(toService.getEntityReference().getType())); + for (String fromTableName : fromTableNames) { + for (String toTableName : toTableNames) { + if (!nullOrEmpty(fromTableName) && !nullOrEmpty(toTableName)) { + String sql = + String.format( + SERVICE_ENTITY_MIGRATION, + fromTableName, + toTableName, + fromServiceHash, + toServiceHash); + int count = handle.createQuery(sql).mapTo(Integer.class).one(); + + if (count > 0) { + LineageDetails serviceLineageDetails = + new LineageDetails() + .withCreatedAt(System.currentTimeMillis()) + .withUpdatedAt(System.currentTimeMillis()) + .withCreatedBy(ADMIN_USER_NAME) + .withUpdatedBy(ADMIN_USER_NAME) + .withSource(LineageDetails.Source.CHILD_ASSETS) + .withAssetEdges(count); + Entity.getCollectionDAO() + .relationshipDAO() + .insert( + fromService.getId(), + toService.getId(), + fromService.getEntityReference().getType(), + toService.getEntityReference().getType(), + Relationship.UPSTREAM.ordinal(), + JsonUtils.pojoToJson(serviceLineageDetails)); + } + } + } + } + + } catch (Exception ex) { + LOG.error( + "Found issue while updating lineage for service from {} , to: {}", + fromService.getFullyQualifiedName(), + toService.getFullyQualifiedName(), + ex); + } + } + + private static List getAllDomains() { + DomainRepository repository = (DomainRepository) Entity.getEntityRepository(Entity.DOMAIN); + return repository.listAll(repository.getFields("id"), new ListFilter(Include.ALL)); + } + public static void updateLineageBotPolicy() { PolicyRepository policyRepository = (PolicyRepository) Entity.getEntityRepository(Entity.POLICY); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java index 7feaf73404a..a5d7695c044 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java @@ -36,6 +36,7 @@ import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; +import javax.validation.constraints.Pattern; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -240,6 +241,41 @@ public class LineageResource { .withIncludeSourceFields(getRequiredLineageFields(includeSourceFields))); } + @GET + @Path("/getPlatformLineage") + @Operation( + operationId = "getPlatformLineage", + summary = "Get Platform Lineage", + responses = { + @ApiResponse( + responseCode = "200", + description = "search response", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SearchResponse.class))) + }) + public SearchLineageResult searchLineage( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "view (service or domain)") + @QueryParam("view") + @Pattern( + regexp = "service|domain|all", + message = "Invalid type. Allowed values: service, domain.") + String view, + @Parameter( + description = + "Elasticsearch query that will be combined with the query_string query generator from the `query` argument") + @QueryParam("query_filter") + String queryFilter, + @Parameter(description = "Filter documents by deleted param. By default deleted is false") + @QueryParam("includeDeleted") + boolean deleted) + throws IOException { + return Entity.getSearchRepository().searchPlatformLineage(view, queryFilter, deleted); + } + @GET @Path("/getLineage/{direction}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java index 3ecec5e4bd0..3d802541aa2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java @@ -15,7 +15,6 @@ package org.openmetadata.service.resources.search; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.jdbi3.RoleRepository.DOMAIN_ONLY_ACCESS_ROLE; -import static org.openmetadata.service.search.SearchRepository.ELASTIC_SEARCH_EXTENSION; import static org.openmetadata.service.security.DefaultAuthorizer.getSubjectContext; import es.org.elasticsearch.action.search.SearchResponse; @@ -31,6 +30,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import javax.validation.Valid; import javax.ws.rs.Consumes; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -46,15 +46,14 @@ import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.search.PreviewSearchRequest; -import org.openmetadata.schema.system.EventPublisherJob; +import org.openmetadata.schema.search.SearchRequest; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.search.SearchRepository; -import org.openmetadata.service.search.SearchRequest; +import org.openmetadata.service.search.SearchUtils; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.SubjectContext; -import org.openmetadata.service.util.JsonUtils; @Slf4j @Path("/v1/search") @@ -195,24 +194,25 @@ public class SearchResource { } SearchRequest request = - new SearchRequest.ElasticSearchRequestBuilder( - query, size, Entity.getSearchRepository().getIndexOrAliasName(index)) - .from(from) - .queryFilter(queryFilter) - .postFilter(postFilter) - .fetchSource(fetchSource) - .trackTotalHits(trackTotalHits) - .sortFieldParam(sortFieldParam) - .deleted(deleted) - .sortOrder(sortOrder) - .includeSourceFields(includeSourceFields) - .getHierarchy(getHierarchy) - .domains(domains) - .applyDomainFilter( + new SearchRequest() + .withQuery(query) + .withSize(size) + .withIndex(Entity.getSearchRepository().getIndexOrAliasName(index)) + .withFrom(from) + .withQueryFilter(queryFilter) + .withPostFilter(postFilter) + .withFetchSource(fetchSource) + .withTrackTotalHits(trackTotalHits) + .withSortFieldParam(sortFieldParam) + .withDeleted(deleted) + .withSortOrder(sortOrder) + .withIncludeSourceFields(includeSourceFields) + .withIsHierarchy(getHierarchy) + .withDomains(domains) + .withApplyDomainFilter( !subjectContext.isAdmin() && subjectContext.hasAnyRole(DOMAIN_ONLY_ACCESS_ROLE)) - .searchAfter(searchAfter) - .explain(explain) - .build(); + .withSearchAfter(SearchUtils.searchAfter(searchAfter)) + .withExplain(explain); return searchRepository.search(request, subjectContext); } @@ -243,20 +243,19 @@ public class SearchResource { SubjectContext subjectContext = getSubjectContext(securityContext); SearchRequest searchRequest = - new SearchRequest.ElasticSearchRequestBuilder( - previewRequest.getQuery(), - previewRequest.getSize(), - Entity.getSearchRepository().getIndexOrAliasName(previewRequest.getIndex())) - .from(previewRequest.getFrom()) - .queryFilter(previewRequest.getQueryFilter()) - .postFilter(previewRequest.getPostFilter()) - .fetchSource(previewRequest.getFetchSource()) - .trackTotalHits(previewRequest.getTrackTotalHits()) - .sortFieldParam(previewRequest.getSortField()) - .sortOrder(previewRequest.getSortOrder().value()) - .includeSourceFields(previewRequest.getIncludeSourceFields()) - .explain(previewRequest.getExplain()) - .build(); + new SearchRequest() + .withQuery(previewRequest.getQuery()) + .withSize(previewRequest.getSize()) + .withIndex(Entity.getSearchRepository().getIndexOrAliasName(previewRequest.getIndex())) + .withFrom(previewRequest.getFrom()) + .withQueryFilter(previewRequest.getQueryFilter()) + .withPostFilter(previewRequest.getPostFilter()) + .withFetchSource(previewRequest.getFetchSource()) + .withTrackTotalHits(previewRequest.getTrackTotalHits()) + .withSortFieldParam(previewRequest.getSortField()) + .withSortOrder(previewRequest.getSortOrder().value()) + .withIncludeSourceFields(previewRequest.getIncludeSourceFields()) + .withExplain(previewRequest.getExplain()); return searchRepository.previewSearch( searchRequest, subjectContext, previewRequest.getSearchSettings()); @@ -296,10 +295,11 @@ public class SearchResource { SubjectContext subjectContext = getSubjectContext(securityContext); SearchRequest request = - new SearchRequest.ElasticSearchRequestBuilder( - nlqQuery, size, Entity.getSearchRepository().getIndexOrAliasName(index)) - .from(from) - .build(); + new SearchRequest() + .withQuery(nlqQuery) + .withSize(size) + .withIndex(Entity.getSearchRepository().getIndexOrAliasName(index)) + .withFrom(from); return searchRepository.searchWithNLQ(request, subjectContext); } @@ -437,12 +437,14 @@ public class SearchResource { } SearchRequest request = - new SearchRequest.ElasticSearchRequestBuilder(query, size, index) - .fieldName(fieldName) - .deleted(deleted) - .fetchSource(fetchSource) - .includeSourceFields(includeSourceFields) - .build(); + new SearchRequest() + .withQuery(query) + .withSize(size) + .withIndex(index) + .withFieldName(fieldName) + .withDeleted(deleted) + .withFetchSource(fetchSource) + .withIncludeSourceFields(includeSourceFields); return searchRepository.suggest(request); } @@ -498,31 +500,30 @@ public class SearchResource { return searchRepository.aggregate(index, fieldName, value, query); } - @GET - @Path("/reindex/stream/status") + @POST + @Path("/aggregate") @Operation( - operationId = "getStreamJobStatus", - summary = "Get Stream Job Latest Status", - description = "Stream Job Status", + operationId = "aggregateSearchRequest", + summary = "Get aggregated Search Request", + description = "Get aggregated fields from entities.", responses = { - @ApiResponse(responseCode = "200", description = "Success"), - @ApiResponse(responseCode = "404", description = "Status not found") + @ApiResponse( + responseCode = "200", + description = "Table Aggregate API", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SearchResponse.class))) }) - public Response reindexAllJobLastStatus( - @Context UriInfo uriInfo, @Context SecurityContext securityContext) { - // Only admins can issue a reindex request - authorizer.authorizeAdmin(securityContext); - // Check if there is a running job for reindex for requested entity - String jobRecord; - jobRecord = - Entity.getCollectionDAO() - .entityExtensionTimeSeriesDao() - .getLatestExtension(ELASTIC_SEARCH_ENTITY_FQN_STREAM, ELASTIC_SEARCH_EXTENSION); - if (jobRecord != null) { - return Response.status(Response.Status.OK) - .entity(JsonUtils.readValue(jobRecord, EventPublisherJob.class)) - .build(); - } - return Response.status(Response.Status.NOT_FOUND).entity("No Last Run.").build(); + public Response aggregateSearchRequest( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid SearchRequest searchRequest) + throws IOException { + return searchRepository.aggregate( + searchRequest.getIndex(), + searchRequest.getFieldName(), + searchRequest.getFieldValue(), + searchRequest.getQuery()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index 521aa05868a..98d8f43420c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -19,6 +19,7 @@ import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList; import org.openmetadata.schema.entity.data.QueryCostSearchResult; +import org.openmetadata.schema.search.SearchRequest; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.tests.DataQualityReport; import org.openmetadata.schema.type.EntityReference; @@ -216,6 +217,9 @@ public interface SearchClient { SearchLineageResult searchLineageWithDirection(SearchLineageRequest lineageRequest) throws IOException; + SearchLineageResult searchPlatformLineage(String index, String queryFilter, boolean deleted) + throws IOException; + Response searchEntityRelationship( String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted) throws IOException; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index b096aa0c1f0..40b44dc4895 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -74,6 +74,7 @@ import org.openmetadata.schema.api.search.SearchSettings; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.entity.classification.Tag; import org.openmetadata.schema.entity.data.QueryCostSearchResult; +import org.openmetadata.schema.search.SearchRequest; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.service.configuration.elasticsearch.NaturalLanguageSearchConfiguration; import org.openmetadata.schema.tests.DataQualityReport; @@ -1092,6 +1093,11 @@ public class SearchRepository { return searchClient.searchLineage(lineageRequest); } + public SearchLineageResult searchPlatformLineage( + String alias, String queryFilter, boolean deleted) throws IOException { + return searchClient.searchPlatformLineage(alias, queryFilter, deleted); + } + public SearchLineageResult searchLineageWithDirection(SearchLineageRequest lineageRequest) throws IOException { return searchClient.searchLineageWithDirection(lineageRequest); @@ -1184,17 +1190,18 @@ public class SearchRepository { ReindexingUtil.escapeDoubleQuotes(entityFQN)); SearchRequest searchRequest = - new SearchRequest.ElasticSearchRequestBuilder( - "", size, Entity.getSearchRepository().getIndexOrAliasName(indexName)) - .from(0) - .queryFilter(queryFilter) - .fetchSource(true) - .trackTotalHits(false) - .sortFieldParam("_score") - .deleted(false) - .sortOrder("desc") - .includeSourceFields(new ArrayList<>()) - .build(); + new SearchRequest() + .withQuery("") + .withSize(size) + .withIndex(Entity.getSearchRepository().getIndexOrAliasName(indexName)) + .withFrom(0) + .withQueryFilter(queryFilter) + .withFetchSource(true) + .withTrackTotalHits(false) + .withSortFieldParam("_score") + .withDeleted(false) + .withSortOrder("desc") + .withIncludeSourceFields(new ArrayList<>()); // Execute the search and parse the response Response response = search(searchRequest, null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java deleted file mode 100644 index d1ff2fde246..00000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java +++ /dev/null @@ -1,168 +0,0 @@ -package org.openmetadata.service.search; - -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import lombok.Getter; -import lombok.Setter; -import org.openmetadata.schema.type.EntityReference; - -@Getter -@Setter -public class SearchRequest { - private final String query; - private int from; - private final int size; - private final String queryFilter; - private final String postFilter; - private final boolean fetchSource; - private final boolean trackTotalHits; - private final String sortFieldParam; - private final boolean deleted; - private final String index; - private final String fieldName; - private final String sortOrder; - private final List includeSourceFields; - private final boolean applyDomainFilter; - private final List domains; - private final boolean getHierarchy; - private final Object[] searchAfter; - private final boolean explain; - - public SearchRequest(ElasticSearchRequestBuilder builder) { - this.query = builder.query; - this.from = builder.from; - this.size = builder.size; - this.queryFilter = builder.queryFilter; - this.postFilter = builder.postFilter; - this.fetchSource = builder.fetchSource; - this.trackTotalHits = builder.trackTotalHits; - this.sortFieldParam = builder.sortFieldParam; - this.deleted = builder.deleted; - this.index = builder.index; - this.sortOrder = builder.sortOrder; - this.includeSourceFields = builder.includeSourceFields; - this.fieldName = builder.fieldName; - this.getHierarchy = builder.getHierarchy; - this.domains = builder.domains; - this.applyDomainFilter = builder.applyDomainFilter; - this.searchAfter = builder.searchAfter; - this.explain = builder.explain; - } - - // Builder class for ElasticSearchRequest - - public static class ElasticSearchRequestBuilder { - private final String index; - private final String query; - private final int size; - private int from; - private String fieldName; - private String queryFilter; - private String postFilter; - private boolean fetchSource; - private boolean trackTotalHits; - private String sortFieldParam; - private boolean deleted; - private String sortOrder; - private List includeSourceFields; - private boolean getHierarchy; - private boolean applyDomainFilter; - private List domains; - private Object[] searchAfter; - private boolean explain; - - public ElasticSearchRequestBuilder(String query, int size, String index) { - this.query = query; - this.size = size; - this.index = index; - } - - public ElasticSearchRequestBuilder from(int from) { - this.from = from; - return this; - } - - public ElasticSearchRequestBuilder queryFilter(String queryFilter) { - this.queryFilter = queryFilter; - return this; - } - - public ElasticSearchRequestBuilder postFilter(String postFilter) { - this.postFilter = postFilter; - return this; - } - - public ElasticSearchRequestBuilder fetchSource(boolean fetchSource) { - this.fetchSource = fetchSource; - return this; - } - - public ElasticSearchRequestBuilder trackTotalHits(boolean trackTotalHits) { - this.trackTotalHits = trackTotalHits; - return this; - } - - public ElasticSearchRequestBuilder sortFieldParam(String sortFieldParam) { - this.sortFieldParam = sortFieldParam; - return this; - } - - public ElasticSearchRequestBuilder deleted(boolean deleted) { - this.deleted = deleted; - return this; - } - - public ElasticSearchRequestBuilder applyDomainFilter(boolean applyDomainFilter) { - this.applyDomainFilter = applyDomainFilter; - return this; - } - - public ElasticSearchRequestBuilder sortOrder(String sortOrder) { - this.sortOrder = sortOrder; - return this; - } - - public ElasticSearchRequestBuilder includeSourceFields(List includeSourceFields) { - this.includeSourceFields = includeSourceFields; - return this; - } - - public ElasticSearchRequestBuilder fieldName(String fieldName) { - this.fieldName = fieldName; - return this; - } - - public ElasticSearchRequestBuilder getHierarchy(boolean getHierarchy) { - this.getHierarchy = getHierarchy; - return this; - } - - public ElasticSearchRequestBuilder domains(List references) { - this.domains = - references.stream() - .map(EntityReference::getFullyQualifiedName) - .collect(Collectors.toList()); - return this; - } - - public ElasticSearchRequestBuilder searchAfter(String searchAfter) { - this.searchAfter = null; - if (!nullOrEmpty(searchAfter)) { - this.searchAfter = Stream.of(searchAfter.split(",")).toArray(Object[]::new); - } - return this; - } - - public ElasticSearchRequestBuilder explain(boolean explain) { - this.explain = explain; - return this; - } - - public SearchRequest build() { - return new SearchRequest(this); - } - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchUtils.java index 2ad0f607150..4e21d2bcb4d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchUtils.java @@ -163,4 +163,11 @@ public final class SearchUtils { Set.of("fullyQualifiedName", "service", "fqnHash", "id", "entityType", "upstreamLineage")); return requiredFields; } + + public static List searchAfter(String searchAfter) { + if (!nullOrEmpty(searchAfter)) { + return List.of(searchAfter.split(",")); + } + return null; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ESLineageGraphBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ESLineageGraphBuilder.java index 93168293186..5354942ed47 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ESLineageGraphBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ESLineageGraphBuilder.java @@ -1,5 +1,6 @@ package org.openmetadata.service.search.elasticsearch; +import static org.openmetadata.common.utils.CommonUtil.collectionOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.FIELD_FULLY_QUALIFIED_NAME_HASH_KEYWORD; import static org.openmetadata.service.search.SearchClient.FQN_FIELD; @@ -21,6 +22,7 @@ import es.org.elasticsearch.search.SearchHit; import es.org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import es.org.elasticsearch.search.aggregations.bucket.terms.Terms; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +46,35 @@ public class ESLineageGraphBuilder { this.esClient = esClient; } + public SearchLineageResult getPlatformLineage(String index, String queryFilter, boolean deleted) + throws IOException { + SearchLineageResult result = + new SearchLineageResult() + .withNodes(new HashMap<>()) + .withUpstreamEdges(new HashMap<>()) + .withDownstreamEdges(new HashMap<>()); + SearchResponse searchResponse = EsUtils.searchEntities(index, queryFilter, deleted); + + // Add Nodes + Arrays.stream(searchResponse.getHits().getHits()) + .map(hit -> collectionOrEmpty(hit.getSourceAsMap())) + .forEach( + sourceMap -> { + String fqn = sourceMap.get(FQN_FIELD).toString(); + result.getNodes().putIfAbsent(fqn, new NodeInformation().withEntity(sourceMap)); + + List upstreamLineageList = getUpstreamLineageListIfExist(sourceMap); + for (EsLineageData esLineageData : upstreamLineageList) { + result + .getUpstreamEdges() + .putIfAbsent( + esLineageData.getDocId(), + esLineageData.withToEntity(getRelationshipRef(sourceMap))); + } + }); + return result; + } + public SearchLineageResult getUpstreamLineage(SearchLineageRequest request) throws IOException { SearchLineageResult result = new SearchLineageResult() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index b778c956a03..986e7ca29ac 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -139,6 +139,7 @@ import org.openmetadata.schema.dataInsight.custom.FormulaHolder; import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.data.QueryCostSearchResult; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.search.SearchRequest; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.settings.SettingsType; import org.openmetadata.schema.tests.DataQualityReport; @@ -160,7 +161,6 @@ import org.openmetadata.service.search.SearchAggregation; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchHealthStatus; import org.openmetadata.service.search.SearchIndexUtils; -import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.SearchResultListMapper; import org.openmetadata.service.search.SearchSortFilter; import org.openmetadata.service.search.UpdateSearchEventsConstant; @@ -393,7 +393,7 @@ public class ElasticSearchClient implements SearchClient { } if (!nullOrEmpty(request.getSearchAfter())) { - searchSourceBuilder.searchAfter(request.getSearchAfter()); + searchSourceBuilder.searchAfter(request.getSearchAfter().toArray()); } /* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */ @@ -409,7 +409,7 @@ public class ElasticSearchClient implements SearchClient { QueryBuilders.boolQuery() .must(searchSourceBuilder.query()) .must(QueryBuilders.existsQuery("deleted")) - .must(QueryBuilders.termQuery("deleted", request.isDeleted()))); + .must(QueryBuilders.termQuery("deleted", request.getDeleted()))); boolQueryBuilder.should( QueryBuilders.boolQuery() .must(searchSourceBuilder.query()) @@ -450,10 +450,10 @@ public class ElasticSearchClient implements SearchClient { searchSourceBuilder.query( QueryBuilders.boolQuery() .must(searchSourceBuilder.query()) - .must(QueryBuilders.termQuery("deleted", request.isDeleted()))); + .must(QueryBuilders.termQuery("deleted", request.getDeleted()))); } - if (!nullOrEmpty(request.getSortFieldParam()) && !request.isGetHierarchy()) { + if (!nullOrEmpty(request.getSortFieldParam()) && !request.getIsHierarchy()) { FieldSortBuilder fieldSortBuilder = new FieldSortBuilder(request.getSortFieldParam()) .order(SortOrder.fromString(request.getSortOrder())); @@ -482,11 +482,11 @@ public class ElasticSearchClient implements SearchClient { https://github.com/elastic/elasticsearch/issues/33028 */ searchSourceBuilder.fetchSource( new FetchSourceContext( - request.isFetchSource(), + request.getFetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {})); - if (request.isTrackTotalHits()) { + if (request.getTrackTotalHits()) { searchSourceBuilder.trackTotalHits(true); } else { searchSourceBuilder.trackTotalHitsUpTo(MAX_RESULT_HITS); @@ -502,7 +502,7 @@ public class ElasticSearchClient implements SearchClient { .source(searchSourceBuilder), RequestOptions.DEFAULT); - if (!request.isGetHierarchy()) { + if (!request.getIsHierarchy()) { return Response.status(OK).entity(searchResponse.toString()).build(); } else { // Build the nested hierarchy from elastic search response @@ -546,7 +546,7 @@ public class ElasticSearchClient implements SearchClient { SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client) throws IOException { - if (!request.isGetHierarchy()) { + if (!request.getIsHierarchy()) { return; } @@ -958,6 +958,12 @@ public class ElasticSearchClient implements SearchClient { } } + @Override + public SearchLineageResult searchPlatformLineage( + String index, String queryFilter, boolean deleted) throws IOException { + return lineageGraphBuilder.getPlatformLineage(index, queryFilter, deleted); + } + private void getEntityRelationship( String fqn, int depth, @@ -1457,7 +1463,7 @@ public class ElasticSearchClient implements SearchClient { "deleted", Collections.singletonList( CategoryQueryContext.builder() - .setCategory(String.valueOf(request.isDeleted())) + .setCategory(String.valueOf(request.getDeleted())) .build()))); } SuggestBuilder suggestBuilder = new SuggestBuilder(); @@ -1467,7 +1473,7 @@ public class ElasticSearchClient implements SearchClient { .timeout(new TimeValue(30, TimeUnit.SECONDS)) .fetchSource( new FetchSourceContext( - request.isFetchSource(), + request.getFetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {})); es.org.elasticsearch.action.search.SearchRequest searchRequest = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUtils.java index 29ba716bf7c..e5394973617 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUtils.java @@ -140,6 +140,24 @@ public class EsUtils { return boolQuery; } + public static SearchResponse searchEntities(String index, String queryFilter, Boolean deleted) + throws IOException { + es.org.elasticsearch.action.search.SearchRequest searchRequest = + new es.org.elasticsearch.action.search.SearchRequest( + Entity.getSearchRepository().getIndexOrAliasName(index)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query( + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("deleted", !nullOrEmpty(deleted) && deleted))); + + buildSearchSourceFilter(queryFilter, searchSourceBuilder); + searchRequest.source(searchSourceBuilder.size(10000)); + + RestHighLevelClient client = + (RestHighLevelClient) Entity.getSearchRepository().getSearchClient().getClient(); + return client.search(searchRequest, RequestOptions.DEFAULT); + } + public static void buildSearchSourceFilter( String queryFilter, SearchSourceBuilder searchSourceBuilder) { if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NLQService.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NLQService.java index c0749235dc0..a2c40f0294c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NLQService.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NLQService.java @@ -1,7 +1,7 @@ package org.openmetadata.service.search.nlq; import java.io.IOException; -import org.openmetadata.service.search.SearchRequest; +import org.openmetadata.schema.search.SearchRequest; /** * Interface for Natural Language Query (NLQ) processing services. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NoOpNLQService.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NoOpNLQService.java index 7a0cf85ee62..cef4e4375d8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NoOpNLQService.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NoOpNLQService.java @@ -3,7 +3,7 @@ package org.openmetadata.service.search.nlq; import java.io.IOException; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.search.NLQConfiguration; -import org.openmetadata.service.search.SearchRequest; +import org.openmetadata.schema.search.SearchRequest; /** * A no-operation implementation of NLQService that returns null/empty responses. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OSLineageGraphBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OSLineageGraphBuilder.java index 703c6ddcfdf..60d98d7634e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OSLineageGraphBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OSLineageGraphBuilder.java @@ -1,5 +1,6 @@ package org.openmetadata.service.search.opensearch; +import static org.openmetadata.common.utils.CommonUtil.collectionOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.FIELD_FULLY_QUALIFIED_NAME_HASH_KEYWORD; import static org.openmetadata.service.search.SearchClient.FQN_FIELD; @@ -15,6 +16,7 @@ import static org.openmetadata.service.search.opensearch.OsUtils.getSearchReques import com.nimbusds.jose.util.Pair; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -43,6 +45,35 @@ public class OSLineageGraphBuilder { this.esClient = esClient; } + public SearchLineageResult getPlatformLineage(String index, String queryFilter, boolean deleted) + throws IOException { + SearchLineageResult result = + new SearchLineageResult() + .withNodes(new HashMap<>()) + .withUpstreamEdges(new HashMap<>()) + .withDownstreamEdges(new HashMap<>()); + SearchResponse searchResponse = OsUtils.searchEntities(index, queryFilter, deleted); + + // Add Nodes + Arrays.stream(searchResponse.getHits().getHits()) + .map(hit -> collectionOrEmpty(hit.getSourceAsMap())) + .forEach( + sourceMap -> { + String fqn = sourceMap.get(FQN_FIELD).toString(); + result.getNodes().putIfAbsent(fqn, new NodeInformation().withEntity(sourceMap)); + + List upstreamLineageList = getUpstreamLineageListIfExist(sourceMap); + for (EsLineageData esLineageData : upstreamLineageList) { + result + .getUpstreamEdges() + .putIfAbsent( + esLineageData.getDocId(), + esLineageData.withToEntity(getRelationshipRef(sourceMap))); + } + }); + return result; + } + public SearchLineageResult getUpstreamLineage(SearchLineageRequest request) throws IOException { if (request.getLayerFrom() < 0 || request.getLayerSize() < 0) { throw new IllegalArgumentException( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 9ebdcd1c530..461dbcdcf55 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -79,6 +79,7 @@ import org.openmetadata.schema.dataInsight.custom.FormulaHolder; import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.data.QueryCostSearchResult; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.search.SearchRequest; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.settings.SettingsType; import org.openmetadata.schema.tests.DataQualityReport; @@ -100,7 +101,6 @@ import org.openmetadata.service.search.SearchAggregation; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchHealthStatus; import org.openmetadata.service.search.SearchIndexUtils; -import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.SearchResultListMapper; import org.openmetadata.service.search.SearchSortFilter; import org.openmetadata.service.search.models.IndexMapping; @@ -419,7 +419,7 @@ public class OpenSearchClient implements SearchClient { } if (!nullOrEmpty(request.getSearchAfter())) { - searchSourceBuilder.searchAfter(request.getSearchAfter()); + searchSourceBuilder.searchAfter(request.getSearchAfter().toArray()); } /* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */ @@ -434,7 +434,7 @@ public class OpenSearchClient implements SearchClient { QueryBuilders.boolQuery() .must(searchSourceBuilder.query()) .must(QueryBuilders.existsQuery("deleted")) - .must(QueryBuilders.termQuery("deleted", request.isDeleted()))); + .must(QueryBuilders.termQuery("deleted", request.getDeleted()))); boolQueryBuilder.should( QueryBuilders.boolQuery() .must(searchSourceBuilder.query()) @@ -475,10 +475,10 @@ public class OpenSearchClient implements SearchClient { searchSourceBuilder.query( QueryBuilders.boolQuery() .must(searchSourceBuilder.query()) - .must(QueryBuilders.termQuery("deleted", request.isDeleted()))); + .must(QueryBuilders.termQuery("deleted", request.getDeleted()))); } - if (!nullOrEmpty(request.getSortFieldParam()) && !request.isGetHierarchy()) { + if (!nullOrEmpty(request.getSortFieldParam()) && !request.getIsHierarchy()) { FieldSortBuilder fieldSortBuilder = new FieldSortBuilder(request.getSortFieldParam()) .order(SortOrder.fromString(request.getSortOrder())); @@ -507,18 +507,18 @@ public class OpenSearchClient implements SearchClient { https://github.com/Open/Opensearch/issues/33028 */ searchSourceBuilder.fetchSource( new FetchSourceContext( - request.isFetchSource(), + request.getFetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {})); - if (request.isTrackTotalHits()) { + if (request.getTrackTotalHits()) { searchSourceBuilder.trackTotalHits(true); } else { searchSourceBuilder.trackTotalHitsUpTo(MAX_RESULT_HITS); } searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS)); - if (request.isExplain()) { + if (request.getExplain()) { searchSourceBuilder.explain(true); } try { @@ -528,8 +528,8 @@ public class OpenSearchClient implements SearchClient { client.search( new os.org.opensearch.action.search.SearchRequest(request.getIndex()) .source(searchSourceBuilder), - OPENSEARCH_REQUEST_OPTIONS); - if (!request.isGetHierarchy()) { + RequestOptions.DEFAULT); + if (Boolean.FALSE.equals(request.getIsHierarchy())) { return Response.status(OK).entity(searchResponse.toString()).build(); } else { List response = buildSearchHierarchy(request, searchResponse); @@ -662,7 +662,7 @@ public class OpenSearchClient implements SearchClient { SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client) throws IOException { - if (!request.isGetHierarchy()) { + if (!request.getIsHierarchy()) { return; } @@ -1029,6 +1029,12 @@ public class OpenSearchClient implements SearchClient { } } + @Override + public SearchLineageResult searchPlatformLineage( + String index, String queryFilter, boolean deleted) throws IOException { + return lineageGraphBuilder.getPlatformLineage(index, queryFilter, deleted); + } + private void getEntityRelationship( String fqn, int depth, @@ -1575,7 +1581,7 @@ public class OpenSearchClient implements SearchClient { "deleted", Collections.singletonList( CategoryQueryContext.builder() - .setCategory(String.valueOf(request.isDeleted())) + .setCategory(String.valueOf(request.getDeleted())) .build()))); } SuggestBuilder suggestBuilder = new SuggestBuilder(); @@ -1585,7 +1591,7 @@ public class OpenSearchClient implements SearchClient { .timeout(new TimeValue(30, TimeUnit.SECONDS)) .fetchSource( new FetchSourceContext( - request.isFetchSource(), + request.getFetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {})); os.org.opensearch.action.search.SearchRequest searchRequest = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java index 384e1991e12..8fdd9fb7d17 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java @@ -140,6 +140,24 @@ public class OsUtils { return boolQuery; } + public static os.org.opensearch.action.search.SearchResponse searchEntities( + String index, String queryFilter, Boolean deleted) throws IOException { + os.org.opensearch.action.search.SearchRequest searchRequest = + new os.org.opensearch.action.search.SearchRequest( + Entity.getSearchRepository().getIndexOrAliasName(index)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query( + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("deleted", !nullOrEmpty(deleted) && deleted))); + + buildSearchSourceFilter(queryFilter, searchSourceBuilder); + searchRequest.source(searchSourceBuilder.size(10000)); + + RestHighLevelClient client = + (RestHighLevelClient) Entity.getSearchRepository().getSearchClient().getClient(); + return client.search(searchRequest, RequestOptions.DEFAULT); + } + public static void buildSearchSourceFilter( String queryFilter, SearchSourceBuilder searchSourceBuilder) { if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java index 5d3fec515a3..bfbe4fb3388 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.Response; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.search.SearchRequest; import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.system.Stats; import org.openmetadata.schema.system.StepStats; @@ -35,7 +36,6 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.EntityTimeSeriesRepository; import org.openmetadata.service.jdbi3.ListFilter; -import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.util.JsonUtils; import os.org.opensearch.action.bulk.BulkItemResponse; import os.org.opensearch.action.bulk.BulkResponse; @@ -148,18 +148,17 @@ public class ReindexingUtil { String matchingKey, String sourceFqn, int from) { String key = "_source"; SearchRequest searchRequest = - new SearchRequest.ElasticSearchRequestBuilder( - String.format("(%s:\"%s\")", matchingKey, sourceFqn), - 100, - Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)) - .from(from) - .fetchSource(true) - .trackTotalHits(false) - .sortFieldParam("_score") - .deleted(false) - .sortOrder("desc") - .includeSourceFields(new ArrayList<>()) - .build(); + new SearchRequest() + .withQuery(String.format("(%s:\"%s\")", matchingKey, sourceFqn)) + .withSize(100) + .withIndex(Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)) + .withFrom(from) + .withFetchSource(true) + .withTrackTotalHits(false) + .withSortFieldParam("_score") + .withDeleted(false) + .withSortOrder("desc") + .withIncludeSourceFields(new ArrayList<>()); List entities = new ArrayList<>(); Response response = Entity.getSearchRepository().search(searchRequest, null); String json = (String) response.getEntity(); diff --git a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json index 0148a7017cb..dce06a772af 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json @@ -213,42 +213,42 @@ "indexName": "database_service_search_index", "indexMappingFile": "/elasticsearch/%s/database_service_index_mapping.json", "alias": "databaseService", - "parentAliases": ["all"], + "parentAliases": ["all", "service"], "childAliases": ["database", "databaseSchema", "storedProcedure", "table", "testSuite", "testCase", "testCaseResolutionStatus", "testCaseResult"] }, "messagingService": { "indexName": "messaging_service_search_index", "indexMappingFile": "/elasticsearch/%s/messaging_service_index_mapping.json", "alias": "messagingService", - "parentAliases": ["all"], + "parentAliases": ["all", "service"], "childAliases": ["topic"] }, "pipelineService": { "indexName": "pipeline_service_search_index", "indexMappingFile": "/elasticsearch/%s/pipeline_service_index_mapping.json", "alias": "pipelineService", - "parentAliases": ["all"], + "parentAliases": ["all", "service"], "childAliases": ["pipeline"] }, "dashboardService": { "indexName": "dashboard_service_search_index", "indexMappingFile": "/elasticsearch/%s/dashboard_service_index_mapping.json", "alias": "dashboardService", - "parentAliases": ["all"], + "parentAliases": ["all", "service"], "childAliases": ["dashboard", "dashboardDataModel", "chart"] }, "searchService": { "indexName": "search_service_search_index", "indexMappingFile": "/elasticsearch/%s/search_service_index_mapping.json", "alias": "searchService", - "parentAliases": ["all"], + "parentAliases": ["all", "service"], "childAliases": ["searchIndex"] }, "storageService": { "indexName": "storage_service_search_index", "indexMappingFile": "/elasticsearch/%s/storage_service_index_mapping.json", "alias": "storageService", - "parentAliases": ["all"], + "parentAliases": ["all", "service"], "childAliases": ["container"] }, "metadataService": { @@ -262,14 +262,14 @@ "indexName": "mlmodel_service_search_index", "indexMappingFile": "/elasticsearch/%s/mlmodel_service_index_mapping.json", "alias": "mlModelService", - "parentAliases": ["all"], + "parentAliases": ["all", "service"], "childAliases": ["mlmodel"] }, "apiService": { "indexName": "api_service_search_index", "indexMappingFile": "/elasticsearch/%s/api_service_index_mapping.json", "alias": "apiService", - "parentAliases": ["all"], + "parentAliases": ["all", "service"], "childAliases": ["apiCollection", "apiEndpoint"] }, "apiCollection": { diff --git a/openmetadata-spec/src/main/resources/json/schema/search/searchRequest.json b/openmetadata-spec/src/main/resources/json/schema/search/searchRequest.json new file mode 100644 index 00000000000..2159fffe6c1 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/search/searchRequest.json @@ -0,0 +1,102 @@ +{ + "$id": "https://open-metadata.org/schema/search/searchRequest.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SearchRequest", + "description": "Search Request to find entities from Elastic Search based on different parameters.", + "javaType": "org.openmetadata.schema.search.SearchRequest", + "type": "object", + "properties": { + "query": { + "description": "Query to be send to Search Engine.", + "type": "string", + "default": "*" + }, + "index": { + "description": "Index Name.", + "type": "string", + "default": "table_search_index" + }, + "fieldName": { + "description": "Field Name to match.", + "type": "string", + "default": "suggest" + }, + "from": { + "description": "Start Index for the req.", + "type": "integer", + "default": 0 + }, + "size": { + "description": "Size to limit the no.of results returned.", + "type": "integer", + "default": 10 + }, + "queryFilter": { + "description": "Elasticsearch query that will be combined with the query_string query generator from the `query` arg", + "type": "string" + }, + "postFilter": { + "description": "Elasticsearch query that will be used as a post_filter", + "type": "string" + }, + "fetchSource": { + "description": "Get document body for each hit", + "type": "boolean", + "default": true + }, + "trackTotalHits": { + "description": "Track Total Hits.", + "type": "boolean", + "default": false + }, + "explain": { + "description": "Explain the results of the query. Defaults to false. Only for debugging purposes.", + "type": "boolean", + "default": false + }, + "deleted": { + "description": "Filter documents by deleted param.", + "type": "boolean", + "default": false + }, + "sortFieldParam": { + "description": "Sort the search results by field, available fields to sort weekly_stats daily_stats, monthly_stats, last_updated_timestamp.", + "type": "string", + "default": "_score" + }, + "sortOrder": { + "description": "Sort order asc for ascending or desc for descending, defaults to desc.", + "type": "string", + "default": "desc" + }, + "includeSourceFields": { + "description": "Get only selected fields of the document body for each hit. Empty value will return all fields", + "type": "array", + "items": { + "type": "string" + } + }, + "searchAfter": { + "description": "When paginating, specify the search_after values. Use it ass search_after=,,...", + "existingJavaType": "java.util.List" + }, + "domains": { + "description": "Internal Object to filter by Domains.", + "existingJavaType": "java.util.List" + }, + "applyDomainFilter": { + "description": "If Need to apply the domain filter.", + "type": "boolean" + }, + "isHierarchy": { + "description": "If true it will try to get the hierarchy of the entity.", + "type": "boolean", + "default": false + }, + "fieldValue": { + "description": "Field Value in case of Aggregations.", + "type": "string" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts index 76aaaaf6194..8357abe2b7d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts @@ -25,6 +25,7 @@ export enum SidebarItem { SETTINGS = 'settings', LOGOUT = 'logout', METRICS = 'metrics', + LINEAGE = 'lineage', } export const SIDEBAR_LIST_ITEMS = { @@ -43,6 +44,7 @@ export const SIDEBAR_LIST_ITEMS = { [SidebarItem.GLOSSARY]: [SidebarItem.GOVERNANCE, SidebarItem.GLOSSARY], [SidebarItem.TAGS]: [SidebarItem.GOVERNANCE, SidebarItem.TAGS], [SidebarItem.METRICS]: [SidebarItem.GOVERNANCE, SidebarItem.METRICS], + [SidebarItem.LINEAGE]: [SidebarItem.GOVERNANCE, SidebarItem.LINEAGE], // Profile Dropdown 'user-name': ['dropdown-profile', 'user-name'], diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts index 5cb05489d8e..fd759256fee 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts @@ -12,6 +12,7 @@ */ import test, { expect } from '@playwright/test'; import { get } from 'lodash'; +import { SidebarItem } from '../../constant/sidebar'; import { ApiEndpointClass } from '../../support/entity/ApiEndpointClass'; import { ContainerClass } from '../../support/entity/ContainerClass'; import { DashboardClass } from '../../support/entity/DashboardClass'; @@ -46,6 +47,7 @@ import { verifyNodePresent, visitLineageTab, } from '../../utils/lineage'; +import { sidebarClick } from '../../utils/sidebar'; // use the admin user to login test.use({ @@ -200,6 +202,18 @@ test('Verify column lineage between table and topic', async ({ browser }) => { const topic = new TopicClass(); await Promise.all([table.create(apiContext), topic.create(apiContext)]); + const tableServiceFqn = get( + table, + 'entityResponseData.service.fullyQualifiedName' + ); + + const tableServiceName = get(table, 'entityResponseData.service.name'); + + const topicServiceFqn = get( + topic, + 'entityResponseData.service.fullyQualifiedName' + ); + const sourceTableFqn = get(table, 'entityResponseData.fullyQualifiedName'); const sourceCol = `${sourceTableFqn}.${get( table, @@ -222,6 +236,32 @@ test('Verify column lineage between table and topic', async ({ browser }) => { await visitLineageTab(page); await verifyColumnLineageInCSV(page, table, topic, sourceCol, targetCol); + await test.step('Verify relation in platform lineage', async () => { + await sidebarClick(page, SidebarItem.LINEAGE); + const searchRes = page.waitForResponse('/api/v1/search/query?*'); + + await page.click('[data-testid="search-entity-select"]'); + await page.keyboard.type(tableServiceFqn); + await searchRes; + + await page.click(`[data-testid="node-suggestion-${tableServiceFqn}"]`); + + await page.waitForLoadState('networkidle'); + + const tableServiceNode = page.locator( + `[data-testid="lineage-node-${tableServiceFqn}"]` + ); + const topicServiceNode = page.locator( + `[data-testid="lineage-node-${topicServiceFqn}"]` + ); + + await expect(tableServiceNode).toBeVisible(); + await expect(topicServiceNode).toBeVisible(); + }); + + await redirectToHomePage(page); + await table.visitEntityPage(page); + await visitLineageTab(page); await page.click('[data-testid="edit-lineage"]'); await removeColumnLineage(page, sourceCol, targetCol); diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-platform-lineage.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-platform-lineage.svg new file mode 100644 index 00000000000..e6fa054003c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-platform-lineage.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx index a9aa45ffc49..a434549671a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx @@ -24,6 +24,7 @@ import AddCustomMetricPage from '../../pages/AddCustomMetricPage/AddCustomMetric import { CustomizablePage } from '../../pages/CustomizablePage/CustomizablePage'; import DataQualityPage from '../../pages/DataQuality/DataQualityPage'; import ForbiddenPage from '../../pages/ForbiddenPage/ForbiddenPage'; +import PlatformLineage from '../../pages/PlatformLineage/PlatformLineage'; import TagPage from '../../pages/TagPage/TagPage'; import { checkPermission, userPermissions } from '../../utils/PermissionsUtils'; import AdminProtectedRoute from './AdminProtectedRoute'; @@ -284,6 +285,11 @@ const AuthenticatedAppRouter: FunctionComponent = () => { + = ({ className, }: ControlProps) => { const { t } = useTranslation(); - const { onQueryFilterUpdate } = useLineageProvider(); + const { onQueryFilterUpdate, nodes } = useLineageProvider(); const [selectedFilter, setSelectedFilter] = useState([]); const [selectedQuickFilters, setSelectedQuickFilters] = useState< ExploreQuickFilterField[] @@ -51,6 +51,24 @@ const CustomControls: FC = ({ setSelectedFilter((prevSelected) => [...prevSelected, key]); }; + const queryFilter = useMemo(() => { + const nodeIds = (nodes ?? []) + .map((node) => node.data?.node?.id) + .filter(Boolean); + + return { + query: { + bool: { + must: { + terms: { + 'id.keyword': nodeIds, + }, + }, + }, + }, + }; + }, [nodes]); + const filterMenu: ItemType[] = useMemo(() => { return filters.map((filter) => ({ key: filter.key, @@ -145,6 +163,7 @@ const CustomControls: FC = ({ = ({ + entity, + heading, + onSelectHandler, + className, + showEntityTypeBadge = false, +}) => { + const serviceIcon = useMemo(() => { + const serviceType = get(entity, 'serviceType', ''); + + if (serviceType) { + return ( + {entity.name} + ); + } + + return getEntityIcon( + (entity as SearchSourceAlias).entityType ?? '', + 'w-4 h-4 m-r-xs' + ); + }, [entity]); + + return ( + + ); +}; + +export default EntitySuggestionOption; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.interface.ts new file mode 100644 index 00000000000..1bb522de502 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.interface.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityReference } from '../../../../generated/entity/type'; + +export interface EntitySuggestionOptionProps { + entity: EntityReference; + heading?: string; + onSelectHandler?: (value: EntityReference) => void; + className?: string; + showEntityTypeBadge?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/entity-suggestion-option.less b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/entity-suggestion-option.less new file mode 100644 index 00000000000..45eb651e19e --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/entity-suggestion-option.less @@ -0,0 +1,24 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.entity-suggestion-option-btn { + height: auto; + padding: 4px 12px; + + &:hover { + color: transparent; + } + + .entity-tag { + text-transform: capitalize; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.interface.ts index 6f9e31938ef..62215d98d98 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.interface.ts @@ -10,9 +10,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { EntityType } from '../../../../enums/entity.enum'; + export interface LineageControlButtonsProps { handleFullScreenViewClick?: () => void; onExitFullScreenViewClick?: () => void; deleted?: boolean; hasEditAccess: boolean; + entityType?: EntityType; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx index 8c59a9ac3fa..ffbe616d7d3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx @@ -28,10 +28,12 @@ import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../../../assets/svg/edit-new.svg'; import { ReactComponent as ExportIcon } from '../../../../assets/svg/ic-export.svg'; import { NO_PERMISSION_FOR_ACTION } from '../../../../constants/HelperTextUtil'; +import { SERVICE_TYPES } from '../../../../constants/Services.constant'; import { useLineageProvider } from '../../../../context/LineageProvider/LineageProvider'; import { LineagePlatformView } from '../../../../context/LineageProvider/LineageProvider.interface'; import { LineageLayer } from '../../../../generated/configuration/lineageSettings'; import { getLoadingStatusValue } from '../../../../utils/EntityLineageUtils'; +import { AssetsUnion } from '../../../DataAssets/AssetsSelectionModal/AssetSelectionModal.interface'; import { LineageConfig } from '../EntityLineage.interface'; import LineageConfigModal from '../LineageConfigModal'; import './lineage-control-buttons.less'; @@ -42,6 +44,7 @@ const LineageControlButtons: FC = ({ onExitFullScreenViewClick, deleted, hasEditAccess, + entityType, }) => { const { t } = useTranslation(); const [dialogVisible, setDialogVisible] = useState(false); @@ -98,23 +101,26 @@ const LineageControlButtons: FC = ({ return ( <>
- {!deleted && platformView === LineagePlatformView.None && ( - + ) +); + +const LineageLayers = ({ entityType }: LineageLayersProps) => { const { activeLayer, onUpdateLayerView, isEditMode, onPlatformViewChange, platformView, + isPlatformLineage, } = useLineageProvider(); - const onButtonClick = (value: LineageLayer) => { - const index = activeLayer.indexOf(value); - if (index === -1) { - onUpdateLayerView([...activeLayer, value]); - } else { - onUpdateLayerView(activeLayer.filter((layer) => layer !== value)); - } - }; + const handleLayerClick = React.useCallback( + (value: LineageLayer) => { + const index = activeLayer.indexOf(value); + if (index === -1) { + onUpdateLayerView([...activeLayer, value]); + } else { + onUpdateLayerView(activeLayer.filter((layer) => layer !== value)); + } + }, + [activeLayer, onUpdateLayerView] + ); + + const handlePlatformViewChange = React.useCallback( + (view: LineagePlatformView) => { + onPlatformViewChange( + platformView === view ? LineagePlatformView.None : view + ); + }, + [platformView, onPlatformViewChange] + ); + + const buttonContent = React.useMemo( + () => ( + + {entityType && !SERVICE_TYPES.includes(entityType as AssetsUnion) && ( + <> + handleLayerClick(LineageLayer.ColumnLevelLineage)} + /> + } + isActive={activeLayer.includes(LineageLayer.DataObservability)} + label={t('label.observability')} + testId="lineage-layer-observability-btn" + onClick={() => handleLayerClick(LineageLayer.DataObservability)} + /> + + )} + + {(isPlatformLineage || + (entityType && + !SERVICE_TYPES.includes(entityType as AssetsUnion))) && ( + } + isActive={platformView === LineagePlatformView.Service} + label={t('label.service')} + testId="lineage-layer-service-btn" + onClick={() => + handlePlatformViewChange(LineagePlatformView.Service) + } + /> + )} + + {(isPlatformLineage || + (entityType && entityType !== EntityType.DOMAIN)) && ( + } + isActive={platformView === LineagePlatformView.Domain} + label={t('label.domain')} + testId="lineage-layer-domain-btn" + onClick={() => handlePlatformViewChange(LineagePlatformView.Domain)} + /> + )} + + ), + [ + activeLayer, + platformView, + entityType, + handleLayerClick, + handlePlatformViewChange, + isPlatformLineage, + ] + ); return ( - - - - - } + content={buttonContent} overlayClassName="lineage-layers-popover" placement="right" trigger="click"> @@ -121,4 +161,4 @@ const LineageLayers = () => { ); }; -export default LineageLayers; +export default React.memo(LineageLayers); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts index 58be341f385..f79b5072bc8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts @@ -26,4 +26,5 @@ export interface ExploreQuickFiltersProps { onChangeShowDeleted?: (showDeleted: boolean) => void; independent?: boolean; // flag to indicate if the filters are independent of aggregations fieldsWithNullValues?: EntityFields[]; + defaultQueryFilter?: Record; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx index e6f52d76ae6..d567a0c3518 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx @@ -26,7 +26,6 @@ import { EntityFields } from '../../enums/AdvancedSearch.enum'; import { SearchIndex } from '../../enums/search.enum'; import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation'; import { QueryFilterInterface } from '../../pages/ExplorePage/ExplorePage.interface'; -import { getAggregateFieldOptions } from '../../rest/miscAPI'; import { getTags } from '../../rest/tagAPI'; import { getOptionsFromAggregationBucket } from '../../utils/AdvancedSearchUtils'; import { getEntityName } from '../../utils/EntityUtils'; @@ -34,6 +33,7 @@ import { getCombinedQueryFilterObject, getQuickFilterWithDeletedFlag, } from '../../utils/ExplorePage/ExplorePageUtils'; +import { getAggregationOptions } from '../../utils/ExploreUtils'; import { showErrorToast } from '../../utils/ToastUtils'; import SearchDropdown from '../SearchDropdown/SearchDropdown'; import { SearchDropdownOption } from '../SearchDropdown/SearchDropdown.interface'; @@ -48,6 +48,7 @@ const ExploreQuickFilters: FC = ({ independent = false, onFieldValueSelect, fieldsWithNullValues = [], + defaultQueryFilter, }) => { const location = useCustomLocation(); const [options, setOptions] = useState(); @@ -75,7 +76,8 @@ const ExploreQuickFilters: FC = ({ const updatedQuickFilters = getAdvancedSearchQuickFilters(); const combinedQueryFilter = getCombinedQueryFilterObject( updatedQuickFilters as QueryFilterInterface, - queryFilter as unknown as QueryFilterInterface + queryFilter as unknown as QueryFilterInterface, + defaultQueryFilter as unknown as QueryFilterInterface ); const fetchDefaultOptions = async ( @@ -87,11 +89,12 @@ const ExploreQuickFilters: FC = ({ buckets = aggregations[key].buckets; } else { const [res, tierTags] = await Promise.all([ - getAggregateFieldOptions( + getAggregationOptions( index, key, '', - JSON.stringify(combinedQueryFilter) + JSON.stringify(combinedQueryFilter), + independent ), key === TIER_FQN_KEY ? getTags({ parent: 'Tier', limit: 50 }) @@ -151,11 +154,12 @@ const ExploreQuickFilters: FC = ({ return; } if (key !== TIER_FQN_KEY) { - const res = await getAggregateFieldOptions( + const res = await getAggregationOptions( index, key, value, - JSON.stringify(combinedQueryFilter) + JSON.stringify(combinedQueryFilter), + independent ); const buckets = res.data.aggregations[`sterms#${key}`].buckets; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx index 586900ef787..cb55b32e586 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx @@ -56,6 +56,7 @@ const Lineage = ({ hasEditAccess, entity, entityType, + isPlatformLineage, }: LineageProps) => { const { t } = useTranslation(); const history = useHistory(); @@ -115,8 +116,8 @@ const Lineage = ({ ); useEffect(() => { - updateEntityData(entityType, entity as SourceType); - }, [entity, entityType]); + updateEntityData(entityType, entity as SourceType, isPlatformLineage); + }, [entity, entityType, isPlatformLineage]); // Loading the react flow component after the nodes and edges are initialised improves performance // considerably. So added an init state for showing loader. @@ -136,6 +137,7 @@ const Lineage = ({ + - + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts index f1d6e74040f..6c79912dc48 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts @@ -22,6 +22,7 @@ export interface LineageProps { hasEditAccess: boolean; isFullScreen?: boolean; entity?: SourceType; + isPlatformLineage?: boolean; } export interface EntityLineageResponse { diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts index 63bca174bc0..caf5795e610 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts @@ -21,6 +21,7 @@ import { ReactComponent as DataQualityIcon } from '../assets/svg/ic-data-contrac import { ReactComponent as DomainsIcon } from '../assets/svg/ic-domain.svg'; import { ReactComponent as IncidentMangerIcon } from '../assets/svg/ic-incident-manager.svg'; import { ReactComponent as ObservabilityIcon } from '../assets/svg/ic-observability.svg'; +import { ReactComponent as PlatformLineageIcon } from '../assets/svg/ic-platform-lineage.svg'; import { ReactComponent as SettingsIcon } from '../assets/svg/ic-settings-v1.svg'; import { ReactComponent as InsightsIcon } from '../assets/svg/lamp-charge.svg'; import { ReactComponent as LogoutIcon } from '../assets/svg/logout.svg'; @@ -43,6 +44,13 @@ export const SIDEBAR_LIST: Array = [ icon: ExploreIcon, dataTestId: `app-bar-item-${SidebarItem.EXPLORE}`, }, + { + key: ROUTES.PLATFORM_LINEAGE, + title: i18next.t('label.lineage'), + redirect_url: ROUTES.PLATFORM_LINEAGE, + icon: PlatformLineageIcon, + dataTestId: `app-bar-item-${SidebarItem.LINEAGE}`, + }, { key: ROUTES.OBSERVABILITY, title: i18next.t('label.observability'), diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts index d1029e45a39..e2f073c0201 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts @@ -256,4 +256,8 @@ export const PAGE_HEADERS = { entity: i18n.t('label.metric-plural'), }), }, + PLATFORM_LINEAGE: { + header: i18n.t('label.lineage'), + subHeader: i18n.t('message.page-sub-header-for-platform-lineage'), + }, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts index 4e9ec40f55a..e175b94aba8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/constants.ts @@ -126,6 +126,8 @@ export const ROUTES = { FORBIDDEN: '/403', UNAUTHORISED: '/unauthorised', LOGOUT: '/logout', + PLATFORM_LINEAGE: '/lineage', + PLATFORM_LINEAGE_WITH_FQN: `/lineage/${PLACEHOLDER_ROUTE_ENTITY_TYPE}/${PLACEHOLDER_ROUTE_FQN}`, MY_DATA: '/my-data', TOUR: '/tour', REPORTS: '/reports', diff --git a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx index ca85925325b..43a925d7d99 100644 --- a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx @@ -71,6 +71,7 @@ export interface LineageContextType { activeLayer: LineageLayer[]; platformView: LineagePlatformView; expandAllColumns: boolean; + isPlatformLineage: boolean; toggleColumnView: () => void; onInitReactFlow: (reactFlowInstance: ReactFlowInstance) => void; onPaneClick: () => void; @@ -101,7 +102,11 @@ export interface LineageContextType { onColumnEdgeRemove: () => void; onAddPipelineClick: () => void; onConnect: (connection: Edge | Connection) => void; - updateEntityData: (entityType: EntityType, entity?: SourceType) => void; + updateEntityData: ( + entityType: EntityType, + entity?: SourceType, + isPlatformLineage?: boolean + ) => void; onUpdateLayerView: (layers: LineageLayer[]) => void; redraw: () => Promise; } diff --git a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx index 198aea786dc..68fbe77645a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx @@ -14,6 +14,7 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import QueryString from 'qs'; import React, { useEffect } from 'react'; import { Edge } from 'reactflow'; +import { SourceType } from '../../components/SearchedData/SearchedData.interface'; import { EntityType } from '../../enums/entity.enum'; import { LineageDirection } from '../../generated/api/lineage/searchLineageRequest'; import { @@ -82,7 +83,12 @@ const DummyChildrenComponent = () => { }; useEffect(() => { - updateEntityData(EntityType.TABLE, undefined); + updateEntityData(EntityType.TABLE, { + id: 'table1', + name: 'table1', + type: 'table', + fullyQualifiedName: 'table1', + } as SourceType); }, []); return ( diff --git a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx index 7c8d28786c4..cd1a917bf83 100644 --- a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx @@ -77,6 +77,7 @@ import { exportLineageAsync, getDataQualityLineage, getLineageDataByFQN, + getPlatformLineage, updateLineageEdge, } from '../../rest/lineageAPI'; import { @@ -193,6 +194,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const deletePressed = useKeyPress('Delete'); const backspacePressed = useKeyPress('Backspace'); const { showModal } = useEntityExportModalProvider(); + const [isPlatformLineage, setIsPlatformLineage] = useState(false); const lineageLayer = useMemo(() => { const param = location.search; @@ -290,6 +292,8 @@ const LineageProvider = ({ children }: LineageProviderProps) => { const rootNode = visibleNodes.find((n) => n.data.isRootNode); if (rootNode) { centerNodePosition(rootNode, reactFlowInstance, zoomValue); + } else if (visibleNodes.length > 0) { + centerNodePosition(visibleNodes[0], reactFlowInstance, zoomValue); } } @@ -339,6 +343,41 @@ const LineageProvider = ({ children }: LineageProviderProps) => { [redrawLineage] ); + const fetchPlatformLineage = useCallback( + async (view: 'service' | 'domain', config?: LineageConfig) => { + try { + setLoading(true); + setInit(false); + const res = await getPlatformLineage({ + config, + view, + }); + + setLineageData(res); + + const { nodes, edges, entity } = parseLineageData(res, ''); + const updatedEntityLineage = { + nodes, + edges, + entity, + }; + + setEntityLineage(updatedEntityLineage); + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } finally { + setInit(true); + setLoading(false); + } + }, + [] + ); + const fetchLineageData = useCallback( async (fqn: string, entityType: string, config?: LineageConfig) => { if (isTourOpen) { @@ -499,9 +538,17 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }, []); const updateEntityData = useCallback( - (entityType: EntityType, entity?: SourceType) => { + ( + entityType: EntityType, + entity?: SourceType, + isPlatformLineage?: boolean + ) => { setEntity(entity); setEntityType(entityType); + setIsPlatformLineage(isPlatformLineage ?? false); + if (isPlatformLineage && !entity) { + setPlatformView(LineagePlatformView.Service); + } }, [] ); @@ -1261,20 +1308,45 @@ const LineageProvider = ({ children }: LineageProviderProps) => { }, [entityLineage, redrawLineage]); const onPlatformViewUpdate = useCallback(() => { - if (entity) { - if (platformView === LineagePlatformView.Service) { - if (entity?.service) { - fetchLineageData( - entity?.service.fullyQualifiedName ?? '', - entity?.service.type, - lineageConfig - ); - } + if (entity && decodedFqn && entityType) { + if (platformView === LineagePlatformView.Service && entity?.service) { + fetchLineageData( + entity?.service.fullyQualifiedName ?? '', + entity?.service.type, + lineageConfig + ); + } else if ( + platformView === LineagePlatformView.Domain && + entity?.domain + ) { + fetchLineageData( + entity?.domain.fullyQualifiedName ?? '', + entity?.domain.type, + lineageConfig + ); } else if (platformView === LineagePlatformView.None) { fetchLineageData(decodedFqn, entityType, lineageConfig); + } else if (isPlatformLineage) { + fetchPlatformLineage( + platformView === LineagePlatformView.Domain ? 'domain' : 'service', + lineageConfig + ); } + } else if (isPlatformLineage) { + fetchPlatformLineage( + platformView === LineagePlatformView.Domain ? 'domain' : 'service', + lineageConfig + ); } - }, [entity, entityType, decodedFqn, lineageConfig, platformView]); + }, [ + entity, + entityType, + decodedFqn, + lineageConfig, + platformView, + queryFilter, + isPlatformLineage, + ]); useEffect(() => { if (defaultLineageConfig) { @@ -1292,12 +1364,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { } }, [defaultLineageConfig]); - useEffect(() => { - if (entityType && isLineageSettingsLoaded) { - fetchLineageData(decodedFqn, entityType, lineageConfig); - } - }, [lineageConfig, queryFilter, entityType, isLineageSettingsLoaded]); - useEffect(() => { if (!isEditMode && updatedEntityLineage !== null) { // On exit of edit mode, use updatedEntityLineage and update data. @@ -1349,7 +1415,13 @@ const LineageProvider = ({ children }: LineageProviderProps) => { useEffect(() => { onPlatformViewUpdate(); - }, [platformView]); + }, [ + platformView, + lineageConfig, + queryFilter, + entityType, + isLineageSettingsLoaded, + ]); const activityFeedContextValues = useMemo(() => { return { @@ -1373,6 +1445,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { columnsHavingLineage, expandAllColumns, platformView, + isPlatformLineage, toggleColumnView, onInitReactFlow, onPaneClick, @@ -1399,7 +1472,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => { onExportClick, dataQualityLineage, redraw, - onPlatformViewChange, }; }, [ @@ -1423,6 +1495,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => { activeLayer, columnsHavingLineage, expandAllColumns, + isPlatformLineage, toggleColumnView, onInitReactFlow, onPaneClick, diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts index e1566b43016..122fd281753 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts @@ -26,4 +26,5 @@ export enum SidebarItem { SETTINGS = 'settings', LOGOUT = 'logout', METRICS = 'metrics', + LINEAGE = 'lineage', } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/search/searchRequest.ts b/openmetadata-ui/src/main/resources/ui/src/generated/search/searchRequest.ts new file mode 100644 index 00000000000..ace35ae25cd --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/search/searchRequest.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Search Request to find entities from Elastic Search based on different parameters. + */ +export interface SearchRequest { + /** + * If Need to apply the domain filter. + */ + applyDomainFilter?: boolean; + /** + * Filter documents by deleted param. + */ + deleted?: boolean; + /** + * Internal Object to filter by Domains. + */ + domains?: any; + /** + * Explain the results of the query. Defaults to false. Only for debugging purposes. + */ + explain?: boolean; + /** + * Get document body for each hit + */ + fetchSource?: boolean; + /** + * Field Name to match. + */ + fieldName?: string; + /** + * Field Value in case of Aggregations. + */ + fieldValue?: string; + /** + * Start Index for the req. + */ + from?: number; + /** + * Get only selected fields of the document body for each hit. Empty value will return all + * fields + */ + includeSourceFields?: string[]; + /** + * Index Name. + */ + index?: string; + /** + * If true it will try to get the hierarchy of the entity. + */ + isHierarchy?: boolean; + /** + * Elasticsearch query that will be used as a post_filter + */ + postFilter?: string; + /** + * Query to be send to Search Engine. + */ + query?: string; + /** + * Elasticsearch query that will be combined with the query_string query generator from the + * `query` arg + */ + queryFilter?: string; + /** + * When paginating, specify the search_after values. Use it ass + * search_after=,,... + */ + searchAfter?: any; + /** + * Size to limit the no.of results returned. + */ + size?: number; + /** + * Sort the search results by field, available fields to sort weekly_stats daily_stats, + * monthly_stats, last_updated_timestamp. + */ + sortFieldParam?: string; + /** + * Sort order asc for ascending or desc for descending, defaults to desc. + */ + sortOrder?: string; + /** + * Track Total Hits. + */ + trackTotalHits?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json index 6719a678d2f..22846675125 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json @@ -1173,6 +1173,7 @@ "search": "Suchen", "search-by-type": "Suchen nach {{type}}", "search-entity": "{{entity}} suchen", + "search-entity-for-lineage": "Nach {{entity}} suchen, um Abstammung anzuzeigen", "search-for-type": "Suchen nach {{type}}", "search-index": "Suchindex", "search-index-ingestion": "Suchindex für die Datenaufnahme", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Überprüfen Sie den Datenbankzugriff, den ES-Status, den Pipeline-Service-Client, die JWKS-Konfiguration und Migrationen.", "page-sub-header-for-persona": "Repräsentieren Sie verschiedene Personas, die ein Benutzer innerhalb von OpenMetadata haben kann.", "page-sub-header-for-pipelines": "Ingestion von Metadaten aus den am häufigsten verwendeten Pipeline-Diensten.", + "page-sub-header-for-platform-lineage": "Visualisieren Sie die Plattform-Abstammung, um Abhängigkeiten und Beziehungen in Ihrer Plattform zu verstehen.", "page-sub-header-for-policies": "Definiere Richtlinien mit einer Reihe von Regeln für feinkörnige Zugriffssteuerung.", "page-sub-header-for-profiler-configuration": "Passen Sie das Verhalten des Profilers global an, indem Sie die zu berechnenden Metriken basierend auf den Spaltendatentypen festlegen.", "page-sub-header-for-roles": "Weise Benutzern oder Teams umfassende rollenbasierte Zugriffsberechtigungen zu.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index 986948ca199..b5ce9dfdee0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -1173,6 +1173,7 @@ "search": "Search", "search-by-type": "Search by {{type}}", "search-entity": "Search {{entity}}", + "search-entity-for-lineage": "Nach {{entity}} suchen, um Abstammung anzuzeigen", "search-for-type": "Search for {{type}}", "search-index": "Search Index", "search-index-ingestion": "Search Index Ingestion", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations", "page-sub-header-for-persona": "Enhance and customize the user experience with Personas.", "page-sub-header-for-pipelines": "Ingest metadata from the most used pipeline services.", + "page-sub-header-for-platform-lineage": "Visualize the lineage to understand dependencies and relationships across your application", "page-sub-header-for-policies": "Define policies with a set of rules for fine-grained access control.", "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "Assign comprehensive role based access to Users or Teams.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index 566efcf87bb..1e06764d47c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -1173,6 +1173,7 @@ "search": "Buscar", "search-by-type": "Buscar por {{type}}", "search-entity": "Buscar {{entity}}", + "search-entity-for-lineage": "Buscar {{entity}} para ver el linaje", "search-for-type": "Buscar {{type}}", "search-index": "Índice de Búsqueda", "search-index-ingestion": "Ingesta de Índices de Búsqueda", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations", "page-sub-header-for-persona": "Representa diferentes personas que un usuario puede tener dentro de OpenMetadata.", "page-sub-header-for-pipelines": "Ingresa metadatos desde los servicios de pipeline más utilizados.", + "page-sub-header-for-platform-lineage": "Visualiza la línea de la plataforma para comprender las dependencias y relaciones en tu plataforma.", "page-sub-header-for-policies": "Define políticas con un conjunto de reglas para el control de acceso detallado.", "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "Asigna un acceso basado en roles integral a Usuarios o Equipos.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index a01ce5b8ea5..34081f78e92 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -1173,6 +1173,7 @@ "search": "Rechercher", "search-by-type": "Rechercher par {{type}}", "search-entity": "Rechercher {{entity}}", + "search-entity-for-lineage": "Rechercher {{entity}} pour afficher le lignage", "search-for-type": "Rechercher pour {{type}}", "search-index": "Index de Recherche", "search-index-ingestion": "Index de Recherche pour l'Ingestion", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Vérifiez l'accès aux bases de données, la santé d'ES, le service de Pipeline Service, la configuration jwks et les migrations", "page-sub-header-for-persona": "Représentez différents personas qu'un utilisateur peut avoir dans OpenMetadata.", "page-sub-header-for-pipelines": "Ingestion de métadonnées à partir des services de pipeline les plus utilisés.", + "page-sub-header-for-platform-lineage": "Visualisez l'ascendance de la plateforme pour comprendre les dépendances et les relations au sein de votre plateforme.", "page-sub-header-for-policies": "Définissez des politiques avec un ensemble de règles pour un contrôle d'accès précis.", "page-sub-header-for-profiler-configuration": "Personnalisez le comportement global du profiler en établissant les métriques à calculer à partir des types de données des colonnes", "page-sub-header-for-roles": "Attribuez des autorisations basées sur les rôles aux utilisateurs ou aux équipes.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json index ed25fea2531..9b616cfbd76 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json @@ -1173,6 +1173,7 @@ "search": "Buscar", "search-by-type": "Buscar por {{type}}", "search-entity": "Buscar {{entity}}", + "search-entity-for-lineage": "Buscar {{entity}} para ver a liñaxe", "search-for-type": "Buscar por {{type}}", "search-index": "Índice de busca", "search-index-ingestion": "Inxestión do índice de busca", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Verifica o acceso á base de datos, a saúde de Elasticsearch, o Cliente de Servizo Pipeline, a configuración de jwks e as migracións", "page-sub-header-for-persona": "Mellora e personaliza a experiencia de usuario con Persoas.", "page-sub-header-for-pipelines": "Inxesta metadatos dos servizos de pipelines máis usados.", + "page-sub-header-for-platform-lineage": "Visualiza a liñaxe da plataforma para comprender as dependencias e relacións na túa plataforma.", "page-sub-header-for-policies": "Define políticas cun conxunto de regras para un control de acceso detallado.", "page-sub-header-for-profiler-configuration": "Personaliza globalmente o comportamento do perfilador definindo as métricas a calcular baseadas nos tipos de datos das columnas", "page-sub-header-for-roles": "Asigna un acceso baseado en roles detallados a Usuarios ou Equipos.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json index e3d5e7f1ff1..a1675847aa2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json @@ -1173,6 +1173,7 @@ "search": "חיפוש", "search-by-type": "חפש לפי {{type}}", "search-entity": "חפש {{entity}}", + "search-entity-for-lineage": "חפש {{entity}} כדי להציג שושלת", "search-for-type": "חפש עבור {{type}}", "search-index": "חיפוש באינדקס", "search-index-ingestion": "פרסום חיפוש באינדקס", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations", "page-sub-header-for-persona": "צור פרופיל משתמש (פרסונה) על מנת לשייך את המשתמש לפרופיל ולהתאים את הממשק לצרכים של המשתמשים כאשר הם נכנסים ל-UI.", "page-sub-header-for-pipelines": "שלב מטה-דאטה ממוצרי טעינת נתונים (ETL, ELT) פופלריים (כגון Airflow, Glue וכד׳)", + "page-sub-header-for-platform-lineage": "המחישו את אילן היוחסין של הפלטפורמה כדי להבין את התלות והקשרים בתוך הפלטפורמה שלכם.", "page-sub-header-for-policies": "הגדר מדיניות עם סט של כללים לבקרת גישה ברמת רזולוציה נמוכה.", "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "הקצאת גישה מבוססת תפקיד למשתמשים או קבוצות.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index 938707f7175..1bffb10ae79 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -1173,6 +1173,7 @@ "search": "検索", "search-by-type": "{{type}}で検索", "search-entity": "{{entity}}を検索", + "search-entity-for-lineage": "系統を表示するには{{entity}}を検索", "search-for-type": "{{type}}を検索", "search-index": "Search Index", "search-index-ingestion": "Search Index Ingestion", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations", "page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.", "page-sub-header-for-pipelines": "Ingest metadata from the most used pipeline services.", + "page-sub-header-for-platform-lineage": "プラットフォームの系統を可視化し、依存関係と相互関係を理解しましょう。", "page-sub-header-for-policies": "Define policies with a set of rules for fine-grained access control.", "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "Assign comprehensive role based access to Users or Teams.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json index 836a24bb6f8..199dafb2e8a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json @@ -1173,6 +1173,7 @@ "search": "검색", "search-by-type": "{{type}}로 검색", "search-entity": "{{entity}} 검색", + "search-entity-for-lineage": "계보를 보려면 {{entity}} 검색", "search-for-type": "{{type}} 검색", "search-index": "검색 인덱스", "search-index-ingestion": "검색 인덱스 수집", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "데이터베이스 접근, Elasticsearch 상태, 파이프라인 서비스 클라이언트, jwks 구성 및 마이그레이션을 확인하세요.", "page-sub-header-for-persona": "페르소나를 통해 사용자 경험을 향상하고 맞춤 설정하세요.", "page-sub-header-for-pipelines": "가장 많이 사용되는 파이프라인 서비스로부터 메타데이터를 수집하세요.", + "page-sub-header-for-platform-lineage": "플랫폼 계보를 시각화하여 플랫폼 내 종속성과 관계를 이해하세요.", "page-sub-header-for-policies": "세분화된 접근 제어를 위한 규칙 집합으로 정책을 정의하세요.", "page-sub-header-for-profiler-configuration": "열 데이터 유형에 따라 계산할 메트릭을 설정하여 프로파일러 동작을 전역적으로 맞춤 설정하세요.", "page-sub-header-for-roles": "사용자 또는 팀에 대해 포괄적인 역할 기반 접근을 할당하세요.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json index 085a3536e6c..b6ac376a9a9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json @@ -1173,6 +1173,7 @@ "search": "शोधा", "search-by-type": "{{type}} द्वारे शोधा", "search-entity": "{{entity}} शोधा", + "search-entity-for-lineage": "वंशावळ पाहण्यासाठी {{entity}} शोधा", "search-for-type": "{{type}} साठी शोधा", "search-index": "शोध अनुक्रमणिका", "search-index-ingestion": "शोध अनुक्रमणिका अंतर्ग्रहण", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "डेटाबेस प्रवेश, ES आरोग्य, पाइपलाइन सेवा क्लायंट, jwks संरचना आणि स्थलांतर तपासा", "page-sub-header-for-persona": "व्यक्तिमत्वांसह वापरकर्ता अनुभव वाढवा आणि सानुकूलित करा.", "page-sub-header-for-pipelines": "सर्वात जास्त वापरल्या जाणार्‍या पाइपलाइन सेवांमधून मेटाडेटा अंतर्ग्रहण करा.", + "page-sub-header-for-platform-lineage": "प्लॅटफॉर्म वंशावळ दृश्यमान करा जेणेकरून तुमच्या प्लॅटफॉर्ममधील अवलंबित्वे आणि नातेसंबंध समजून घेता येतील.", "page-sub-header-for-policies": "सूक्ष्म प्रवेश नियंत्रणासाठी नियमांचा संच असलेली धोरणे परिभाषित करा.", "page-sub-header-for-profiler-configuration": "स्तंभ डेटा प्रकारांवर आधारित मेट्रिक्स सेट करून प्रोफाइलरचे वर्तन जागतिक स्तरावर सानुकूलित करा", "page-sub-header-for-roles": "वापरकर्ते किंवा टीम्सना व्यापक भूमिका आधारित प्रवेश नियुक्त करा.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json index e6e94a6db7c..4906f47991e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json @@ -1173,6 +1173,7 @@ "search": "Zoeken", "search-by-type": "Zoeken op {{type}}", "search-entity": "Zoek {{entity}}", + "search-entity-for-lineage": "Zoek naar {{entity}} om afstamming te bekijken", "search-for-type": "Zoeken naar {{type}}", "search-index": "Zoekindex", "search-index-ingestion": "Zoekindexingestie", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations", "page-sub-header-for-persona": "De gebruikerservaring verbeteren en aanpassen met Persona's.", "page-sub-header-for-pipelines": "Ingest metadata van de meestgebruikte pipelineservices.", + "page-sub-header-for-platform-lineage": "Visualiseer de platformafstamming om afhankelijkheden en relaties binnen uw platform te begrijpen.", "page-sub-header-for-policies": "Definieer beleid met een reeks regels voor fijnmazige toegangscontrole.", "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "Wijs uitgebreide rolgebaseerde toegang toe aan gebruikers of teams.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json index 1c59d254ad5..27c476f2dc1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json @@ -1173,6 +1173,7 @@ "search": "جستجو", "search-by-type": "جستجو بر اساس {{type}}", "search-entity": "جستجو {{entity}}", + "search-entity-for-lineage": "برای مشاهده تبارنامه، {{entity}} را جستجو کنید", "search-for-type": "جستجو برای {{type}}", "search-index": "شاخص جستجو", "search-index-ingestion": "ورود شاخص جستجو", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "بررسی دسترسی به پایگاه داده، سلامت Elasticsearch، سرویس کلاینت Pipeline، پیکربندی jwks و مهاجرت‌ها.", "page-sub-header-for-persona": "تجربه کاربر را با شخصیت‌های مختلف بهبود بخشید و سفارشی‌سازی کنید.", "page-sub-header-for-pipelines": "ورود متادیتا از پرکاربردترین سرویس‌های خطوط لوله.", + "page-sub-header-for-platform-lineage": "Visualiza a linhagem da plataforma para entender dependências e relacionamentos na sua plataforma.", "page-sub-header-for-policies": "تعریف سیاست‌ها با مجموعه‌ای از قوانین برای کنترل دقیق دسترسی.", "page-sub-header-for-profiler-configuration": "تنظیم رفتار پروفایلر در سطح جهانی با تعیین معیارها بر اساس نوع داده‌های ستون‌ها.", "page-sub-header-for-roles": "تخصیص دسترسی جامع مبتنی بر نقش به کاربران یا تیم‌ها.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index 04b84dbce51..d0a95a28f2a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -1173,6 +1173,7 @@ "search": "Pesquisar", "search-by-type": "Pesquisar por {{type}}", "search-entity": "Pesquisar {{entity}}", + "search-entity-for-lineage": "Pesquisar {{entity}} para visualizar a linhagem", "search-for-type": "Pesquisar por {{type}}", "search-index": "Índice de Pesquisa", "search-index-ingestion": "Ingestão de Índice de Pesquisa", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations", "page-sub-header-for-persona": "Crie Personas para associar a persona do usuário ao OpenMetadata", "page-sub-header-for-pipelines": "Ingestão de metadados dos serviços de pipeline mais utilizados.", + "page-sub-header-for-platform-lineage": "Visualize a linhagem da plataforma para entender dependências e relacionamentos em sua plataforma.", "page-sub-header-for-policies": "Defina políticas com um conjunto de regras para controle de acesso detalhado.", "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "Atribua acesso baseado em funções abrangentes a usuários ou equipes.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json index 02e65bcafea..a9da048a4fc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json @@ -1173,6 +1173,7 @@ "search": "Pesquisar", "search-by-type": "Pesquisar por {{type}}", "search-entity": "Pesquisar {{entity}}", + "search-entity-for-lineage": "Pesquisar {{entity}} para visualizar a linhagem", "search-for-type": "Pesquisar por {{type}}", "search-index": "Índice de Pesquisa", "search-index-ingestion": "Ingestão de Índice de Pesquisa", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations", "page-sub-header-for-persona": "Crie Personas para associar a persona do utilizador ao OpenMetadata", "page-sub-header-for-pipelines": "Ingestão de metadados dos serviços de pipeline mais utilizados.", + "page-sub-header-for-platform-lineage": "Visualize a linhagem da plataforma para compreender dependências e relações na sua plataforma.", "page-sub-header-for-policies": "Defina políticas com um conjunto de regras para controle de acesso detalhado.", "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "Atribua acesso baseado em funções abrangentes a Utilizadores ou equipas.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json index b22203f75bc..8a3f03f76b4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json @@ -1173,6 +1173,7 @@ "search": "Поиск", "search-by-type": "Поиск {{type}}", "search-entity": "Поиск {{entity}}", + "search-entity-for-lineage": "Найдите {{entity}} для просмотра происхождения", "search-for-type": "Поиск для {{type}}", "search-index": "Search Index", "search-index-ingestion": "Получение поискового индекса", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations", "page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.", "page-sub-header-for-pipelines": "Принимать метаданные из наиболее часто используемых конвейерных служб.", + "page-sub-header-for-platform-lineage": "Визуализируйте родословную платформы, чтобы понять зависимости и взаимосвязи в вашей платформе.", "page-sub-header-for-policies": "Определите политики с набором правил для точного контроля доступа.", "page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types", "page-sub-header-for-roles": "Назначьте полный доступ на основе ролей пользователям или командам.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json index e8b10d1b44f..4595880e118 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json @@ -1173,6 +1173,7 @@ "search": "ค้นหา", "search-by-type": "ค้นหาตาม {{type}}", "search-entity": "ค้นหา {{entity}}", + "search-entity-for-lineage": "ค้นหา {{entity}} เพื่อดูสายพันธุ์", "search-for-type": "ค้นหาสำหรับ {{type}}", "search-index": "ดัชนีการค้นหา", "search-index-ingestion": "การนำเข้าดัชนีการค้นหา", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "ตรวจสอบการเข้าถึงฐานข้อมูล, สถานะ ES, ไคลเอนต์บริการท่อ, การกำหนดค่า jwks, และการโยกย้าย", "page-sub-header-for-persona": "เสริมสร้างและปรับแต่งประสบการณ์ของผู้ใช้ด้วยบุคลิกภาพ", "page-sub-header-for-pipelines": "นำเข้าข้อมูลเมตาจากบริการท่อที่ใช้มากที่สุด", + "page-sub-header-for-platform-lineage": "แสดงลำดับของแพลตฟอร์มเพื่อให้เข้าใจถึงการพึ่งพาและความสัมพันธ์ภายในแพลตฟอร์มของคุณ", "page-sub-header-for-policies": "กำหนดนโยบายพร้อมกฎชุดสำหรับการควบคุมการเข้าถึงที่มีความละเอียด", "page-sub-header-for-profiler-configuration": "ปรับแต่งพฤติกรรมของโปรไฟล์เลอร์ในระดับโลกโดยการตั้งค่าเมตริกที่จะคำนวณตามประเภทของข้อมูลในคอลัมน์", "page-sub-header-for-roles": "มอบบทบาทการเข้าถึงที่ครอบคลุมให้กับผู้ใช้หรือทีม", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index 0d96fb0973f..c688d97da07 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -1173,6 +1173,7 @@ "search": "搜索", "search-by-type": "通过{{type}}搜索", "search-entity": "搜索{{entity}}", + "search-entity-for-lineage": "搜索{{entity}}以查看血统", "search-for-type": "搜索{{type}}", "search-index": "搜索索引", "search-index-ingestion": "搜索索引提取", @@ -1928,6 +1929,7 @@ "page-sub-header-for-om-health-configuration": "检查数据库访问、ES 健康状况、工作流服务客户端、jwks 配置和迁移情况", "page-sub-header-for-persona": "代表用户在 OpenMetadata 中可能拥有的不同用户角色", "page-sub-header-for-pipelines": "从最常用的工作流类型服务中提取元数据", + "page-sub-header-for-platform-lineage": "可视化平台谱系，以理解平台内的依赖关系和关联。", "page-sub-header-for-policies": "通过组合一系列规则定义权限策略以精细化控制访问权限", "page-sub-header-for-profiler-configuration": "根据列数据类型计算的指标, 设置全局自定义分析器的行为", "page-sub-header-for-roles": "分配基于角色的访问权限给用户或团队", diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.interface.ts new file mode 100644 index 00000000000..30d5d2df401 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.interface.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export enum LineagePlatformView { + Service = 'service', + Domain = 'domain', +} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.tsx new file mode 100644 index 00000000000..6474e744c4a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.tsx @@ -0,0 +1,211 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Col, Divider, Row, Select } from 'antd'; +import { DefaultOptionType } from 'antd/lib/select'; +import { AxiosError } from 'axios'; +import { debounce } from 'lodash'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useHistory, useParams } from 'react-router-dom'; +import Loader from '../../components/common/Loader/Loader'; +import { AssetsUnion } from '../../components/DataAssets/AssetsSelectionModal/AssetSelectionModal.interface'; +import EntitySuggestionOption from '../../components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.component'; +import Lineage from '../../components/Lineage/Lineage.component'; +import PageHeader from '../../components/PageHeader/PageHeader.component'; +import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; +import { SourceType } from '../../components/SearchedData/SearchedData.interface'; +import { PAGE_SIZE_BASE } from '../../constants/constants'; +import { PAGE_HEADERS } from '../../constants/PageHeaders.constant'; +import LineageProvider from '../../context/LineageProvider/LineageProvider'; +import { + OperationPermission, + ResourceEntity, +} from '../../context/PermissionProvider/PermissionProvider.interface'; +import { EntityType } from '../../enums/entity.enum'; +import { SearchIndex } from '../../enums/search.enum'; +import { EntityReference } from '../../generated/entity/type'; +import { useFqn } from '../../hooks/useFqn'; +import { getEntityPermissionByFqn } from '../../rest/permissionAPI'; +import { searchQuery } from '../../rest/searchAPI'; +import { getEntityAPIfromSource } from '../../utils/Assets/AssetsUtils'; +import { getLineageEntityExclusionFilter } from '../../utils/EntityLineageUtils'; +import { getOperationPermissions } from '../../utils/PermissionsUtils'; +import { showErrorToast } from '../../utils/ToastUtils'; +import './platform-lineage.less'; + +const PlatformLineage = () => { + const { t } = useTranslation(); + const history = useHistory(); + const { entityType } = useParams<{ entityType: EntityType }>(); + const { fqn: decodedFqn } = useFqn(); + const [selectedEntity, setSelectedEntity] = useState(); + const [loading, setLoading] = useState(false); + const [options, setOptions] = useState([]); + const [isSearchLoading, setIsSearchLoading] = useState(false); + const [defaultValue, setDefaultValue] = useState( + decodedFqn || undefined + ); + const [permissions, setPermissions] = useState(); + + const debouncedSearch = useCallback( + debounce(async (value: string) => { + try { + setIsSearchLoading(true); + const searchIndices = [ + SearchIndex.DATA_ASSET, + SearchIndex.DOMAIN, + SearchIndex.DATABASE_SERVICE, + SearchIndex.DASHBOARD_SERVICE, + SearchIndex.PIPELINE_SERVICE, + SearchIndex.ML_MODEL_SERVICE, + SearchIndex.STORAGE_SERVICE, + SearchIndex.MESSAGING_SERVICE, + SearchIndex.SEARCH_SERVICE, + SearchIndex.API_SERVICE_INDEX, + ]; + + const response = await searchQuery({ + query: `*${value}*`, + searchIndex: searchIndices, + pageSize: PAGE_SIZE_BASE, + queryFilter: getLineageEntityExclusionFilter(), + }); + + setOptions( + response.hits.hits.map((hit) => ({ + value: hit._source.fullyQualifiedName ?? '', + label: ( + + ), + data: hit, + })) + ); + } finally { + setIsSearchLoading(false); + } + }, 300), + [] + ); + + const handleEntitySelect = useCallback( + (value: EntityReference) => { + history.push( + `/lineage/${(value as SourceType).entityType}/${ + value.fullyQualifiedName + }` + ); + }, + [history] + ); + + const init = useCallback(async () => { + if (!decodedFqn || !entityType) { + setDefaultValue(undefined); + + return; + } + + try { + setLoading(true); + const [entityResponse, permissionResponse] = await Promise.allSettled([ + getEntityAPIfromSource(entityType as AssetsUnion)(decodedFqn), + getEntityPermissionByFqn( + entityType as unknown as ResourceEntity, + decodedFqn + ), + ]); + + if (entityResponse.status === 'fulfilled') { + setSelectedEntity(entityResponse.value); + setDefaultValue(decodedFqn || undefined); + } + + if (permissionResponse.status === 'fulfilled') { + const operationPermission = getOperationPermissions( + permissionResponse.value + ); + setPermissions(operationPermission); + } + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setLoading(false); + } + }, [decodedFqn, entityType]); + + useEffect(() => { + init(); + }, [init]); + + const lineageElement = useMemo(() => { + if (loading) { + return ; + } + + return ( + + + + ); + }, [selectedEntity, loading, permissions, entityType]); + + return ( + + + + + + + + +
+