yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-30 01:23:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Post Call for Aggregate (#20220)

Browse Source 
* Add Post Call for Aggregate

* update aggregation calls to post for lineage

* add platform lineage view

* add unit tests

* cleanup

* update platform lineage page

* add icon for other entities

* cleanup

* Add Lineage Migration

* Move fetch all service to common class

* Add view for service or domain

* remove source field

* add validation for inputs

* add platform lineage page

* fix tests

* Checkstyle fix

* fix tests

* resolved conflicts

* fix pytest

---------

Co-authored-by: Karan Hotchandani <33024356+karanh37@users.noreply.github.com>
Co-authored-by: karanh37 <karanh37@gmail.com>
Co-authored-by: sonikashah <sonikashah94@gmail.com>
This commit is contained in:
Mohit Yadav 2025-03-19 12:07:05 +05:30 committed by GitHub
parent 9c59d6f74a
commit d4631d6d1c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
73 changed files with 1585 additions and 422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -49,6 +49,7 @@ import org.jdbi.v3.core.Jdbi;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.EntityTimeSeriesInterface;
import org.openmetadata.schema.FieldInterface;
import org.openmetadata.schema.ServiceEntityInterface;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
@ -62,6 +63,7 @@ import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.EntityTimeSeriesRepository;
import org.openmetadata.service.jdbi3.FeedRepository;
import org.openmetadata.service.jdbi3.LineageRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.PolicyRepository;
import org.openmetadata.service.jdbi3.Repository;
import org.openmetadata.service.jdbi3.RoleRepository;
@ -719,4 +721,21 @@ public final class Entity {
EntityRepository<?> entityRepository = Entity.getEntityRepository(entityType);
return entityRepository.getAllowedFields().contains(field);
}
public static List<ServiceEntityInterface> getAllServicesForLineage() {
List<ServiceEntityInterface> allServices = new ArrayList<>();
Set<ServiceType> serviceTypes = new HashSet<>(List.of(ServiceType.values()));
serviceTypes.remove(ServiceType.METADATA);
for (ServiceType serviceType : serviceTypes) {
EntityRepository<? extends EntityInterface> repository =
Entity.getServiceEntityRepository(serviceType);
ListFilter filter = new ListFilter(Include.ALL);
List<ServiceEntityInterface> services =
(List<ServiceEntityInterface>) repository.listAll(repository.getFields("id"), filter);
allServices.addAll(services);
}
return allServices;
}
}

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -1243,6 +1243,12 @@ public interface CollectionDAO {
@BindUUID("toId") UUID toId,
@Bind("relation") int relation);
@SqlQuery(
"SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship where relation = :relation ORDER BY fromId, toId LIMIT :limit OFFSET :offset")
@RegisterRowMapper(RelationshipObjectMapper.class)
List<EntityRelationshipObject> getRecordWithOffset(
@Bind("relation") int relation, @Bind("offset") long offset, @Bind("limit") int limit);
//
// Delete Operations
//

26
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java
View File

@ -72,6 +72,7 @@ import org.openmetadata.schema.entity.data.GlossaryTerm;
import org.openmetadata.schema.entity.data.GlossaryTerm.Status;
import org.openmetadata.schema.entity.feed.Thread;
import org.openmetadata.schema.entity.teams.Team;
import org.openmetadata.schema.search.SearchRequest;
import org.openmetadata.schema.type.ApiStatus;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
@ -94,7 +95,6 @@ import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext;
import org.openmetadata.service.resources.feeds.MessageParser;
import org.openmetadata.service.resources.feeds.MessageParser.EntityLink;
import org.openmetadata.service.resources.glossary.GlossaryTermResource;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.security.AuthorizationException;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.EntityUtil.Fields;
@ -491,20 +491,20 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
try {
String key = "_source";
SearchRequest searchRequest =
new SearchRequest.ElasticSearchRequestBuilder(
new SearchRequest()
.withQuery(
String.format(
"** AND (tags.tagFQN:\"%s\")",
ReindexingUtil.escapeDoubleQuotes(glossaryFqn)),
size,
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS))
.from(0)
.fetchSource(true)
.trackTotalHits(false)
.sortFieldParam("_score")
.deleted(false)
.sortOrder("desc")
.includeSourceFields(new ArrayList<>())
.build();
ReindexingUtil.escapeDoubleQuotes(glossaryFqn)))
.withSize(size)
.withIndex(Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS))
.withFrom(0)
.withFetchSource(true)
.withTrackTotalHits(false)
.withSortFieldParam("_score")
.withDeleted(false)
.withSortOrder("desc")
.withIncludeSourceFields(new ArrayList<>());
Response response = searchRepository.search(searchRequest, null);
String json = (String) response.getEntity();
Set<EntityReference> fqns = new TreeSet<>(compareEntityReferenceById);

4
openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java
View File

@ -3,6 +3,8 @@ package org.openmetadata.service.migration.mysql.v170;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.createServiceCharts;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNonNullColumn;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNullColumn;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationForDomainLineage;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationServiceLineage;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateDataInsightsApplication;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateGovernanceWorkflowDefinitions;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateLineageBotPolicy;
@ -28,6 +30,8 @@ public class Migration extends MigrationProcessImpl {
// Lineage
runLineageMigrationForNullColumn(handle);
runLineageMigrationForNonNullColumn(handle);
runMigrationServiceLineage(handle);
runMigrationForDomainLineage(handle);
// DI
createServiceCharts();

4
openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java
View File

@ -3,6 +3,8 @@ package org.openmetadata.service.migration.postgres.v170;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.createServiceCharts;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNonNullColumn;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runLineageMigrationForNullColumn;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationForDomainLineage;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.runMigrationServiceLineage;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateDataInsightsApplication;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateGovernanceWorkflowDefinitions;
import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateLineageBotPolicy;
@ -28,6 +30,8 @@ public class Migration extends MigrationProcessImpl {
// Lineage
runLineageMigrationForNullColumn(handle);
runLineageMigrationForNonNullColumn(handle);
runMigrationServiceLineage(handle);
runMigrationForDomainLineage(handle);
// DI
createServiceCharts();

148
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java
View File

@ -1,9 +1,13 @@
package org.openmetadata.service.migration.utils.v170;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.ADMIN_USER_NAME;
import static org.openmetadata.service.Entity.getAllServicesForLineage;
import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE;
import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@ -11,33 +15,58 @@ import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Handle;
import org.jdbi.v3.core.statement.UnableToExecuteStatementException;
import org.openmetadata.schema.ServiceEntityInterface;
import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart;
import org.openmetadata.schema.dataInsight.custom.LineChart;
import org.openmetadata.schema.dataInsight.custom.LineChartMetric;
import org.openmetadata.schema.entity.domains.Domain;
import org.openmetadata.schema.entity.policies.Policy;
import org.openmetadata.schema.entity.policies.accessControl.Rule;
import org.openmetadata.schema.governance.workflows.WorkflowConfiguration;
import org.openmetadata.schema.governance.workflows.WorkflowDefinition;
import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.LineageDetails;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.governance.workflows.flowable.MainWorkflow;
import org.openmetadata.service.jdbi3.AppMarketPlaceRepository;
import org.openmetadata.service.jdbi3.AppRepository;
import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository;
import org.openmetadata.service.jdbi3.DomainRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.PolicyRepository;
import org.openmetadata.service.jdbi3.WorkflowDefinitionRepository;
import org.openmetadata.service.resources.databases.DatasourceConfig;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;
@Slf4j
public class MigrationUtil {
static final Map<String, List<String>> SERVICE_TYPE_ENTITY_MAP = new HashMap<>();
static {
SERVICE_TYPE_ENTITY_MAP.put(Entity.DATABASE_SERVICE, List.of("table_entity"));
SERVICE_TYPE_ENTITY_MAP.put(Entity.MESSAGING_SERVICE, List.of("topic_entity"));
SERVICE_TYPE_ENTITY_MAP.put(
Entity.DASHBOARD_SERVICE, List.of("dashboard_entity", "dashboard_data_model_entity"));
SERVICE_TYPE_ENTITY_MAP.put(Entity.PIPELINE_SERVICE, List.of("pipeline_entity"));
SERVICE_TYPE_ENTITY_MAP.put(Entity.MLMODEL_SERVICE, List.of("ml_model_entity"));
SERVICE_TYPE_ENTITY_MAP.put(Entity.STORAGE_SERVICE, List.of("storage_container_entity"));
SERVICE_TYPE_ENTITY_MAP.put(Entity.SEARCH_SERVICE, List.of("search_index_entity"));
SERVICE_TYPE_ENTITY_MAP.put(Entity.API_SERVICE, List.of("api_endpoint_entity"));
}
private MigrationUtil() {}
public static final String DOMAIN_LINEAGE =
"select count(*) from entity_relationship where fromId in (select toId from entity_relationship where fromId = '%s' and relation = 10) AND toId in (select toId from entity_relationship where fromId = '%s' and relation = 10) and relation = 13";
public static final String SERVICE_ENTITY_MIGRATION =
"SELECT COUNT(*) FROM entity_relationship er JOIN %s f ON er.fromID = f.id JOIN %s t ON er.toID = t.id WHERE er.relation = 13 AND f.fqnHash LIKE '%s.%%' AND t.fqnHash LIKE '%s.%%'";
private static final String UPDATE_NULL_JSON =
"UPDATE entity_relationship SET json = :json WHERE json IS NULL AND relation = 13";
@ -341,6 +370,125 @@ public class MigrationUtil {
.withName("ai"))));
}
public static void runMigrationForDomainLineage(Handle handle) {
try {
List<Domain> allDomains = getAllDomains();
for (Domain fromDomain : allDomains) {
for (Domain toDomain : allDomains) {
if (fromDomain.getId().equals(toDomain.getId())) {
continue;
}
String sql =
String.format(
DOMAIN_LINEAGE, fromDomain.getId().toString(), toDomain.getId().toString());
int count = handle.createQuery(sql).mapTo(Integer.class).one();
if (count > 0) {
LineageDetails domainLineageDetails =
new LineageDetails()
.withCreatedAt(System.currentTimeMillis())
.withUpdatedAt(System.currentTimeMillis())
.withCreatedBy(ADMIN_USER_NAME)
.withUpdatedBy(ADMIN_USER_NAME)
.withSource(LineageDetails.Source.CHILD_ASSETS)
.withAssetEdges(count);
Entity.getCollectionDAO()
.relationshipDAO()
.insert(
fromDomain.getId(),
toDomain.getId(),
fromDomain.getEntityReference().getType(),
toDomain.getEntityReference().getType(),
Relationship.UPSTREAM.ordinal(),
JsonUtils.pojoToJson(domainLineageDetails));
}
}
}
} catch (Exception ex) {
LOG.error(
"Error while updating null json rows with createdAt, createdBy, updatedAt and updatedBy for lineage.",
ex);
}
}
public static void runMigrationServiceLineage(Handle handle) {
try {
List<ServiceEntityInterface> allServices = getAllServicesForLineage();
for (ServiceEntityInterface fromService : allServices) {
for (ServiceEntityInterface toService : allServices) {
insertServiceLineageDetails(handle, fromService, toService);
}
}
} catch (Exception ex) {
LOG.error(
"Error while updating null json rows with createdAt, createdBy, updatedAt and updatedBy for lineage.",
ex);
}
}
private static void insertServiceLineageDetails(
Handle handle, ServiceEntityInterface fromService, ServiceEntityInterface toService) {
try {
if (fromService.getId().equals(toService.getId())
&& fromService.getServiceType().equals(toService.getServiceType())) {
return;
}
String fromServiceHash = FullyQualifiedName.buildHash(fromService.getFullyQualifiedName());
String toServiceHash = FullyQualifiedName.buildHash(toService.getFullyQualifiedName());
List<String> fromTableNames =
listOrEmpty(SERVICE_TYPE_ENTITY_MAP.get(fromService.getEntityReference().getType()));
List<String> toTableNames =
listOrEmpty(SERVICE_TYPE_ENTITY_MAP.get(toService.getEntityReference().getType()));
for (String fromTableName : fromTableNames) {
for (String toTableName : toTableNames) {
if (!nullOrEmpty(fromTableName) && !nullOrEmpty(toTableName)) {
String sql =
String.format(
SERVICE_ENTITY_MIGRATION,
fromTableName,
toTableName,
fromServiceHash,
toServiceHash);
int count = handle.createQuery(sql).mapTo(Integer.class).one();
if (count > 0) {
LineageDetails serviceLineageDetails =
new LineageDetails()
.withCreatedAt(System.currentTimeMillis())
.withUpdatedAt(System.currentTimeMillis())
.withCreatedBy(ADMIN_USER_NAME)
.withUpdatedBy(ADMIN_USER_NAME)
.withSource(LineageDetails.Source.CHILD_ASSETS)
.withAssetEdges(count);
Entity.getCollectionDAO()
.relationshipDAO()
.insert(
fromService.getId(),
toService.getId(),
fromService.getEntityReference().getType(),
toService.getEntityReference().getType(),
Relationship.UPSTREAM.ordinal(),
JsonUtils.pojoToJson(serviceLineageDetails));
}
}
}
}
} catch (Exception ex) {
LOG.error(
"Found issue while updating lineage for service from {} , to: {}",
fromService.getFullyQualifiedName(),
toService.getFullyQualifiedName(),
ex);
}
}
private static List<Domain> getAllDomains() {
DomainRepository repository = (DomainRepository) Entity.getEntityRepository(Entity.DOMAIN);
return repository.listAll(repository.getFields("id"), new ListFilter(Include.ALL));
}
public static void updateLineageBotPolicy() {
PolicyRepository policyRepository =
(PolicyRepository) Entity.getEntityRepository(Entity.POLICY);

36
openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java
View File

@ -36,6 +36,7 @@ import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.Pattern;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
@ -240,6 +241,41 @@ public class LineageResource {
.withIncludeSourceFields(getRequiredLineageFields(includeSourceFields)));
}
@GET
@Path("/getPlatformLineage")
@Operation(
operationId = "getPlatformLineage",
summary = "Get Platform Lineage",
responses = {
@ApiResponse(
responseCode = "200",
description = "search response",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = SearchResponse.class)))
})
public SearchLineageResult searchLineage(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "view (service or domain)")
@QueryParam("view")
@Pattern(
regexp = "service|domain|all",
message = "Invalid type. Allowed values: service, domain.")
String view,
@Parameter(
description =
"Elasticsearch query that will be combined with the query_string query generator from the `query` argument")
@QueryParam("query_filter")
String queryFilter,
@Parameter(description = "Filter documents by deleted param. By default deleted is false")
@QueryParam("includeDeleted")
boolean deleted)
throws IOException {
return Entity.getSearchRepository().searchPlatformLineage(view, queryFilter, deleted);
}
@GET
@Path("/getLineage/{direction}")
@Operation(

137
openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
View File

@ -15,7 +15,6 @@ package org.openmetadata.service.resources.search;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.jdbi3.RoleRepository.DOMAIN_ONLY_ACCESS_ROLE;
import static org.openmetadata.service.search.SearchRepository.ELASTIC_SEARCH_EXTENSION;
import static org.openmetadata.service.security.DefaultAuthorizer.getSubjectContext;
import es.org.elasticsearch.action.search.SearchResponse;
@ -31,6 +30,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import javax.validation.Valid;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
@ -46,15 +46,14 @@ import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.search.PreviewSearchRequest;
import org.openmetadata.schema.system.EventPublisherJob;
import org.openmetadata.schema.search.SearchRequest;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.search.SearchRepository;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.search.SearchUtils;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.SubjectContext;
import org.openmetadata.service.util.JsonUtils;
@Slf4j
@Path("/v1/search")
@ -195,24 +194,25 @@ public class SearchResource {
}
SearchRequest request =
new SearchRequest.ElasticSearchRequestBuilder(
query, size, Entity.getSearchRepository().getIndexOrAliasName(index))
.from(from)
.queryFilter(queryFilter)
.postFilter(postFilter)
.fetchSource(fetchSource)
.trackTotalHits(trackTotalHits)
.sortFieldParam(sortFieldParam)
.deleted(deleted)
.sortOrder(sortOrder)
.includeSourceFields(includeSourceFields)
.getHierarchy(getHierarchy)
.domains(domains)
.applyDomainFilter(
new SearchRequest()
.withQuery(query)
.withSize(size)
.withIndex(Entity.getSearchRepository().getIndexOrAliasName(index))
.withFrom(from)
.withQueryFilter(queryFilter)
.withPostFilter(postFilter)
.withFetchSource(fetchSource)
.withTrackTotalHits(trackTotalHits)
.withSortFieldParam(sortFieldParam)
.withDeleted(deleted)
.withSortOrder(sortOrder)
.withIncludeSourceFields(includeSourceFields)
.withIsHierarchy(getHierarchy)
.withDomains(domains)
.withApplyDomainFilter(
!subjectContext.isAdmin() && subjectContext.hasAnyRole(DOMAIN_ONLY_ACCESS_ROLE))
.searchAfter(searchAfter)
.explain(explain)
.build();
.withSearchAfter(SearchUtils.searchAfter(searchAfter))
.withExplain(explain);
return searchRepository.search(request, subjectContext);
}
@ -243,20 +243,19 @@ public class SearchResource {
SubjectContext subjectContext = getSubjectContext(securityContext);
SearchRequest searchRequest =
new SearchRequest.ElasticSearchRequestBuilder(
previewRequest.getQuery(),
previewRequest.getSize(),
Entity.getSearchRepository().getIndexOrAliasName(previewRequest.getIndex()))
.from(previewRequest.getFrom())
.queryFilter(previewRequest.getQueryFilter())
.postFilter(previewRequest.getPostFilter())
.fetchSource(previewRequest.getFetchSource())
.trackTotalHits(previewRequest.getTrackTotalHits())
.sortFieldParam(previewRequest.getSortField())
.sortOrder(previewRequest.getSortOrder().value())
.includeSourceFields(previewRequest.getIncludeSourceFields())
.explain(previewRequest.getExplain())
.build();
new SearchRequest()
.withQuery(previewRequest.getQuery())
.withSize(previewRequest.getSize())
.withIndex(Entity.getSearchRepository().getIndexOrAliasName(previewRequest.getIndex()))
.withFrom(previewRequest.getFrom())
.withQueryFilter(previewRequest.getQueryFilter())
.withPostFilter(previewRequest.getPostFilter())
.withFetchSource(previewRequest.getFetchSource())
.withTrackTotalHits(previewRequest.getTrackTotalHits())
.withSortFieldParam(previewRequest.getSortField())
.withSortOrder(previewRequest.getSortOrder().value())
.withIncludeSourceFields(previewRequest.getIncludeSourceFields())
.withExplain(previewRequest.getExplain());
return searchRepository.previewSearch(
searchRequest, subjectContext, previewRequest.getSearchSettings());
@ -296,10 +295,11 @@ public class SearchResource {
SubjectContext subjectContext = getSubjectContext(securityContext);
SearchRequest request =
new SearchRequest.ElasticSearchRequestBuilder(
nlqQuery, size, Entity.getSearchRepository().getIndexOrAliasName(index))
.from(from)
.build();
new SearchRequest()
.withQuery(nlqQuery)
.withSize(size)
.withIndex(Entity.getSearchRepository().getIndexOrAliasName(index))
.withFrom(from);
return searchRepository.searchWithNLQ(request, subjectContext);
}
@ -437,12 +437,14 @@ public class SearchResource {
}
SearchRequest request =
new SearchRequest.ElasticSearchRequestBuilder(query, size, index)
.fieldName(fieldName)
.deleted(deleted)
.fetchSource(fetchSource)
.includeSourceFields(includeSourceFields)
.build();
new SearchRequest()
.withQuery(query)
.withSize(size)
.withIndex(index)
.withFieldName(fieldName)
.withDeleted(deleted)
.withFetchSource(fetchSource)
.withIncludeSourceFields(includeSourceFields);
return searchRepository.suggest(request);
}
@ -498,31 +500,30 @@ public class SearchResource {
return searchRepository.aggregate(index, fieldName, value, query);
}
@GET
@Path("/reindex/stream/status")
@POST
@Path("/aggregate")
@Operation(
operationId = "getStreamJobStatus",
summary = "Get Stream Job Latest Status",
description = "Stream Job Status",
operationId = "aggregateSearchRequest",
summary = "Get aggregated Search Request",
description = "Get aggregated fields from entities.",
responses = {
@ApiResponse(responseCode = "200", description = "Success"),
@ApiResponse(responseCode = "404", description = "Status not found")
@ApiResponse(
responseCode = "200",
description = "Table Aggregate API",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = SearchResponse.class)))
})
public Response reindexAllJobLastStatus(
@Context UriInfo uriInfo, @Context SecurityContext securityContext) {
// Only admins can issue a reindex request
authorizer.authorizeAdmin(securityContext);
// Check if there is a running job for reindex for requested entity
String jobRecord;
jobRecord =
Entity.getCollectionDAO()
.entityExtensionTimeSeriesDao()
.getLatestExtension(ELASTIC_SEARCH_ENTITY_FQN_STREAM, ELASTIC_SEARCH_EXTENSION);
if (jobRecord != null) {
return Response.status(Response.Status.OK)
.entity(JsonUtils.readValue(jobRecord, EventPublisherJob.class))
.build();
}
return Response.status(Response.Status.NOT_FOUND).entity("No Last Run.").build();
public Response aggregateSearchRequest(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Valid SearchRequest searchRequest)
throws IOException {
return searchRepository.aggregate(
searchRequest.getIndex(),
searchRequest.getFieldName(),
searchRequest.getFieldValue(),
searchRequest.getQuery());
}
}

4
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
View File

@ -19,6 +19,7 @@ import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart;
import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList;
import org.openmetadata.schema.entity.data.QueryCostSearchResult;
import org.openmetadata.schema.search.SearchRequest;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.schema.tests.DataQualityReport;
import org.openmetadata.schema.type.EntityReference;
@ -216,6 +217,9 @@ public interface SearchClient {
SearchLineageResult searchLineageWithDirection(SearchLineageRequest lineageRequest)
throws IOException;
SearchLineageResult searchPlatformLineage(String index, String queryFilter, boolean deleted)
throws IOException;
Response searchEntityRelationship(
String fqn, int upstreamDepth, int downstreamDepth, String queryFilter, boolean deleted)
throws IOException;

29
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
View File

@ -74,6 +74,7 @@ import org.openmetadata.schema.api.search.SearchSettings;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.entity.classification.Tag;
import org.openmetadata.schema.entity.data.QueryCostSearchResult;
import org.openmetadata.schema.search.SearchRequest;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.schema.service.configuration.elasticsearch.NaturalLanguageSearchConfiguration;
import org.openmetadata.schema.tests.DataQualityReport;
@ -1092,6 +1093,11 @@ public class SearchRepository {
return searchClient.searchLineage(lineageRequest);
}
public SearchLineageResult searchPlatformLineage(
String alias, String queryFilter, boolean deleted) throws IOException {
return searchClient.searchPlatformLineage(alias, queryFilter, deleted);
}
public SearchLineageResult searchLineageWithDirection(SearchLineageRequest lineageRequest)
throws IOException {
return searchClient.searchLineageWithDirection(lineageRequest);
@ -1184,17 +1190,18 @@ public class SearchRepository {
ReindexingUtil.escapeDoubleQuotes(entityFQN));
SearchRequest searchRequest =
new SearchRequest.ElasticSearchRequestBuilder(
"", size, Entity.getSearchRepository().getIndexOrAliasName(indexName))
.from(0)
.queryFilter(queryFilter)
.fetchSource(true)
.trackTotalHits(false)
.sortFieldParam("_score")
.deleted(false)
.sortOrder("desc")
.includeSourceFields(new ArrayList<>())
.build();
new SearchRequest()
.withQuery("")
.withSize(size)
.withIndex(Entity.getSearchRepository().getIndexOrAliasName(indexName))
.withFrom(0)
.withQueryFilter(queryFilter)
.withFetchSource(true)
.withTrackTotalHits(false)
.withSortFieldParam("_score")
.withDeleted(false)
.withSortOrder("desc")
.withIncludeSourceFields(new ArrayList<>());
// Execute the search and parse the response
Response response = search(searchRequest, null);

168
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java
View File

@ -1,168 +0,0 @@
package org.openmetadata.service.search;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.Getter;
import lombok.Setter;
import org.openmetadata.schema.type.EntityReference;
@Getter
@Setter
public class SearchRequest {
private final String query;
private int from;
private final int size;
private final String queryFilter;
private final String postFilter;
private final boolean fetchSource;
private final boolean trackTotalHits;
private final String sortFieldParam;
private final boolean deleted;
private final String index;
private final String fieldName;
private final String sortOrder;
private final List<String> includeSourceFields;
private final boolean applyDomainFilter;
private final List<String> domains;
private final boolean getHierarchy;
private final Object[] searchAfter;
private final boolean explain;
public SearchRequest(ElasticSearchRequestBuilder builder) {
this.query = builder.query;
this.from = builder.from;
this.size = builder.size;
this.queryFilter = builder.queryFilter;
this.postFilter = builder.postFilter;
this.fetchSource = builder.fetchSource;
this.trackTotalHits = builder.trackTotalHits;
this.sortFieldParam = builder.sortFieldParam;
this.deleted = builder.deleted;
this.index = builder.index;
this.sortOrder = builder.sortOrder;
this.includeSourceFields = builder.includeSourceFields;
this.fieldName = builder.fieldName;
this.getHierarchy = builder.getHierarchy;
this.domains = builder.domains;
this.applyDomainFilter = builder.applyDomainFilter;
this.searchAfter = builder.searchAfter;
this.explain = builder.explain;
}
// Builder class for ElasticSearchRequest
public static class ElasticSearchRequestBuilder {
private final String index;
private final String query;
private final int size;
private int from;
private String fieldName;
private String queryFilter;
private String postFilter;
private boolean fetchSource;
private boolean trackTotalHits;
private String sortFieldParam;
private boolean deleted;
private String sortOrder;
private List<String> includeSourceFields;
private boolean getHierarchy;
private boolean applyDomainFilter;
private List<String> domains;
private Object[] searchAfter;
private boolean explain;
public ElasticSearchRequestBuilder(String query, int size, String index) {
this.query = query;
this.size = size;
this.index = index;
}
public ElasticSearchRequestBuilder from(int from) {
this.from = from;
return this;
}
public ElasticSearchRequestBuilder queryFilter(String queryFilter) {
this.queryFilter = queryFilter;
return this;
}
public ElasticSearchRequestBuilder postFilter(String postFilter) {
this.postFilter = postFilter;
return this;
}
public ElasticSearchRequestBuilder fetchSource(boolean fetchSource) {
this.fetchSource = fetchSource;
return this;
}
public ElasticSearchRequestBuilder trackTotalHits(boolean trackTotalHits) {
this.trackTotalHits = trackTotalHits;
return this;
}
public ElasticSearchRequestBuilder sortFieldParam(String sortFieldParam) {
this.sortFieldParam = sortFieldParam;
return this;
}
public ElasticSearchRequestBuilder deleted(boolean deleted) {
this.deleted = deleted;
return this;
}
public ElasticSearchRequestBuilder applyDomainFilter(boolean applyDomainFilter) {
this.applyDomainFilter = applyDomainFilter;
return this;
}
public ElasticSearchRequestBuilder sortOrder(String sortOrder) {
this.sortOrder = sortOrder;
return this;
}
public ElasticSearchRequestBuilder includeSourceFields(List<String> includeSourceFields) {
this.includeSourceFields = includeSourceFields;
return this;
}
public ElasticSearchRequestBuilder fieldName(String fieldName) {
this.fieldName = fieldName;
return this;
}
public ElasticSearchRequestBuilder getHierarchy(boolean getHierarchy) {
this.getHierarchy = getHierarchy;
return this;
}
public ElasticSearchRequestBuilder domains(List<EntityReference> references) {
this.domains =
references.stream()
.map(EntityReference::getFullyQualifiedName)
.collect(Collectors.toList());
return this;
}
public ElasticSearchRequestBuilder searchAfter(String searchAfter) {
this.searchAfter = null;
if (!nullOrEmpty(searchAfter)) {
this.searchAfter = Stream.of(searchAfter.split(",")).toArray(Object[]::new);
}
return this;
}
public ElasticSearchRequestBuilder explain(boolean explain) {
this.explain = explain;
return this;
}
public SearchRequest build() {
return new SearchRequest(this);
}
}
}

7
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchUtils.java
View File

@ -163,4 +163,11 @@ public final class SearchUtils {
Set.of("fullyQualifiedName", "service", "fqnHash", "id", "entityType", "upstreamLineage"));
return requiredFields;
}
public static List<Object> searchAfter(String searchAfter) {
if (!nullOrEmpty(searchAfter)) {
return List.of(searchAfter.split(","));
}
return null;
}
}

31
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ESLineageGraphBuilder.java
View File

@ -1,5 +1,6 @@
package org.openmetadata.service.search.elasticsearch;
import static org.openmetadata.common.utils.CommonUtil.collectionOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.FIELD_FULLY_QUALIFIED_NAME_HASH_KEYWORD;
import static org.openmetadata.service.search.SearchClient.FQN_FIELD;
@ -21,6 +22,7 @@ import es.org.elasticsearch.search.SearchHit;
import es.org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import es.org.elasticsearch.search.aggregations.bucket.terms.Terms;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -44,6 +46,35 @@ public class ESLineageGraphBuilder {
this.esClient = esClient;
}
public SearchLineageResult getPlatformLineage(String index, String queryFilter, boolean deleted)
throws IOException {
SearchLineageResult result =
new SearchLineageResult()
.withNodes(new HashMap<>())
.withUpstreamEdges(new HashMap<>())
.withDownstreamEdges(new HashMap<>());
SearchResponse searchResponse = EsUtils.searchEntities(index, queryFilter, deleted);
// Add Nodes
Arrays.stream(searchResponse.getHits().getHits())
.map(hit -> collectionOrEmpty(hit.getSourceAsMap()))
.forEach(
sourceMap -> {
String fqn = sourceMap.get(FQN_FIELD).toString();
result.getNodes().putIfAbsent(fqn, new NodeInformation().withEntity(sourceMap));
List<EsLineageData> upstreamLineageList = getUpstreamLineageListIfExist(sourceMap);
for (EsLineageData esLineageData : upstreamLineageList) {
result
.getUpstreamEdges()
.putIfAbsent(
esLineageData.getDocId(),
esLineageData.withToEntity(getRelationshipRef(sourceMap)));
}
});
return result;
}
public SearchLineageResult getUpstreamLineage(SearchLineageRequest request) throws IOException {
SearchLineageResult result =
new SearchLineageResult()

28
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -139,6 +139,7 @@ import org.openmetadata.schema.dataInsight.custom.FormulaHolder;
import org.openmetadata.schema.entity.data.EntityHierarchy;
import org.openmetadata.schema.entity.data.QueryCostSearchResult;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.search.SearchRequest;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.schema.tests.DataQualityReport;
@ -160,7 +161,6 @@ import org.openmetadata.service.search.SearchAggregation;
import org.openmetadata.service.search.SearchClient;
import org.openmetadata.service.search.SearchHealthStatus;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.search.SearchResultListMapper;
import org.openmetadata.service.search.SearchSortFilter;
import org.openmetadata.service.search.UpdateSearchEventsConstant;
@ -393,7 +393,7 @@ public class ElasticSearchClient implements SearchClient {
}
if (!nullOrEmpty(request.getSearchAfter())) {
searchSourceBuilder.searchAfter(request.getSearchAfter());
searchSourceBuilder.searchAfter(request.getSearchAfter().toArray());
}
/* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */
@ -409,7 +409,7 @@ public class ElasticSearchClient implements SearchClient {
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
.must(QueryBuilders.existsQuery("deleted"))
.must(QueryBuilders.termQuery("deleted", request.isDeleted())));
.must(QueryBuilders.termQuery("deleted", request.getDeleted())));
boolQueryBuilder.should(
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
@ -450,10 +450,10 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder.query(
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
.must(QueryBuilders.termQuery("deleted", request.isDeleted())));
.must(QueryBuilders.termQuery("deleted", request.getDeleted())));
}
if (!nullOrEmpty(request.getSortFieldParam()) && !request.isGetHierarchy()) {
if (!nullOrEmpty(request.getSortFieldParam()) && !request.getIsHierarchy()) {
FieldSortBuilder fieldSortBuilder =
new FieldSortBuilder(request.getSortFieldParam())
.order(SortOrder.fromString(request.getSortOrder()));
@ -482,11 +482,11 @@ public class ElasticSearchClient implements SearchClient {
https://github.com/elastic/elasticsearch/issues/33028 */
searchSourceBuilder.fetchSource(
new FetchSourceContext(
request.isFetchSource(),
request.getFetchSource(),
request.getIncludeSourceFields().toArray(String[]::new),
new String[] {}));
if (request.isTrackTotalHits()) {
if (request.getTrackTotalHits()) {
searchSourceBuilder.trackTotalHits(true);
} else {
searchSourceBuilder.trackTotalHitsUpTo(MAX_RESULT_HITS);
@ -502,7 +502,7 @@ public class ElasticSearchClient implements SearchClient {
.source(searchSourceBuilder),
RequestOptions.DEFAULT);
if (!request.isGetHierarchy()) {
if (!request.getIsHierarchy()) {
return Response.status(OK).entity(searchResponse.toString()).build();
} else {
// Build the nested hierarchy from elastic search response
@ -546,7 +546,7 @@ public class ElasticSearchClient implements SearchClient {
SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client)
throws IOException {
if (!request.isGetHierarchy()) {
if (!request.getIsHierarchy()) {
return;
}
@ -958,6 +958,12 @@ public class ElasticSearchClient implements SearchClient {
}
}
@Override
public SearchLineageResult searchPlatformLineage(
String index, String queryFilter, boolean deleted) throws IOException {
return lineageGraphBuilder.getPlatformLineage(index, queryFilter, deleted);
}
private void getEntityRelationship(
String fqn,
int depth,
@ -1457,7 +1463,7 @@ public class ElasticSearchClient implements SearchClient {
"deleted",
Collections.singletonList(
CategoryQueryContext.builder()
.setCategory(String.valueOf(request.isDeleted()))
.setCategory(String.valueOf(request.getDeleted()))
.build())));
}
SuggestBuilder suggestBuilder = new SuggestBuilder();
@ -1467,7 +1473,7 @@ public class ElasticSearchClient implements SearchClient {
.timeout(new TimeValue(30, TimeUnit.SECONDS))
.fetchSource(
new FetchSourceContext(
request.isFetchSource(),
request.getFetchSource(),
request.getIncludeSourceFields().toArray(String[]::new),
new String[] {}));
es.org.elasticsearch.action.search.SearchRequest searchRequest =

18
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUtils.java
View File

@ -140,6 +140,24 @@ public class EsUtils {
return boolQuery;
}
public static SearchResponse searchEntities(String index, String queryFilter, Boolean deleted)
throws IOException {
es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(index));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(
QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("deleted", !nullOrEmpty(deleted) && deleted)));
buildSearchSourceFilter(queryFilter, searchSourceBuilder);
searchRequest.source(searchSourceBuilder.size(10000));
RestHighLevelClient client =
(RestHighLevelClient) Entity.getSearchRepository().getSearchClient().getClient();
return client.search(searchRequest, RequestOptions.DEFAULT);
}
public static void buildSearchSourceFilter(
String queryFilter, SearchSourceBuilder searchSourceBuilder) {
if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) {

2
openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NLQService.java
View File

@ -1,7 +1,7 @@
package org.openmetadata.service.search.nlq;
import java.io.IOException;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.schema.search.SearchRequest;
/**
* Interface for Natural Language Query (NLQ) processing services.

2
openmetadata-service/src/main/java/org/openmetadata/service/search/nlq/NoOpNLQService.java
View File

@ -3,7 +3,7 @@ package org.openmetadata.service.search.nlq;
import java.io.IOException;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.api.search.NLQConfiguration;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.schema.search.SearchRequest;
/**
* A no-operation implementation of NLQService that returns null/empty responses.

31
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OSLineageGraphBuilder.java
View File

@ -1,5 +1,6 @@
package org.openmetadata.service.search.opensearch;
import static org.openmetadata.common.utils.CommonUtil.collectionOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.FIELD_FULLY_QUALIFIED_NAME_HASH_KEYWORD;
import static org.openmetadata.service.search.SearchClient.FQN_FIELD;
@ -15,6 +16,7 @@ import static org.openmetadata.service.search.opensearch.OsUtils.getSearchReques
import com.nimbusds.jose.util.Pair;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -43,6 +45,35 @@ public class OSLineageGraphBuilder {
this.esClient = esClient;
}
public SearchLineageResult getPlatformLineage(String index, String queryFilter, boolean deleted)
throws IOException {
SearchLineageResult result =
new SearchLineageResult()
.withNodes(new HashMap<>())
.withUpstreamEdges(new HashMap<>())
.withDownstreamEdges(new HashMap<>());
SearchResponse searchResponse = OsUtils.searchEntities(index, queryFilter, deleted);
// Add Nodes
Arrays.stream(searchResponse.getHits().getHits())
.map(hit -> collectionOrEmpty(hit.getSourceAsMap()))
.forEach(
sourceMap -> {
String fqn = sourceMap.get(FQN_FIELD).toString();
result.getNodes().putIfAbsent(fqn, new NodeInformation().withEntity(sourceMap));
List<EsLineageData> upstreamLineageList = getUpstreamLineageListIfExist(sourceMap);
for (EsLineageData esLineageData : upstreamLineageList) {
result
.getUpstreamEdges()
.putIfAbsent(
esLineageData.getDocId(),
esLineageData.withToEntity(getRelationshipRef(sourceMap)));
}
});
return result;
}
public SearchLineageResult getUpstreamLineage(SearchLineageRequest request) throws IOException {
if (request.getLayerFrom() < 0 || request.getLayerSize() < 0) {
throw new IllegalArgumentException(

32
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -79,6 +79,7 @@ import org.openmetadata.schema.dataInsight.custom.FormulaHolder;
import org.openmetadata.schema.entity.data.EntityHierarchy;
import org.openmetadata.schema.entity.data.QueryCostSearchResult;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.search.SearchRequest;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.schema.tests.DataQualityReport;
@ -100,7 +101,6 @@ import org.openmetadata.service.search.SearchAggregation;
import org.openmetadata.service.search.SearchClient;
import org.openmetadata.service.search.SearchHealthStatus;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.search.SearchResultListMapper;
import org.openmetadata.service.search.SearchSortFilter;
import org.openmetadata.service.search.models.IndexMapping;
@ -419,7 +419,7 @@ public class OpenSearchClient implements SearchClient {
}
if (!nullOrEmpty(request.getSearchAfter())) {
searchSourceBuilder.searchAfter(request.getSearchAfter());
searchSourceBuilder.searchAfter(request.getSearchAfter().toArray());
}
/* For backward-compatibility we continue supporting the deleted argument, this should be removed in future versions */
@ -434,7 +434,7 @@ public class OpenSearchClient implements SearchClient {
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
.must(QueryBuilders.existsQuery("deleted"))
.must(QueryBuilders.termQuery("deleted", request.isDeleted())));
.must(QueryBuilders.termQuery("deleted", request.getDeleted())));
boolQueryBuilder.should(
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
@ -475,10 +475,10 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder.query(
QueryBuilders.boolQuery()
.must(searchSourceBuilder.query())
.must(QueryBuilders.termQuery("deleted", request.isDeleted())));
.must(QueryBuilders.termQuery("deleted", request.getDeleted())));
}
if (!nullOrEmpty(request.getSortFieldParam()) && !request.isGetHierarchy()) {
if (!nullOrEmpty(request.getSortFieldParam()) && !request.getIsHierarchy()) {
FieldSortBuilder fieldSortBuilder =
new FieldSortBuilder(request.getSortFieldParam())
.order(SortOrder.fromString(request.getSortOrder()));
@ -507,18 +507,18 @@ public class OpenSearchClient implements SearchClient {
https://github.com/Open/Opensearch/issues/33028 */
searchSourceBuilder.fetchSource(
new FetchSourceContext(
request.isFetchSource(),
request.getFetchSource(),
request.getIncludeSourceFields().toArray(String[]::new),
new String[] {}));
if (request.isTrackTotalHits()) {
if (request.getTrackTotalHits()) {
searchSourceBuilder.trackTotalHits(true);
} else {
searchSourceBuilder.trackTotalHitsUpTo(MAX_RESULT_HITS);
}
searchSourceBuilder.timeout(new TimeValue(30, TimeUnit.SECONDS));
if (request.isExplain()) {
if (request.getExplain()) {
searchSourceBuilder.explain(true);
}
try {
@ -528,8 +528,8 @@ public class OpenSearchClient implements SearchClient {
client.search(
new os.org.opensearch.action.search.SearchRequest(request.getIndex())
.source(searchSourceBuilder),
OPENSEARCH_REQUEST_OPTIONS);
if (!request.isGetHierarchy()) {
RequestOptions.DEFAULT);
if (Boolean.FALSE.equals(request.getIsHierarchy())) {
return Response.status(OK).entity(searchResponse.toString()).build();
} else {
List<?> response = buildSearchHierarchy(request, searchResponse);
@ -662,7 +662,7 @@ public class OpenSearchClient implements SearchClient {
SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client)
throws IOException {
if (!request.isGetHierarchy()) {
if (!request.getIsHierarchy()) {
return;
}
@ -1029,6 +1029,12 @@ public class OpenSearchClient implements SearchClient {
}
}
@Override
public SearchLineageResult searchPlatformLineage(
String index, String queryFilter, boolean deleted) throws IOException {
return lineageGraphBuilder.getPlatformLineage(index, queryFilter, deleted);
}
private void getEntityRelationship(
String fqn,
int depth,
@ -1575,7 +1581,7 @@ public class OpenSearchClient implements SearchClient {
"deleted",
Collections.singletonList(
CategoryQueryContext.builder()
.setCategory(String.valueOf(request.isDeleted()))
.setCategory(String.valueOf(request.getDeleted()))
.build())));
}
SuggestBuilder suggestBuilder = new SuggestBuilder();
@ -1585,7 +1591,7 @@ public class OpenSearchClient implements SearchClient {
.timeout(new TimeValue(30, TimeUnit.SECONDS))
.fetchSource(
new FetchSourceContext(
request.isFetchSource(),
request.getFetchSource(),
request.getIncludeSourceFields().toArray(String[]::new),
new String[] {}));
os.org.opensearch.action.search.SearchRequest searchRequest =

18
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUtils.java
View File

@ -140,6 +140,24 @@ public class OsUtils {
return boolQuery;
}
public static os.org.opensearch.action.search.SearchResponse searchEntities(
String index, String queryFilter, Boolean deleted) throws IOException {
os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(
Entity.getSearchRepository().getIndexOrAliasName(index));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(
QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("deleted", !nullOrEmpty(deleted) && deleted)));
buildSearchSourceFilter(queryFilter, searchSourceBuilder);
searchRequest.source(searchSourceBuilder.size(10000));
RestHighLevelClient client =
(RestHighLevelClient) Entity.getSearchRepository().getSearchClient().getClient();
return client.search(searchRequest, RequestOptions.DEFAULT);
}
public static void buildSearchSourceFilter(
String queryFilter, SearchSourceBuilder searchSourceBuilder) {
if (!nullOrEmpty(queryFilter) && !queryFilter.equals("{}")) {

25
openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java
View File

@ -27,6 +27,7 @@ import javax.ws.rs.core.Response;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.search.SearchRequest;
import org.openmetadata.schema.system.EntityError;
import org.openmetadata.schema.system.Stats;
import org.openmetadata.schema.system.StepStats;
@ -35,7 +36,6 @@ import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.EntityTimeSeriesRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.util.JsonUtils;
import os.org.opensearch.action.bulk.BulkItemResponse;
import os.org.opensearch.action.bulk.BulkResponse;
@ -148,18 +148,17 @@ public class ReindexingUtil {
String matchingKey, String sourceFqn, int from) {
String key = "_source";
SearchRequest searchRequest =
new SearchRequest.ElasticSearchRequestBuilder(
String.format("(%s:\"%s\")", matchingKey, sourceFqn),
100,
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS))
.from(from)
.fetchSource(true)
.trackTotalHits(false)
.sortFieldParam("_score")
.deleted(false)
.sortOrder("desc")
.includeSourceFields(new ArrayList<>())
.build();
new SearchRequest()
.withQuery(String.format("(%s:\"%s\")", matchingKey, sourceFqn))
.withSize(100)
.withIndex(Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS))
.withFrom(from)
.withFetchSource(true)
.withTrackTotalHits(false)
.withSortFieldParam("_score")
.withDeleted(false)
.withSortOrder("desc")
.withIncludeSourceFields(new ArrayList<>());
List<EntityReference> entities = new ArrayList<>();
Response response = Entity.getSearchRepository().search(searchRequest, null);
String json = (String) response.getEntity();

16
openmetadata-service/src/main/resources/elasticsearch/indexMapping.json
View File

@ -213,42 +213,42 @@
"indexName": "database_service_search_index",
"indexMappingFile": "/elasticsearch/%s/database_service_index_mapping.json",
"alias": "databaseService",
"parentAliases": ["all"],
"parentAliases": ["all", "service"],
"childAliases": ["database", "databaseSchema", "storedProcedure", "table", "testSuite", "testCase", "testCaseResolutionStatus", "testCaseResult"]
},
"messagingService": {
"indexName": "messaging_service_search_index",
"indexMappingFile": "/elasticsearch/%s/messaging_service_index_mapping.json",
"alias": "messagingService",
"parentAliases": ["all"],
"parentAliases": ["all", "service"],
"childAliases": ["topic"]
},
"pipelineService": {
"indexName": "pipeline_service_search_index",
"indexMappingFile": "/elasticsearch/%s/pipeline_service_index_mapping.json",
"alias": "pipelineService",
"parentAliases": ["all"],
"parentAliases": ["all", "service"],
"childAliases": ["pipeline"]
},
"dashboardService": {
"indexName": "dashboard_service_search_index",
"indexMappingFile": "/elasticsearch/%s/dashboard_service_index_mapping.json",
"alias": "dashboardService",
"parentAliases": ["all"],
"parentAliases": ["all", "service"],
"childAliases": ["dashboard", "dashboardDataModel", "chart"]
},
"searchService": {
"indexName": "search_service_search_index",
"indexMappingFile": "/elasticsearch/%s/search_service_index_mapping.json",
"alias": "searchService",
"parentAliases": ["all"],
"parentAliases": ["all", "service"],
"childAliases": ["searchIndex"]
},
"storageService": {
"indexName": "storage_service_search_index",
"indexMappingFile": "/elasticsearch/%s/storage_service_index_mapping.json",
"alias": "storageService",
"parentAliases": ["all"],
"parentAliases": ["all", "service"],
"childAliases": ["container"]
},
"metadataService": {
@ -262,14 +262,14 @@
"indexName": "mlmodel_service_search_index",
"indexMappingFile": "/elasticsearch/%s/mlmodel_service_index_mapping.json",
"alias": "mlModelService",
"parentAliases": ["all"],
"parentAliases": ["all", "service"],
"childAliases": ["mlmodel"]
},
"apiService": {
"indexName": "api_service_search_index",
"indexMappingFile": "/elasticsearch/%s/api_service_index_mapping.json",
"alias": "apiService",
"parentAliases": ["all"],
"parentAliases": ["all", "service"],
"childAliases": ["apiCollection", "apiEndpoint"]
},
"apiCollection": {

102
openmetadata-spec/src/main/resources/json/schema/search/searchRequest.json Normal file
View File

@ -0,0 +1,102 @@
{
"$id": "https://open-metadata.org/schema/search/searchRequest.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "SearchRequest",
"description": "Search Request to find entities from Elastic Search based on different parameters.",
"javaType": "org.openmetadata.schema.search.SearchRequest",
"type": "object",
"properties": {
"query": {
"description": "Query to be send to Search Engine.",
"type": "string",
"default": "*"
},
"index": {
"description": "Index Name.",
"type": "string",
"default": "table_search_index"
},
"fieldName": {
"description": "Field Name to match.",
"type": "string",
"default": "suggest"
},
"from": {
"description": "Start Index for the req.",
"type": "integer",
"default": 0
},
"size": {
"description": "Size to limit the no.of results returned.",
"type": "integer",
"default": 10
},
"queryFilter": {
"description": "Elasticsearch query that will be combined with the query_string query generator from the `query` arg",
"type": "string"
},
"postFilter": {
"description": "Elasticsearch query that will be used as a post_filter",
"type": "string"
},
"fetchSource": {
"description": "Get document body for each hit",
"type": "boolean",
"default": true
},
"trackTotalHits": {
"description": "Track Total Hits.",
"type": "boolean",
"default": false
},
"explain": {
"description": "Explain the results of the query. Defaults to false. Only for debugging purposes.",
"type": "boolean",
"default": false
},
"deleted": {
"description": "Filter documents by deleted param.",
"type": "boolean",
"default": false
},
"sortFieldParam": {
"description": "Sort the search results by field, available fields to sort weekly_stats daily_stats, monthly_stats, last_updated_timestamp.",
"type": "string",
"default": "_score"
},
"sortOrder": {
"description": "Sort order asc for ascending or desc for descending, defaults to desc.",
"type": "string",
"default": "desc"
},
"includeSourceFields": {
"description": "Get only selected fields of the document body for each hit. Empty value will return all fields",
"type": "array",
"items": {
"type": "string"
}
},
"searchAfter": {
"description": "When paginating, specify the search_after values. Use it ass search_after=<val1>,<val2>,...",
"existingJavaType": "java.util.List<java.lang.Object>"
},
"domains": {
"description": "Internal Object to filter by Domains.",
"existingJavaType": "java.util.List<org.openmetadata.schema.type.EntityReference>"
},
"applyDomainFilter": {
"description": "If Need to apply the domain filter.",
"type": "boolean"
},
"isHierarchy": {
"description": "If true it will try to get the hierarchy of the entity.",
"type": "boolean",
"default": false
},
"fieldValue": {
"description": "Field Value in case of Aggregations.",
"type": "string"
}
},
"additionalProperties": false
}

2
openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts
View File

@ -25,6 +25,7 @@ export enum SidebarItem {
SETTINGS = 'settings',
LOGOUT = 'logout',
METRICS = 'metrics',
LINEAGE = 'lineage',
}
export const SIDEBAR_LIST_ITEMS = {
@ -43,6 +44,7 @@ export const SIDEBAR_LIST_ITEMS = {
[SidebarItem.GLOSSARY]: [SidebarItem.GOVERNANCE, SidebarItem.GLOSSARY],
[SidebarItem.TAGS]: [SidebarItem.GOVERNANCE, SidebarItem.TAGS],
[SidebarItem.METRICS]: [SidebarItem.GOVERNANCE, SidebarItem.METRICS],
[SidebarItem.LINEAGE]: [SidebarItem.GOVERNANCE, SidebarItem.LINEAGE],
// Profile Dropdown
'user-name': ['dropdown-profile', 'user-name'],

40
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts
View File

@ -12,6 +12,7 @@
*/
import test, { expect } from '@playwright/test';
import { get } from 'lodash';
import { SidebarItem } from '../../constant/sidebar';
import { ApiEndpointClass } from '../../support/entity/ApiEndpointClass';
import { ContainerClass } from '../../support/entity/ContainerClass';
import { DashboardClass } from '../../support/entity/DashboardClass';
@ -46,6 +47,7 @@ import {
verifyNodePresent,
visitLineageTab,
} from '../../utils/lineage';
import { sidebarClick } from '../../utils/sidebar';
// use the admin user to login
test.use({
@ -200,6 +202,18 @@ test('Verify column lineage between table and topic', async ({ browser }) => {
const topic = new TopicClass();
await Promise.all([table.create(apiContext), topic.create(apiContext)]);
const tableServiceFqn = get(
table,
'entityResponseData.service.fullyQualifiedName'
);
const tableServiceName = get(table, 'entityResponseData.service.name');
const topicServiceFqn = get(
topic,
'entityResponseData.service.fullyQualifiedName'
);
const sourceTableFqn = get(table, 'entityResponseData.fullyQualifiedName');
const sourceCol = `${sourceTableFqn}.${get(
table,
@ -222,6 +236,32 @@ test('Verify column lineage between table and topic', async ({ browser }) => {
await visitLineageTab(page);
await verifyColumnLineageInCSV(page, table, topic, sourceCol, targetCol);
await test.step('Verify relation in platform lineage', async () => {
await sidebarClick(page, SidebarItem.LINEAGE);
const searchRes = page.waitForResponse('/api/v1/search/query?*');
await page.click('[data-testid="search-entity-select"]');
await page.keyboard.type(tableServiceFqn);
await searchRes;
await page.click(`[data-testid="node-suggestion-${tableServiceFqn}"]`);
await page.waitForLoadState('networkidle');
const tableServiceNode = page.locator(
`[data-testid="lineage-node-${tableServiceFqn}"]`
);
const topicServiceNode = page.locator(
`[data-testid="lineage-node-${topicServiceFqn}"]`
);
await expect(tableServiceNode).toBeVisible();
await expect(topicServiceNode).toBeVisible();
});
await redirectToHomePage(page);
await table.visitEntityPage(page);
await visitLineageTab(page);
await page.click('[data-testid="edit-lineage"]');
await removeColumnLineage(page, sourceCol, targetCol);

1
openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-platform-lineage.svg Normal file
View File

@ -0,0 +1 @@
<svg viewBox="-15 -15 130 130" xmlns="http://www.w3.org/2000/svg"><path fill="currentColor" d="m58.988 25.051c0-1.104-.896-2-2-2h-13.975c-1.104 0-2 .896-2 2s.896 2 2 2h13.976c1.104 0 1.999-.896 1.999-2z"></path><path fill="currentColor" d="m11.113 86.063h13.976c1.104 0 2-.896 2-2s-.896-2-2-2h-13.976c-1.104 0-2 .896-2 2s.896 2 2 2z"></path><path fill="currentColor" d="m98 63.683h-14.102v-11.259c0-1.104-.896-2-2-2h-29.898v-14.109h14.102c1.104 0 2-.896 2-2v-18.531-9.113c0-1.104-.896-2-2-2h-32.203c-1.104 0-2 .896-2 2v9.113 18.531c0 1.104.896 2 2 2h14.101v14.108h-29.899c-1.104 0-2 .896-2 2v11.259h-14.101c-1.104 0-2 .896-2 2v9.116 18.53c0 1.104.896 2 2 2h32.203c1.104 0 2-.896 2-2v-18.53-9.116c0-1.104-.896-2-2-2h-14.102v-9.259h29.899 29.898v9.259h-14.1c-1.104 0-2 .896-2 2v9.116 18.53c0 1.104.896 2 2 2h32.202c1.104 0 2-.896 2-2v-18.53-9.116c0-1.104-.896-1.999-2-1.999zm-62.101-55.012h28.202v5.113h-28.202zm0 9.113h28.202v14.531h-28.202zm-3.696 73.545h-28.203v-14.53h28.203zm0-18.53h-28.203v-5.116h28.203zm35.596-5.116h28.201v5.116h-28.201zm28.201 23.646h-28.201v-14.53h28.201z"></path><path fill="currentColor" d="m74.912 86.063h13.975c1.104 0 2-.896 2-2s-.896-2-2-2h-13.975c-1.104 0-2 .896-2 2s.896 2 2 2z"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

6
openmetadata-ui/src/main/resources/ui/src/components/AppRouter/AuthenticatedAppRouter.tsx
View File

@ -24,6 +24,7 @@ import AddCustomMetricPage from '../../pages/AddCustomMetricPage/AddCustomMetric
import { CustomizablePage } from '../../pages/CustomizablePage/CustomizablePage';
import DataQualityPage from '../../pages/DataQuality/DataQualityPage';
import ForbiddenPage from '../../pages/ForbiddenPage/ForbiddenPage';
import PlatformLineage from '../../pages/PlatformLineage/PlatformLineage';
import TagPage from '../../pages/TagPage/TagPage';
import { checkPermission, userPermissions } from '../../utils/PermissionsUtils';
import AdminProtectedRoute from './AdminProtectedRoute';
@ -284,6 +285,11 @@ const AuthenticatedAppRouter: FunctionComponent = () => {
<Route exact component={MyDataPage} path={ROUTES.MY_DATA} />
<Route exact component={TourPageComponent} path={ROUTES.TOUR} />
<Route exact component={ExplorePageV1} path={ROUTES.EXPLORE} />
<Route
exact
component={PlatformLineage}
path={[ROUTES.PLATFORM_LINEAGE, ROUTES.PLATFORM_LINEAGE_WITH_FQN]}
/>
<Route component={ExplorePageV1} path={ROUTES.EXPLORE_WITH_TAB} />
<Route
exact

2
openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.interface.ts
View File

@ -28,6 +28,7 @@ import { SearchIndex } from '../../../generated/entity/data/searchIndex';
import { StoredProcedure } from '../../../generated/entity/data/storedProcedure';
import { Table } from '../../../generated/entity/data/table';
import { Topic } from '../../../generated/entity/data/topic';
import { Domain } from '../../../generated/entity/domains/domain';
import { APIService } from '../../../generated/entity/services/apiService';
import { DashboardService } from '../../../generated/entity/services/dashboardService';
import { DatabaseService } from '../../../generated/entity/services/databaseService';
@ -104,4 +105,5 @@ export type MapPatchAPIResponse = {
[EntityType.API_ENDPOINT]: APIEndpoint;
[EntityType.METRIC]: Metric;
[EntityType.TAG]: Tag;
[EntityType.DOMAIN]: Domain;
};

21
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomControls.component.tsx
View File

@ -40,7 +40,7 @@ const CustomControls: FC<ControlProps> = ({
className,
}: ControlProps) => {
const { t } = useTranslation();
const { onQueryFilterUpdate } = useLineageProvider();
const { onQueryFilterUpdate, nodes } = useLineageProvider();
const [selectedFilter, setSelectedFilter] = useState<string[]>([]);
const [selectedQuickFilters, setSelectedQuickFilters] = useState<
ExploreQuickFilterField[]
@ -51,6 +51,24 @@ const CustomControls: FC<ControlProps> = ({
setSelectedFilter((prevSelected) => [...prevSelected, key]);
};
const queryFilter = useMemo(() => {
const nodeIds = (nodes ?? [])
.map((node) => node.data?.node?.id)
.filter(Boolean);
return {
query: {
bool: {
must: {
terms: {
'id.keyword': nodeIds,
},
},
},
},
};
}, [nodes]);
const filterMenu: ItemType[] = useMemo(() => {
return filters.map((filter) => ({
key: filter.key,
@ -145,6 +163,7 @@ const CustomControls: FC<ControlProps> = ({
<ExploreQuickFilters
independent
aggregations={{}}
defaultQueryFilter={queryFilter}
fields={selectedQuickFilters}
index={SearchIndex.ALL}
showDeleted={false}

97
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.component.tsx Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Button, Tag } from 'antd';
import classNames from 'classnames';
import { get } from 'lodash';
import React, { FC, useMemo } from 'react';
import { PRIMARY_COLOR } from '../../../../constants/Color.constants';
import { SearchSourceAlias } from '../../../../interface/search.interface';
import { getEntityName } from '../../../../utils/EntityUtils';
import serviceUtilClassBase from '../../../../utils/ServiceUtilClassBase';
import { getEntityIcon } from '../../../../utils/TableUtils';
import { SourceType } from '../../../SearchedData/SearchedData.interface';
import './entity-suggestion-option.less';
import { EntitySuggestionOptionProps } from './EntitySuggestionOption.interface';
const EntitySuggestionOption: FC<EntitySuggestionOptionProps> = ({
entity,
heading,
onSelectHandler,
className,
showEntityTypeBadge = false,
}) => {
const serviceIcon = useMemo(() => {
const serviceType = get(entity, 'serviceType', '');
if (serviceType) {
return (
<img
alt={entity.name}
className="m-r-xs"
height="16px"
src={serviceUtilClassBase.getServiceTypeLogo(
entity as SearchSourceAlias
)}
width="16px"
/>
);
}
return getEntityIcon(
(entity as SearchSourceAlias).entityType ?? '',
'w-4 h-4 m-r-xs'
);
}, [entity]);
return (
<Button
block
className={classNames(
'd-flex items-center entity-suggestion-option-btn p-0',
className
)}
data-testid={`node-suggestion-${entity.fullyQualifiedName}`}
key={entity.fullyQualifiedName}
type="text"
onClick={() => {
onSelectHandler?.(entity);
}}>
<div className="d-flex items-center w-full overflow-hidden justify-between">
<div className="d-flex items-center flex-1 overflow-hidden">
{serviceIcon}
<div className="d-flex text-left align-start flex-column flex-1 min-w-0">
{heading && (
<p className="d-block text-xs text-grey-muted p-b-xss break-all whitespace-normal text-left w-full truncate">
{heading}
</p>
)}
<p className="text-xs text-grey-muted truncate line-height-normal w-full">
{entity.name}
</p>
<p className="text-sm font-medium truncate w-full">
{getEntityName(entity)}
</p>
</div>
</div>
{showEntityTypeBadge && (
<Tag
className="entity-tag text-xs ml-2 whitespace-nowrap"
color={PRIMARY_COLOR}>
{(entity as SourceType)?.entityType}
</Tag>
)}
</div>
</Button>
);
};
export default EntitySuggestionOption;

21
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.interface.ts Normal file
View File

@ -0,0 +1,21 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityReference } from '../../../../generated/entity/type';
export interface EntitySuggestionOptionProps {
entity: EntityReference;
heading?: string;
onSelectHandler?: (value: EntityReference) => void;
className?: string;
showEntityTypeBadge?: boolean;
}

24
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/EntitySuggestionOption/entity-suggestion-option.less Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
.entity-suggestion-option-btn {
height: auto;
padding: 4px 12px;
&:hover {
color: transparent;
}
.entity-tag {
text-transform: capitalize;
}
}

4
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.interface.ts
View File

@ -10,9 +10,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityType } from '../../../../enums/entity.enum';
export interface LineageControlButtonsProps {
handleFullScreenViewClick?: () => void;
onExitFullScreenViewClick?: () => void;
deleted?: boolean;
hasEditAccess: boolean;
entityType?: EntityType;
}

40
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageControlButtons/LineageControlButtons.tsx
View File

@ -28,10 +28,12 @@ import { useTranslation } from 'react-i18next';
import { ReactComponent as EditIcon } from '../../../../assets/svg/edit-new.svg';
import { ReactComponent as ExportIcon } from '../../../../assets/svg/ic-export.svg';
import { NO_PERMISSION_FOR_ACTION } from '../../../../constants/HelperTextUtil';
import { SERVICE_TYPES } from '../../../../constants/Services.constant';
import { useLineageProvider } from '../../../../context/LineageProvider/LineageProvider';
import { LineagePlatformView } from '../../../../context/LineageProvider/LineageProvider.interface';
import { LineageLayer } from '../../../../generated/configuration/lineageSettings';
import { getLoadingStatusValue } from '../../../../utils/EntityLineageUtils';
import { AssetsUnion } from '../../../DataAssets/AssetsSelectionModal/AssetSelectionModal.interface';
import { LineageConfig } from '../EntityLineage.interface';
import LineageConfigModal from '../LineageConfigModal';
import './lineage-control-buttons.less';
@ -42,6 +44,7 @@ const LineageControlButtons: FC<LineageControlButtonsProps> = ({
onExitFullScreenViewClick,
deleted,
hasEditAccess,
entityType,
}) => {
const { t } = useTranslation();
const [dialogVisible, setDialogVisible] = useState<boolean>(false);
@ -98,23 +101,26 @@ const LineageControlButtons: FC<LineageControlButtonsProps> = ({
return (
<>
<div className="lineage-control-buttons">
{!deleted && platformView === LineagePlatformView.None && (
<Button
className={classNames('lineage-button', {
active: isEditMode,
})}
data-testid="edit-lineage"
disabled={!hasEditAccess}
icon={getLoadingStatusValue(editIcon, loading, status)}
title={
hasEditAccess
? t('label.edit-entity', { entity: t('label.lineage') })
: NO_PERMISSION_FOR_ACTION
}
type="text"
onClick={onLineageEditClick}
/>
)}
{!deleted &&
platformView === LineagePlatformView.None &&
entityType &&
!SERVICE_TYPES.includes(entityType as AssetsUnion) && (
<Button
className={classNames('lineage-button', {
active: isEditMode,
})}
data-testid="edit-lineage"
disabled={!hasEditAccess}
icon={getLoadingStatusValue(editIcon, loading, status)}
title={
hasEditAccess
? t('label.edit-entity', { entity: t('label.lineage') })
: NO_PERMISSION_FOR_ACTION
}
type="text"
onClick={onLineageEditClick}
/>
)}
{isColumnLayerActive && !isEditMode && (
<Button

27
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageLayers/LineageLayers.interface.ts Normal file
View File

@ -0,0 +1,27 @@
import { EntityType } from '../../../../enums/entity.enum';
import { SourceType } from '../../../SearchedData/SearchedData.interface';
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface LayerButtonProps {
isActive: boolean;
onClick: () => void;
icon: React.ReactNode;
label: string;
testId: string;
}
export interface LineageLayersProps {
entityType?: EntityType;
entity?: SourceType;
}

5
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageLayers/LineageLayers.test.tsx
View File

@ -14,6 +14,7 @@ import { act, queryByText, render, screen } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import React from 'react';
import { ReactFlowProvider } from 'reactflow';
import { EntityType } from '../../../../enums/entity.enum';
import { LineageLayer } from '../../../../generated/settings/settings';
import LineageLayers from './LineageLayers';
@ -71,7 +72,7 @@ describe('LineageLayers component', () => {
it('renders LineageLayers component', () => {
const { container } = render(
<ReactFlowProvider>
<LineageLayers />
<LineageLayers entityType={EntityType.TABLE} />
</ReactFlowProvider>
);
const layerBtn = screen.getByText('label.layer-plural');
@ -90,7 +91,7 @@ describe('LineageLayers component', () => {
it('calls onUpdateLayerView when a button is clicked', async () => {
render(
<ReactFlowProvider>
<LineageLayers />
<LineageLayers entityType={EntityType.TABLE} />
</ReactFlowProvider>
);

170
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageLayers/LineageLayers.tsx
View File

@ -16,90 +16,130 @@ import classNames from 'classnames';
import { t } from 'i18next';
import React from 'react';
import { ReactComponent as DataQualityIcon } from '../../../../assets/svg/ic-data-contract.svg';
import { ReactComponent as DomainIcon } from '../../../../assets/svg/ic-domain.svg';
import { ReactComponent as Layers } from '../../../../assets/svg/ic-layers.svg';
import { ReactComponent as ServiceView } from '../../../../assets/svg/services.svg';
import { SERVICE_TYPES } from '../../../../constants/Services.constant';
import { useLineageProvider } from '../../../../context/LineageProvider/LineageProvider';
import { LineagePlatformView } from '../../../../context/LineageProvider/LineageProvider.interface';
import { EntityType } from '../../../../enums/entity.enum';
import { LineageLayer } from '../../../../generated/settings/settings';
import searchClassBase from '../../../../utils/SearchClassBase';
import { AssetsUnion } from '../../../DataAssets/AssetsSelectionModal/AssetSelectionModal.interface';
import './lineage-layers.less';
import {
LayerButtonProps,
LineageLayersProps,
} from './LineageLayers.interface';
const LineageLayers = () => {
const LayerButton: React.FC<LayerButtonProps> = React.memo(
({ isActive, onClick, icon, label, testId }) => (
<Button
className={classNames('lineage-layer-button h-15', {
active: isActive,
})}
data-testid={testId}
onClick={onClick}>
<div className="lineage-layer-btn">
<div className="layer-icon">{icon}</div>
<Typography.Text className="text-xss">{label}</Typography.Text>
</div>
</Button>
)
);
const LineageLayers = ({ entityType }: LineageLayersProps) => {
const {
activeLayer,
onUpdateLayerView,
isEditMode,
onPlatformViewChange,
platformView,
isPlatformLineage,
} = useLineageProvider();
const onButtonClick = (value: LineageLayer) => {
const index = activeLayer.indexOf(value);
if (index === -1) {
onUpdateLayerView([...activeLayer, value]);
} else {
onUpdateLayerView(activeLayer.filter((layer) => layer !== value));
}
};
const handleLayerClick = React.useCallback(
(value: LineageLayer) => {
const index = activeLayer.indexOf(value);
if (index === -1) {
onUpdateLayerView([...activeLayer, value]);
} else {
onUpdateLayerView(activeLayer.filter((layer) => layer !== value));
}
},
[activeLayer, onUpdateLayerView]
);
const handlePlatformViewChange = React.useCallback(
(view: LineagePlatformView) => {
onPlatformViewChange(
platformView === view ? LineagePlatformView.None : view
);
},
[platformView, onPlatformViewChange]
);
const buttonContent = React.useMemo(
() => (
<ButtonGroup>
{entityType && !SERVICE_TYPES.includes(entityType as AssetsUnion) && (
<>
<LayerButton
icon={searchClassBase.getEntityIcon(EntityType.TABLE)}
isActive={activeLayer.includes(LineageLayer.ColumnLevelLineage)}
label={t('label.column')}
testId="lineage-layer-column-btn"
onClick={() => handleLayerClick(LineageLayer.ColumnLevelLineage)}
/>
<LayerButton
icon={<DataQualityIcon />}
isActive={activeLayer.includes(LineageLayer.DataObservability)}
label={t('label.observability')}
testId="lineage-layer-observability-btn"
onClick={() => handleLayerClick(LineageLayer.DataObservability)}
/>
</>
)}
{(isPlatformLineage ||
(entityType &&
!SERVICE_TYPES.includes(entityType as AssetsUnion))) && (
<LayerButton
icon={<ServiceView />}
isActive={platformView === LineagePlatformView.Service}
label={t('label.service')}
testId="lineage-layer-service-btn"
onClick={() =>
handlePlatformViewChange(LineagePlatformView.Service)
}
/>
)}
{(isPlatformLineage ||
(entityType && entityType !== EntityType.DOMAIN)) && (
<LayerButton
icon={<DomainIcon />}
isActive={platformView === LineagePlatformView.Domain}
label={t('label.domain')}
testId="lineage-layer-domain-btn"
onClick={() => handlePlatformViewChange(LineagePlatformView.Domain)}
/>
)}
</ButtonGroup>
),
[
activeLayer,
platformView,
entityType,
handleLayerClick,
handlePlatformViewChange,
isPlatformLineage,
]
);
return (
<Popover
content={
<ButtonGroup>
<Button
className={classNames('lineage-layer-button h-15', {
active: activeLayer.includes(LineageLayer.ColumnLevelLineage),
})}
data-testid="lineage-layer-column-btn"
onClick={() => onButtonClick(LineageLayer.ColumnLevelLineage)}>
<div className="lineage-layer-btn">
<div className="layer-icon">
{searchClassBase.getEntityIcon(EntityType.TABLE)}
</div>
<Typography.Text className="text-xss">
{t('label.column')}
</Typography.Text>
</div>
</Button>
<Button
className={classNames('lineage-layer-button h-15', {
active: activeLayer.includes(LineageLayer.DataObservability),
})}
data-testid="lineage-layer-observability-btn"
onClick={() => onButtonClick(LineageLayer.DataObservability)}>
<div className="lineage-layer-btn">
<div className="layer-icon">
<DataQualityIcon />
</div>
<Typography.Text className="text-xss">
{t('label.observability')}
</Typography.Text>
</div>
</Button>
<Button
className={classNames('lineage-layer-button h-15', {
active: platformView === LineagePlatformView.Service,
})}
data-testid="lineage-layer-observability-btn"
onClick={() =>
onPlatformViewChange(
platformView === LineagePlatformView.Service
? LineagePlatformView.None
: LineagePlatformView.Service
)
}>
<div className="lineage-layer-btn">
<div className="layer-icon">
<ServiceView />
</div>
<Typography.Text className="text-xss">
{t('label.service')}
</Typography.Text>
</div>
</Button>
</ButtonGroup>
}
content={buttonContent}
overlayClassName="lineage-layers-popover"
placement="right"
trigger="click">
@ -121,4 +161,4 @@ const LineageLayers = () => {
);
};
export default LineageLayers;
export default React.memo(LineageLayers);

1
openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts
View File

@ -26,4 +26,5 @@ export interface ExploreQuickFiltersProps {
onChangeShowDeleted?: (showDeleted: boolean) => void;
independent?: boolean; // flag to indicate if the filters are independent of aggregations
fieldsWithNullValues?: EntityFields[];
defaultQueryFilter?: Record<string, unknown>;
}

16
openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx
View File

@ -26,7 +26,6 @@ import { EntityFields } from '../../enums/AdvancedSearch.enum';
import { SearchIndex } from '../../enums/search.enum';
import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation';
import { QueryFilterInterface } from '../../pages/ExplorePage/ExplorePage.interface';
import { getAggregateFieldOptions } from '../../rest/miscAPI';
import { getTags } from '../../rest/tagAPI';
import { getOptionsFromAggregationBucket } from '../../utils/AdvancedSearchUtils';
import { getEntityName } from '../../utils/EntityUtils';
@ -34,6 +33,7 @@ import {
getCombinedQueryFilterObject,
getQuickFilterWithDeletedFlag,
} from '../../utils/ExplorePage/ExplorePageUtils';
import { getAggregationOptions } from '../../utils/ExploreUtils';
import { showErrorToast } from '../../utils/ToastUtils';
import SearchDropdown from '../SearchDropdown/SearchDropdown';
import { SearchDropdownOption } from '../SearchDropdown/SearchDropdown.interface';
@ -48,6 +48,7 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
independent = false,
onFieldValueSelect,
fieldsWithNullValues = [],
defaultQueryFilter,
}) => {
const location = useCustomLocation();
const [options, setOptions] = useState<SearchDropdownOption[]>();
@ -75,7 +76,8 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
const updatedQuickFilters = getAdvancedSearchQuickFilters();
const combinedQueryFilter = getCombinedQueryFilterObject(
updatedQuickFilters as QueryFilterInterface,
queryFilter as unknown as QueryFilterInterface
queryFilter as unknown as QueryFilterInterface,
defaultQueryFilter as unknown as QueryFilterInterface
);
const fetchDefaultOptions = async (
@ -87,11 +89,12 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
buckets = aggregations[key].buckets;
} else {
const [res, tierTags] = await Promise.all([
getAggregateFieldOptions(
getAggregationOptions(
index,
key,
'',
JSON.stringify(combinedQueryFilter)
JSON.stringify(combinedQueryFilter),
independent
),
key === TIER_FQN_KEY
? getTags({ parent: 'Tier', limit: 50 })
@ -151,11 +154,12 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
return;
}
if (key !== TIER_FQN_KEY) {
const res = await getAggregateFieldOptions(
const res = await getAggregationOptions(
index,
key,
value,
JSON.stringify(combinedQueryFilter)
JSON.stringify(combinedQueryFilter),
independent
);
const buckets = res.data.aggregations[`sterms#${key}`].buckets;

9
openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.component.tsx
View File

@ -56,6 +56,7 @@ const Lineage = ({
hasEditAccess,
entity,
entityType,
isPlatformLineage,
}: LineageProps) => {
const { t } = useTranslation();
const history = useHistory();
@ -115,8 +116,8 @@ const Lineage = ({
);
useEffect(() => {
updateEntityData(entityType, entity as SourceType);
}, [entity, entityType]);
updateEntityData(entityType, entity as SourceType, isPlatformLineage);
}, [entity, entityType, isPlatformLineage]);
// Loading the react flow component after the nodes and edges are initialised improves performance
// considerably. So added an init state for showing loader.
@ -136,6 +137,7 @@ const Lineage = ({
<CustomControlsComponent className="absolute top-1 right-1 p-xs" />
<LineageControlButtons
deleted={deleted}
entityType={entityType}
handleFullScreenViewClick={
!isFullScreen ? onFullScreenClick : undefined
}
@ -192,8 +194,9 @@ const Lineage = ({
onPaneClick={onPaneClick}>
<Background gap={12} size={1} />
<MiniMap position="bottom-right" />
<Panel position="bottom-left">
<LineageLayers />
<LineageLayers entityType={entityType} />
</Panel>
</ReactFlow>
</ReactFlowProvider>

1
openmetadata-ui/src/main/resources/ui/src/components/Lineage/Lineage.interface.ts
View File

@ -22,6 +22,7 @@ export interface LineageProps {
hasEditAccess: boolean;
isFullScreen?: boolean;
entity?: SourceType;
isPlatformLineage?: boolean;
}
export interface EntityLineageResponse {

8
openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts
View File

@ -21,6 +21,7 @@ import { ReactComponent as DataQualityIcon } from '../assets/svg/ic-data-contrac
import { ReactComponent as DomainsIcon } from '../assets/svg/ic-domain.svg';
import { ReactComponent as IncidentMangerIcon } from '../assets/svg/ic-incident-manager.svg';
import { ReactComponent as ObservabilityIcon } from '../assets/svg/ic-observability.svg';
import { ReactComponent as PlatformLineageIcon } from '../assets/svg/ic-platform-lineage.svg';
import { ReactComponent as SettingsIcon } from '../assets/svg/ic-settings-v1.svg';
import { ReactComponent as InsightsIcon } from '../assets/svg/lamp-charge.svg';
import { ReactComponent as LogoutIcon } from '../assets/svg/logout.svg';
@ -43,6 +44,13 @@ export const SIDEBAR_LIST: Array<LeftSidebarItem> = [
icon: ExploreIcon,
dataTestId: `app-bar-item-${SidebarItem.EXPLORE}`,
},
{
key: ROUTES.PLATFORM_LINEAGE,
title: i18next.t('label.lineage'),
redirect_url: ROUTES.PLATFORM_LINEAGE,
icon: PlatformLineageIcon,
dataTestId: `app-bar-item-${SidebarItem.LINEAGE}`,
},
{
key: ROUTES.OBSERVABILITY,
title: i18next.t('label.observability'),

4
openmetadata-ui/src/main/resources/ui/src/constants/PageHeaders.constant.ts
View File

@ -256,4 +256,8 @@ export const PAGE_HEADERS = {
entity: i18n.t('label.metric-plural'),
}),
},
PLATFORM_LINEAGE: {
header: i18n.t('label.lineage'),
subHeader: i18n.t('message.page-sub-header-for-platform-lineage'),
},
};

2
openmetadata-ui/src/main/resources/ui/src/constants/constants.ts
View File

@ -126,6 +126,8 @@ export const ROUTES = {
FORBIDDEN: '/403',
UNAUTHORISED: '/unauthorised',
LOGOUT: '/logout',
PLATFORM_LINEAGE: '/lineage',
PLATFORM_LINEAGE_WITH_FQN: `/lineage/${PLACEHOLDER_ROUTE_ENTITY_TYPE}/${PLACEHOLDER_ROUTE_FQN}`,
MY_DATA: '/my-data',
TOUR: '/tour',
REPORTS: '/reports',

7
openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx
View File

@ -71,6 +71,7 @@ export interface LineageContextType {
activeLayer: LineageLayer[];
platformView: LineagePlatformView;
expandAllColumns: boolean;
isPlatformLineage: boolean;
toggleColumnView: () => void;
onInitReactFlow: (reactFlowInstance: ReactFlowInstance) => void;
onPaneClick: () => void;
@ -101,7 +102,11 @@ export interface LineageContextType {
onColumnEdgeRemove: () => void;
onAddPipelineClick: () => void;
onConnect: (connection: Edge | Connection) => void;
updateEntityData: (entityType: EntityType, entity?: SourceType) => void;
updateEntityData: (
entityType: EntityType,
entity?: SourceType,
isPlatformLineage?: boolean
) => void;
onUpdateLayerView: (layers: LineageLayer[]) => void;
redraw: () => Promise<void>;
}

8
openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx
View File

@ -14,6 +14,7 @@ import { act, fireEvent, render, screen } from '@testing-library/react';
import QueryString from 'qs';
import React, { useEffect } from 'react';
import { Edge } from 'reactflow';
import { SourceType } from '../../components/SearchedData/SearchedData.interface';
import { EntityType } from '../../enums/entity.enum';
import { LineageDirection } from '../../generated/api/lineage/searchLineageRequest';
import {
@ -82,7 +83,12 @@ const DummyChildrenComponent = () => {
};
useEffect(() => {
updateEntityData(EntityType.TABLE, undefined);
updateEntityData(EntityType.TABLE, {
id: 'table1',
name: 'table1',
type: 'table',
fullyQualifiedName: 'table1',
} as SourceType);
}, []);
return (

111
openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx
View File

@ -77,6 +77,7 @@ import {
exportLineageAsync,
getDataQualityLineage,
getLineageDataByFQN,
getPlatformLineage,
updateLineageEdge,
} from '../../rest/lineageAPI';
import {
@ -193,6 +194,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
const deletePressed = useKeyPress('Delete');
const backspacePressed = useKeyPress('Backspace');
const { showModal } = useEntityExportModalProvider();
const [isPlatformLineage, setIsPlatformLineage] = useState(false);
const lineageLayer = useMemo(() => {
const param = location.search;
@ -290,6 +292,8 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
const rootNode = visibleNodes.find((n) => n.data.isRootNode);
if (rootNode) {
centerNodePosition(rootNode, reactFlowInstance, zoomValue);
} else if (visibleNodes.length > 0) {
centerNodePosition(visibleNodes[0], reactFlowInstance, zoomValue);
}
}
@ -339,6 +343,41 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
[redrawLineage]
);
const fetchPlatformLineage = useCallback(
async (view: 'service' | 'domain', config?: LineageConfig) => {
try {
setLoading(true);
setInit(false);
const res = await getPlatformLineage({
config,
view,
});
setLineageData(res);
const { nodes, edges, entity } = parseLineageData(res, '');
const updatedEntityLineage = {
nodes,
edges,
entity,
};
setEntityLineage(updatedEntityLineage);
} catch (err) {
showErrorToast(
err as AxiosError,
t('server.entity-fetch-error', {
entity: t('label.lineage-data-lowercase'),
})
);
} finally {
setInit(true);
setLoading(false);
}
},
[]
);
const fetchLineageData = useCallback(
async (fqn: string, entityType: string, config?: LineageConfig) => {
if (isTourOpen) {
@ -499,9 +538,17 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
}, []);
const updateEntityData = useCallback(
(entityType: EntityType, entity?: SourceType) => {
(
entityType: EntityType,
entity?: SourceType,
isPlatformLineage?: boolean
) => {
setEntity(entity);
setEntityType(entityType);
setIsPlatformLineage(isPlatformLineage ?? false);
if (isPlatformLineage && !entity) {
setPlatformView(LineagePlatformView.Service);
}
},
[]
);
@ -1261,20 +1308,45 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
}, [entityLineage, redrawLineage]);
const onPlatformViewUpdate = useCallback(() => {
if (entity) {
if (platformView === LineagePlatformView.Service) {
if (entity?.service) {
fetchLineageData(
entity?.service.fullyQualifiedName ?? '',
entity?.service.type,
lineageConfig
);
}
if (entity && decodedFqn && entityType) {
if (platformView === LineagePlatformView.Service && entity?.service) {
fetchLineageData(
entity?.service.fullyQualifiedName ?? '',
entity?.service.type,
lineageConfig
);
} else if (
platformView === LineagePlatformView.Domain &&
entity?.domain
) {
fetchLineageData(
entity?.domain.fullyQualifiedName ?? '',
entity?.domain.type,
lineageConfig
);
} else if (platformView === LineagePlatformView.None) {
fetchLineageData(decodedFqn, entityType, lineageConfig);
} else if (isPlatformLineage) {
fetchPlatformLineage(
platformView === LineagePlatformView.Domain ? 'domain' : 'service',
lineageConfig
);
}
} else if (isPlatformLineage) {
fetchPlatformLineage(
platformView === LineagePlatformView.Domain ? 'domain' : 'service',
lineageConfig
);
}
}, [entity, entityType, decodedFqn, lineageConfig, platformView]);
}, [
entity,
entityType,
decodedFqn,
lineageConfig,
platformView,
queryFilter,
isPlatformLineage,
]);
useEffect(() => {
if (defaultLineageConfig) {
@ -1292,12 +1364,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
}
}, [defaultLineageConfig]);
useEffect(() => {
if (entityType && isLineageSettingsLoaded) {
fetchLineageData(decodedFqn, entityType, lineageConfig);
}
}, [lineageConfig, queryFilter, entityType, isLineageSettingsLoaded]);
useEffect(() => {
if (!isEditMode && updatedEntityLineage !== null) {
// On exit of edit mode, use updatedEntityLineage and update data.
@ -1349,7 +1415,13 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
useEffect(() => {
onPlatformViewUpdate();
}, [platformView]);
}, [
platformView,
lineageConfig,
queryFilter,
entityType,
isLineageSettingsLoaded,
]);
const activityFeedContextValues = useMemo(() => {
return {
@ -1373,6 +1445,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
columnsHavingLineage,
expandAllColumns,
platformView,
isPlatformLineage,
toggleColumnView,
onInitReactFlow,
onPaneClick,
@ -1399,7 +1472,6 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
onExportClick,
dataQualityLineage,
redraw,
onPlatformViewChange,
};
}, [
@ -1423,6 +1495,7 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
activeLayer,
columnsHavingLineage,
expandAllColumns,
isPlatformLineage,
toggleColumnView,
onInitReactFlow,
onPaneClick,

1
openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts
View File

@ -26,4 +26,5 @@ export enum SidebarItem {
SETTINGS = 'settings',
LOGOUT = 'logout',
METRICS = 'metrics',
LINEAGE = 'lineage',
}

97
openmetadata-ui/src/main/resources/ui/src/generated/search/searchRequest.ts Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Search Request to find entities from Elastic Search based on different parameters.
*/
export interface SearchRequest {
/**
* If Need to apply the domain filter.
*/
applyDomainFilter?: boolean;
/**
* Filter documents by deleted param.
*/
deleted?: boolean;
/**
* Internal Object to filter by Domains.
*/
domains?: any;
/**
* Explain the results of the query. Defaults to false. Only for debugging purposes.
*/
explain?: boolean;
/**
* Get document body for each hit
*/
fetchSource?: boolean;
/**
* Field Name to match.
*/
fieldName?: string;
/**
* Field Value in case of Aggregations.
*/
fieldValue?: string;
/**
* Start Index for the req.
*/
from?: number;
/**
* Get only selected fields of the document body for each hit. Empty value will return all
* fields
*/
includeSourceFields?: string[];
/**
* Index Name.
*/
index?: string;
/**
* If true it will try to get the hierarchy of the entity.
*/
isHierarchy?: boolean;
/**
* Elasticsearch query that will be used as a post_filter
*/
postFilter?: string;
/**
* Query to be send to Search Engine.
*/
query?: string;
/**
* Elasticsearch query that will be combined with the query_string query generator from the
* `query` arg
*/
queryFilter?: string;
/**
* When paginating, specify the search_after values. Use it ass
* search_after=<val1>,<val2>,...
*/
searchAfter?: any;
/**
* Size to limit the no.of results returned.
*/
size?: number;
/**
* Sort the search results by field, available fields to sort weekly_stats daily_stats,
* monthly_stats, last_updated_timestamp.
*/
sortFieldParam?: string;
/**
* Sort order asc for ascending or desc for descending, defaults to desc.
*/
sortOrder?: string;
/**
* Track Total Hits.
*/
trackTotalHits?: boolean;
}

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json
View File

@ -1173,6 +1173,7 @@
"search": "Suchen",
"search-by-type": "Suchen nach {{type}}",
"search-entity": "{{entity}} suchen",
"search-entity-for-lineage": "Nach {{entity}} suchen, um Abstammung anzuzeigen",
"search-for-type": "Suchen nach {{type}}",
"search-index": "Suchindex",
"search-index-ingestion": "Suchindex für die Datenaufnahme",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Überprüfen Sie den Datenbankzugriff, den ES-Status, den Pipeline-Service-Client, die JWKS-Konfiguration und Migrationen.",
"page-sub-header-for-persona": "Repräsentieren Sie verschiedene Personas, die ein Benutzer innerhalb von OpenMetadata haben kann.",
"page-sub-header-for-pipelines": "Ingestion von Metadaten aus den am häufigsten verwendeten Pipeline-Diensten.",
"page-sub-header-for-platform-lineage": "Visualisieren Sie die Plattform-Abstammung, um Abhängigkeiten und Beziehungen in Ihrer Plattform zu verstehen.",
"page-sub-header-for-policies": "Definiere Richtlinien mit einer Reihe von Regeln für feinkörnige Zugriffssteuerung.",
"page-sub-header-for-profiler-configuration": "Passen Sie das Verhalten des Profilers global an, indem Sie die zu berechnenden Metriken basierend auf den Spaltendatentypen festlegen.",
"page-sub-header-for-roles": "Weise Benutzern oder Teams umfassende rollenbasierte Zugriffsberechtigungen zu.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
View File

@ -1173,6 +1173,7 @@
"search": "Search",
"search-by-type": "Search by {{type}}",
"search-entity": "Search {{entity}}",
"search-entity-for-lineage": "Nach {{entity}} suchen, um Abstammung anzuzeigen",
"search-for-type": "Search for {{type}}",
"search-index": "Search Index",
"search-index-ingestion": "Search Index Ingestion",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations",
"page-sub-header-for-persona": "Enhance and customize the user experience with Personas.",
"page-sub-header-for-pipelines": "Ingest metadata from the most used pipeline services.",
"page-sub-header-for-platform-lineage": "Visualize the lineage to understand dependencies and relationships across your application",
"page-sub-header-for-policies": "Define policies with a set of rules for fine-grained access control.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Assign comprehensive role based access to Users or Teams.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
View File

@ -1173,6 +1173,7 @@
"search": "Buscar",
"search-by-type": "Buscar por {{type}}",
"search-entity": "Buscar {{entity}}",
"search-entity-for-lineage": "Buscar {{entity}} para ver el linaje",
"search-for-type": "Buscar {{type}}",
"search-index": "Índice de Búsqueda",
"search-index-ingestion": "Ingesta de Índices de Búsqueda",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations",
"page-sub-header-for-persona": "Representa diferentes personas que un usuario puede tener dentro de OpenMetadata.",
"page-sub-header-for-pipelines": "Ingresa metadatos desde los servicios de pipeline más utilizados.",
"page-sub-header-for-platform-lineage": "Visualiza la línea de la plataforma para comprender las dependencias y relaciones en tu plataforma.",
"page-sub-header-for-policies": "Define políticas con un conjunto de reglas para el control de acceso detallado.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Asigna un acceso basado en roles integral a Usuarios o Equipos.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
View File

@ -1173,6 +1173,7 @@
"search": "Rechercher",
"search-by-type": "Rechercher par {{type}}",
"search-entity": "Rechercher {{entity}}",
"search-entity-for-lineage": "Rechercher {{entity}} pour afficher le lignage",
"search-for-type": "Rechercher pour {{type}}",
"search-index": "Index de Recherche",
"search-index-ingestion": "Index de Recherche pour l'Ingestion",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Vérifiez l'accès aux bases de données, la santé d'ES, le service de Pipeline Service, la configuration jwks et les migrations",
"page-sub-header-for-persona": "Représentez différents personas qu'un utilisateur peut avoir dans OpenMetadata.",
"page-sub-header-for-pipelines": "Ingestion de métadonnées à partir des services de pipeline les plus utilisés.",
"page-sub-header-for-platform-lineage": "Visualisez l'ascendance de la plateforme pour comprendre les dépendances et les relations au sein de votre plateforme.",
"page-sub-header-for-policies": "Définissez des politiques avec un ensemble de règles pour un contrôle d'accès précis.",
"page-sub-header-for-profiler-configuration": "Personnalisez le comportement global du profiler en établissant les métriques à calculer à partir des types de données des colonnes",
"page-sub-header-for-roles": "Attribuez des autorisations basées sur les rôles aux utilisateurs ou aux équipes.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json
View File

@ -1173,6 +1173,7 @@
"search": "Buscar",
"search-by-type": "Buscar por {{type}}",
"search-entity": "Buscar {{entity}}",
"search-entity-for-lineage": "Buscar {{entity}} para ver a liñaxe",
"search-for-type": "Buscar por {{type}}",
"search-index": "Índice de busca",
"search-index-ingestion": "Inxestión do índice de busca",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Verifica o acceso á base de datos, a saúde de Elasticsearch, o Cliente de Servizo Pipeline, a configuración de jwks e as migracións",
"page-sub-header-for-persona": "Mellora e personaliza a experiencia de usuario con Persoas.",
"page-sub-header-for-pipelines": "Inxesta metadatos dos servizos de pipelines máis usados.",
"page-sub-header-for-platform-lineage": "Visualiza a liñaxe da plataforma para comprender as dependencias e relacións na túa plataforma.",
"page-sub-header-for-policies": "Define políticas cun conxunto de regras para un control de acceso detallado.",
"page-sub-header-for-profiler-configuration": "Personaliza globalmente o comportamento do perfilador definindo as métricas a calcular baseadas nos tipos de datos das columnas",
"page-sub-header-for-roles": "Asigna un acceso baseado en roles detallados a Usuarios ou Equipos.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json
View File

@ -1173,6 +1173,7 @@
"search": "חיפוש",
"search-by-type": "חפש לפי {{type}}",
"search-entity": "חפש {{entity}}",
"search-entity-for-lineage": "חפש {{entity}} כדי להציג שושלת",
"search-for-type": "חפש עבור {{type}}",
"search-index": "חיפוש באינדקס",
"search-index-ingestion": "פרסום חיפוש באינדקס",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations",
"page-sub-header-for-persona": "צור פרופיל משתמש (פרסונה) על מנת לשייך את המשתמש לפרופיל ולהתאים את הממשק לצרכים של המשתמשים כאשר הם נכנסים ל-UI.",
"page-sub-header-for-pipelines": "שלב מטה-דאטה ממוצרי טעינת נתונים (ETL, ELT) פופלריים (כגון Airflow, Glue וכד׳)",
"page-sub-header-for-platform-lineage": "המחישו את אילן היוחסין של הפלטפורמה כדי להבין את התלות והקשרים בתוך הפלטפורמה שלכם.",
"page-sub-header-for-policies": "הגדר מדיניות עם סט של כללים לבקרת גישה ברמת רזולוציה נמוכה.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "הקצאת גישה מבוססת תפקיד למשתמשים או קבוצות.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
View File

@ -1173,6 +1173,7 @@
"search": "検索",
"search-by-type": "{{type}}で検索",
"search-entity": "{{entity}}を検索",
"search-entity-for-lineage": "系統を表示するには{{entity}}を検索",
"search-for-type": "{{type}}を検索",
"search-index": "Search Index",
"search-index-ingestion": "Search Index Ingestion",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations",
"page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.",
"page-sub-header-for-pipelines": "Ingest metadata from the most used pipeline services.",
"page-sub-header-for-platform-lineage": "プラットフォームの系統を可視化し、依存関係と相互関係を理解しましょう。",
"page-sub-header-for-policies": "Define policies with a set of rules for fine-grained access control.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Assign comprehensive role based access to Users or Teams.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json
View File

@ -1173,6 +1173,7 @@
"search": "검색",
"search-by-type": "{{type}}로 검색",
"search-entity": "{{entity}} 검색",
"search-entity-for-lineage": "계보를 보려면 {{entity}} 검색",
"search-for-type": "{{type}} 검색",
"search-index": "검색 인덱스",
"search-index-ingestion": "검색 인덱스 수집",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "데이터베이스 접근, Elasticsearch 상태, 파이프라인 서비스 클라이언트, jwks 구성 및 마이그레이션을 확인하세요.",
"page-sub-header-for-persona": "페르소나를 통해 사용자 경험을 향상하고 맞춤 설정하세요.",
"page-sub-header-for-pipelines": "가장 많이 사용되는 파이프라인 서비스로부터 메타데이터를 수집하세요.",
"page-sub-header-for-platform-lineage": "플랫폼 계보를 시각화하여 플랫폼 내 종속성과 관계를 이해하세요.",
"page-sub-header-for-policies": "세분화된 접근 제어를 위한 규칙 집합으로 정책을 정의하세요.",
"page-sub-header-for-profiler-configuration": "열 데이터 유형에 따라 계산할 메트릭을 설정하여 프로파일러 동작을 전역적으로 맞춤 설정하세요.",
"page-sub-header-for-roles": "사용자 또는 팀에 대해 포괄적인 역할 기반 접근을 할당하세요.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json
View File

@ -1173,6 +1173,7 @@
"search": "शोधा",
"search-by-type": "{{type}} द्वारे शोधा",
"search-entity": "{{entity}} शोधा",
"search-entity-for-lineage": "वंशावळ पाहण्यासाठी {{entity}} शोधा",
"search-for-type": "{{type}} साठी शोधा",
"search-index": "शोध अनुक्रमणिका",
"search-index-ingestion": "शोध अनुक्रमणिका अंतर्ग्रहण",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "डेटाबेस प्रवेश, ES आरोग्य, पाइपलाइन सेवा क्लायंट, jwks संरचना आणि स्थलांतर तपासा",
"page-sub-header-for-persona": "व्यक्तिमत्वांसह वापरकर्ता अनुभव वाढवा आणि सानुकूलित करा.",
"page-sub-header-for-pipelines": "सर्वात जास्त वापरल्या जाणार्‍या पाइपलाइन सेवांमधून मेटाडेटा अंतर्ग्रहण करा.",
"page-sub-header-for-platform-lineage": "प्लॅटफॉर्म वंशावळ दृश्यमान करा जेणेकरून तुमच्या प्लॅटफॉर्ममधील अवलंबित्वे आणि नातेसंबंध समजून घेता येतील.",
"page-sub-header-for-policies": "सूक्ष्म प्रवेश नियंत्रणासाठी नियमांचा संच असलेली धोरणे परिभाषित करा.",
"page-sub-header-for-profiler-configuration": "स्तंभ डेटा प्रकारांवर आधारित मेट्रिक्स सेट करून प्रोफाइलरचे वर्तन जागतिक स्तरावर सानुकूलित करा",
"page-sub-header-for-roles": "वापरकर्ते किंवा टीम्सना व्यापक भूमिका आधारित प्रवेश नियुक्त करा.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json
View File

@ -1173,6 +1173,7 @@
"search": "Zoeken",
"search-by-type": "Zoeken op {{type}}",
"search-entity": "Zoek {{entity}}",
"search-entity-for-lineage": "Zoek naar {{entity}} om afstamming te bekijken",
"search-for-type": "Zoeken naar {{type}}",
"search-index": "Zoekindex",
"search-index-ingestion": "Zoekindexingestie",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations",
"page-sub-header-for-persona": "De gebruikerservaring verbeteren en aanpassen met Persona's.",
"page-sub-header-for-pipelines": "Ingest metadata van de meestgebruikte pipelineservices.",
"page-sub-header-for-platform-lineage": "Visualiseer de platformafstamming om afhankelijkheden en relaties binnen uw platform te begrijpen.",
"page-sub-header-for-policies": "Definieer beleid met een reeks regels voor fijnmazige toegangscontrole.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Wijs uitgebreide rolgebaseerde toegang toe aan gebruikers of teams.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json
View File

@ -1173,6 +1173,7 @@
"search": "جستجو",
"search-by-type": "جستجو بر اساس {{type}}",
"search-entity": "جستجو {{entity}}",
"search-entity-for-lineage": "برای مشاهده تبارنامه، {{entity}} را جستجو کنید",
"search-for-type": "جستجو برای {{type}}",
"search-index": "شاخص جستجو",
"search-index-ingestion": "ورود شاخص جستجو",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "بررسی دسترسی به پایگاه داده، سلامت Elasticsearch، سرویس کلاینت Pipeline، پیکربندی jwks و مهاجرت‌ها.",
"page-sub-header-for-persona": "تجربه کاربر را با شخصیت‌های مختلف بهبود بخشید و سفارشی‌سازی کنید.",
"page-sub-header-for-pipelines": "ورود متادیتا از پرکاربردترین سرویس‌های خطوط لوله.",
"page-sub-header-for-platform-lineage": "Visualiza a linhagem da plataforma para entender dependências e relacionamentos na sua plataforma.",
"page-sub-header-for-policies": "تعریف سیاست‌ها با مجموعه‌ای از قوانین برای کنترل دقیق دسترسی.",
"page-sub-header-for-profiler-configuration": "تنظیم رفتار پروفایلر در سطح جهانی با تعیین معیارها بر اساس نوع داده‌های ستون‌ها.",
"page-sub-header-for-roles": "تخصیص دسترسی جامع مبتنی بر نقش به کاربران یا تیم‌ها.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
View File

@ -1173,6 +1173,7 @@
"search": "Pesquisar",
"search-by-type": "Pesquisar por {{type}}",
"search-entity": "Pesquisar {{entity}}",
"search-entity-for-lineage": "Pesquisar {{entity}} para visualizar a linhagem",
"search-for-type": "Pesquisar por {{type}}",
"search-index": "Índice de Pesquisa",
"search-index-ingestion": "Ingestão de Índice de Pesquisa",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations",
"page-sub-header-for-persona": "Crie Personas para associar a persona do usuário ao OpenMetadata",
"page-sub-header-for-pipelines": "Ingestão de metadados dos serviços de pipeline mais utilizados.",
"page-sub-header-for-platform-lineage": "Visualize a linhagem da plataforma para entender dependências e relacionamentos em sua plataforma.",
"page-sub-header-for-policies": "Defina políticas com um conjunto de regras para controle de acesso detalhado.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Atribua acesso baseado em funções abrangentes a usuários ou equipes.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json
View File

@ -1173,6 +1173,7 @@
"search": "Pesquisar",
"search-by-type": "Pesquisar por {{type}}",
"search-entity": "Pesquisar {{entity}}",
"search-entity-for-lineage": "Pesquisar {{entity}} para visualizar a linhagem",
"search-for-type": "Pesquisar por {{type}}",
"search-index": "Índice de Pesquisa",
"search-index-ingestion": "Ingestão de Índice de Pesquisa",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations",
"page-sub-header-for-persona": "Crie Personas para associar a persona do utilizador ao OpenMetadata",
"page-sub-header-for-pipelines": "Ingestão de metadados dos serviços de pipeline mais utilizados.",
"page-sub-header-for-platform-lineage": "Visualize a linhagem da plataforma para compreender dependências e relações na sua plataforma.",
"page-sub-header-for-policies": "Defina políticas com um conjunto de regras para controle de acesso detalhado.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Atribua acesso baseado em funções abrangentes a Utilizadores ou equipas.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json
View File

@ -1173,6 +1173,7 @@
"search": "Поиск",
"search-by-type": "Поиск {{type}}",
"search-entity": "Поиск {{entity}}",
"search-entity-for-lineage": "Найдите {{entity}} для просмотра происхождения",
"search-for-type": "Поиск для {{type}}",
"search-index": "Search Index",
"search-index-ingestion": "Получение поискового индекса",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "Check database access, ES health, Pipeline Service Client, jwks configuration and migrations",
"page-sub-header-for-persona": "Represent different persona that a user may have withing OpenMetadata.",
"page-sub-header-for-pipelines": "Принимать метаданные из наиболее часто используемых конвейерных служб.",
"page-sub-header-for-platform-lineage": "Визуализируйте родословную платформы, чтобы понять зависимости и взаимосвязи в вашей платформе.",
"page-sub-header-for-policies": "Определите политики с набором правил для точного контроля доступа.",
"page-sub-header-for-profiler-configuration": "Customize globally the behavior of the profiler by setting the metrics to compute based on columns data types",
"page-sub-header-for-roles": "Назначьте полный доступ на основе ролей пользователям или командам.",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json
View File

@ -1173,6 +1173,7 @@
"search": "ค้นหา",
"search-by-type": "ค้นหาตาม {{type}}",
"search-entity": "ค้นหา {{entity}}",
"search-entity-for-lineage": "ค้นหา {{entity}} เพื่อดูสายพันธุ์",
"search-for-type": "ค้นหาสำหรับ {{type}}",
"search-index": "ดัชนีการค้นหา",
"search-index-ingestion": "การนำเข้าดัชนีการค้นหา",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "ตรวจสอบการเข้าถึงฐานข้อมูล, สถานะ ES, ไคลเอนต์บริการท่อ, การกำหนดค่า jwks, และการโยกย้าย",
"page-sub-header-for-persona": "เสริมสร้างและปรับแต่งประสบการณ์ของผู้ใช้ด้วยบุคลิกภาพ",
"page-sub-header-for-pipelines": "นำเข้าข้อมูลเมตาจากบริการท่อที่ใช้มากที่สุด",
"page-sub-header-for-platform-lineage": "แสดงลำดับของแพลตฟอร์มเพื่อให้เข้าใจถึงการพึ่งพาและความสัมพันธ์ภายในแพลตฟอร์มของคุณ",
"page-sub-header-for-policies": "กำหนดนโยบายพร้อมกฎชุดสำหรับการควบคุมการเข้าถึงที่มีความละเอียด",
"page-sub-header-for-profiler-configuration": "ปรับแต่งพฤติกรรมของโปรไฟล์เลอร์ในระดับโลกโดยการตั้งค่าเมตริกที่จะคำนวณตามประเภทของข้อมูลในคอลัมน์",
"page-sub-header-for-roles": "มอบบทบาทการเข้าถึงที่ครอบคลุมให้กับผู้ใช้หรือทีม",

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
View File

@ -1173,6 +1173,7 @@
"search": "搜索",
"search-by-type": "通过{{type}}搜索",
"search-entity": "搜索{{entity}}",
"search-entity-for-lineage": "搜索{{entity}}以查看血统",
"search-for-type": "搜索{{type}}",
"search-index": "搜索索引",
"search-index-ingestion": "搜索索引提取",
@ -1928,6 +1929,7 @@
"page-sub-header-for-om-health-configuration": "检查数据库访问、ES 健康状况、工作流服务客户端、jwks 配置和迁移情况",
"page-sub-header-for-persona": "代表用户在 OpenMetadata 中可能拥有的不同用户角色",
"page-sub-header-for-pipelines": "从最常用的工作流类型服务中提取元数据",
"page-sub-header-for-platform-lineage": "可视化平台谱系，以理解平台内的依赖关系和关联。",
"page-sub-header-for-policies": "通过组合一系列规则定义权限策略以精细化控制访问权限",
"page-sub-header-for-profiler-configuration": "根据列数据类型计算的指标, 设置全局自定义分析器的行为",
"page-sub-header-for-roles": "分配基于角色的访问权限给用户或团队",

16
openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.interface.ts Normal file
View File

@ -0,0 +1,16 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export enum LineagePlatformView {
Service = 'service',
Domain = 'domain',
}

211
openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/PlatformLineage.tsx Normal file
View File

@ -0,0 +1,211 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Col, Divider, Row, Select } from 'antd';
import { DefaultOptionType } from 'antd/lib/select';
import { AxiosError } from 'axios';
import { debounce } from 'lodash';
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useHistory, useParams } from 'react-router-dom';
import Loader from '../../components/common/Loader/Loader';
import { AssetsUnion } from '../../components/DataAssets/AssetsSelectionModal/AssetSelectionModal.interface';
import EntitySuggestionOption from '../../components/Entity/EntityLineage/EntitySuggestionOption/EntitySuggestionOption.component';
import Lineage from '../../components/Lineage/Lineage.component';
import PageHeader from '../../components/PageHeader/PageHeader.component';
import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1';
import { SourceType } from '../../components/SearchedData/SearchedData.interface';
import { PAGE_SIZE_BASE } from '../../constants/constants';
import { PAGE_HEADERS } from '../../constants/PageHeaders.constant';
import LineageProvider from '../../context/LineageProvider/LineageProvider';
import {
OperationPermission,
ResourceEntity,
} from '../../context/PermissionProvider/PermissionProvider.interface';
import { EntityType } from '../../enums/entity.enum';
import { SearchIndex } from '../../enums/search.enum';
import { EntityReference } from '../../generated/entity/type';
import { useFqn } from '../../hooks/useFqn';
import { getEntityPermissionByFqn } from '../../rest/permissionAPI';
import { searchQuery } from '../../rest/searchAPI';
import { getEntityAPIfromSource } from '../../utils/Assets/AssetsUtils';
import { getLineageEntityExclusionFilter } from '../../utils/EntityLineageUtils';
import { getOperationPermissions } from '../../utils/PermissionsUtils';
import { showErrorToast } from '../../utils/ToastUtils';
import './platform-lineage.less';
const PlatformLineage = () => {
const { t } = useTranslation();
const history = useHistory();
const { entityType } = useParams<{ entityType: EntityType }>();
const { fqn: decodedFqn } = useFqn();
const [selectedEntity, setSelectedEntity] = useState<SourceType>();
const [loading, setLoading] = useState(false);
const [options, setOptions] = useState<DefaultOptionType[]>([]);
const [isSearchLoading, setIsSearchLoading] = useState(false);
const [defaultValue, setDefaultValue] = useState<string | undefined>(
decodedFqn || undefined
);
const [permissions, setPermissions] = useState<OperationPermission>();
const debouncedSearch = useCallback(
debounce(async (value: string) => {
try {
setIsSearchLoading(true);
const searchIndices = [
SearchIndex.DATA_ASSET,
SearchIndex.DOMAIN,
SearchIndex.DATABASE_SERVICE,
SearchIndex.DASHBOARD_SERVICE,
SearchIndex.PIPELINE_SERVICE,
SearchIndex.ML_MODEL_SERVICE,
SearchIndex.STORAGE_SERVICE,
SearchIndex.MESSAGING_SERVICE,
SearchIndex.SEARCH_SERVICE,
SearchIndex.API_SERVICE_INDEX,
];
const response = await searchQuery({
query: `*${value}*`,
searchIndex: searchIndices,
pageSize: PAGE_SIZE_BASE,
queryFilter: getLineageEntityExclusionFilter(),
});
setOptions(
response.hits.hits.map((hit) => ({
value: hit._source.fullyQualifiedName ?? '',
label: (
<EntitySuggestionOption
showEntityTypeBadge
entity={hit._source as EntityReference}
onSelectHandler={handleEntitySelect}
/>
),
data: hit,
}))
);
} finally {
setIsSearchLoading(false);
}
}, 300),
[]
);
const handleEntitySelect = useCallback(
(value: EntityReference) => {
history.push(
`/lineage/${(value as SourceType).entityType}/${
value.fullyQualifiedName
}`
);
},
[history]
);
const init = useCallback(async () => {
if (!decodedFqn || !entityType) {
setDefaultValue(undefined);
return;
}
try {
setLoading(true);
const [entityResponse, permissionResponse] = await Promise.allSettled([
getEntityAPIfromSource(entityType as AssetsUnion)(decodedFqn),
getEntityPermissionByFqn(
entityType as unknown as ResourceEntity,
decodedFqn
),
]);
if (entityResponse.status === 'fulfilled') {
setSelectedEntity(entityResponse.value);
setDefaultValue(decodedFqn || undefined);
}
if (permissionResponse.status === 'fulfilled') {
const operationPermission = getOperationPermissions(
permissionResponse.value
);
setPermissions(operationPermission);
}
} catch (error) {
showErrorToast(error as AxiosError);
} finally {
setLoading(false);
}
}, [decodedFqn, entityType]);
useEffect(() => {
init();
}, [init]);
const lineageElement = useMemo(() => {
if (loading) {
return <Loader />;
}
return (
<LineageProvider>
<Lineage
isPlatformLineage
entity={selectedEntity}
entityType={entityType}
hasEditAccess={
permissions?.EditAll || permissions?.EditLineage || false
}
/>
</LineageProvider>
);
}, [selectedEntity, loading, permissions, entityType]);
return (
<PageLayoutV1 pageTitle={t('label.lineage')}>
<Row gutter={[0, 16]}>
<Col span={24}>
<Row className="p-x-lg">
<Col span={24}>
<PageHeader data={PAGE_HEADERS.PLATFORM_LINEAGE} />
</Col>
<Col span={12}>
<div className="m-t-md w-full">
<Select
showSearch
className="w-full"
data-testid="search-entity-select"
filterOption={false}
loading={isSearchLoading}
optionLabelProp="value"
options={options}
placeholder={t('label.search-entity-for-lineage', {
entity: 'entity',
})}
value={defaultValue}
onFocus={() => !defaultValue && debouncedSearch('')}
onSearch={debouncedSearch}
/>
</div>
</Col>
</Row>
</Col>
<Col span={24}>
<Divider className="m-0" />
<div className="platform-lineage-container">{lineageElement}</div>
</Col>
</Row>
</PageLayoutV1>
);
};
export default PlatformLineage;

23
openmetadata-ui/src/main/resources/ui/src/pages/PlatformLineage/platform-lineage.less Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
.platform-lineage-container {
.lineage-card {
height: calc(100vh - 190px);
}
.full-screen-lineage {
.lineage-card {
height: calc(100vh - 50px);
}
}
}

26
openmetadata-ui/src/main/resources/ui/src/rest/lineageAPI.ts
View File

@ -87,6 +87,32 @@ export const getLineageDataByFQN = async ({
return response.data;
};
export const getPlatformLineage = async ({
config,
queryFilter,
view,
}: {
config?: LineageConfig;
queryFilter?: string;
view: 'service' | 'domain';
}) => {
const { upstreamDepth = 1, downstreamDepth = 1 } = config ?? {};
const API_PATH = `lineage/getPlatformLineage`;
const response = await APIClient.get<LineageData>(API_PATH, {
params: {
view,
upstreamDepth,
downstreamDepth,
query_filter: queryFilter,
includeDeleted: false,
size: config?.nodesPerLayer,
},
});
return response.data;
};
export const getDataQualityLineage = async (
fqn: string,
config?: Partial<LineageConfig>,

33
openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts
View File

@ -19,6 +19,7 @@ import { AsyncDeleteJob } from '../context/AsyncDeleteProvider/AsyncDeleteProvid
import { SearchIndex } from '../enums/search.enum';
import { AuthenticationConfiguration } from '../generated/configuration/authenticationConfiguration';
import { AuthorizerConfiguration } from '../generated/configuration/authorizerConfiguration';
import { SearchRequest } from '../generated/search/searchRequest';
import { ValidationResponse } from '../generated/system/validationResponse';
import { Paging } from '../generated/type/paging';
import { SearchResponse } from '../interface/search.interface';
@ -186,6 +187,38 @@ export const getAggregateFieldOptions = (
);
};
/**
* Posts aggregate field options request with parameters in the body.
*
* @param {SearchIndex | SearchIndex[]} index - The search index or array of search indexes.
* @param {string} field - The field to aggregate on. Example owner.displayName.keyword
* @param {string} value - The value to filter the aggregation on.
* @param {string} q - The search query.
* @return {Promise<SearchResponse<ExploreSearchIndex>>} A promise that resolves to the search response
* containing the aggregate field options.
*/
export const postAggregateFieldOptions = (
index: SearchIndex | SearchIndex[],
field: string,
value: string,
q: string
) => {
const withWildCardValue = value
? `.*${escapeESReservedCharacters(value)}.*`
: '.*';
const body: SearchRequest = {
index: index as string,
fieldName: field,
fieldValue: withWildCardValue,
query: q,
};
return APIClient.post<SearchResponse<ExploreSearchIndex>>(
`/search/aggregate`,
body
);
};
export const getEntityCount = async (
path: string,
database?: string

5
openmetadata-ui/src/main/resources/ui/src/utils/Assets/AssetsUtils.ts
View File

@ -38,6 +38,7 @@ import {
getDataModelByFqn,
patchDataModelDetails,
} from '../../rest/dataModelsAPI';
import { getDomainByName, patchDomains } from '../../rest/domainAPI';
import {
getGlossariesByName,
getGlossaryTermByFQN,
@ -115,6 +116,8 @@ export const getAPIfromSource = (
return patchApiEndPoint;
case EntityType.METRIC:
return patchMetric;
case EntityType.DOMAIN:
return patchDomains;
case EntityType.MESSAGING_SERVICE:
case EntityType.DASHBOARD_SERVICE:
case EntityType.PIPELINE_SERVICE:
@ -176,6 +179,8 @@ export const getEntityAPIfromSource = (
return getApiEndPointByFQN;
case EntityType.METRIC:
return getMetricByFqn;
case EntityType.DOMAIN:
return getDomainByName;
case EntityType.MESSAGING_SERVICE:
case EntityType.DASHBOARD_SERVICE:
case EntityType.PIPELINE_SERVICE:

26
openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx
View File

@ -1666,3 +1666,29 @@ export const removeUnconnectedNodes = (
return updatedNodes;
};
export const getLineageEntityExclusionFilter = () => {
return {
query: {
bool: {
must_not: [
{
term: {
entityType: EntityType.GLOSSARY_TERM,
},
},
{
term: {
entityType: EntityType.TAG,
},
},
{
term: {
entityType: EntityType.DATA_PRODUCT,
},
},
],
},
},
};
};

17
openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.ts
View File

@ -26,6 +26,7 @@ import { SearchDropdownOption } from '../components/SearchDropdown/SearchDropdow
import { NULL_OPTION_KEY } from '../constants/AdvancedSearch.constants';
import { EntityFields } from '../enums/AdvancedSearch.enum';
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import { Aggregations } from '../interface/search.interface';
import {
EsBoolQuery,
@ -33,6 +34,10 @@ import {
QueryFilterInterface,
TabsInfoData,
} from '../pages/ExplorePage/ExplorePage.interface';
import {
getAggregateFieldOptions,
postAggregateFieldOptions,
} from '../rest/miscAPI';
/**
* It takes an array of filters and a data lookup and returns a new object with the filters grouped by
@ -317,3 +322,15 @@ export const getQuickFilterObjectForEntities = (
})),
};
};
export const getAggregationOptions = async (
index: SearchIndex | SearchIndex[],
key: string,
value: string,
filter: string,
isIndependent: boolean
) => {
return isIndependent
? postAggregateFieldOptions(index, key, value, filter)
: getAggregateFieldOptions(index, key, value, filter);
};