refactor(gms): Refactoring util + entity client class locations (#5902)

This commit is contained in:
John Joyce 2022-09-12 12:11:28 -07:00 committed by GitHub
parent 46bc1fe367
commit 450fb2effb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 180 additions and 139 deletions

View File

@ -9,7 +9,7 @@ import com.linkedin.entity.Aspect;
import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EntityResponse;
import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspect;
import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.EnvelopedAspectMap;
import com.linkedin.entity.client.JavaEntityClient; import com.linkedin.metadata.client.JavaEntityClient;
import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.config.IngestionConfiguration;
import com.linkedin.ingestion.DataHubIngestionSourceConfig; import com.linkedin.ingestion.DataHubIngestionSourceConfig;
import com.linkedin.ingestion.DataHubIngestionSourceInfo; import com.linkedin.ingestion.DataHubIngestionSourceInfo;

View File

@ -13,6 +13,7 @@ dependencies {
compile project(':metadata-events:mxe-registration') compile project(':metadata-events:mxe-registration')
compile project(':metadata-events:mxe-utils-avro-1.7') compile project(':metadata-events:mxe-utils-avro-1.7')
compile project(path: ':metadata-models') compile project(path: ':metadata-models')
compile project(':metadata-service:restli-client')
compile spec.product.pegasus.data compile spec.product.pegasus.data
compile spec.product.pegasus.generator compile spec.product.pegasus.generator

View File

@ -1,4 +1,4 @@
package com.linkedin.entity.client; package com.linkedin.metadata.client;
import com.datahub.authentication.Authentication; import com.datahub.authentication.Authentication;
import com.datahub.util.RecordUtils; import com.datahub.util.RecordUtils;
@ -13,6 +13,8 @@ import com.linkedin.data.template.RecordTemplate;
import com.linkedin.data.template.StringArray; import com.linkedin.data.template.StringArray;
import com.linkedin.entity.Entity; import com.linkedin.entity.Entity;
import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EntityResponse;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.entity.client.RestliEntityClient;
import com.linkedin.metadata.Constants; import com.linkedin.metadata.Constants;
import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.aspect.EnvelopedAspect;
import com.linkedin.metadata.aspect.EnvelopedAspectArray; import com.linkedin.metadata.aspect.EnvelopedAspectArray;
@ -27,14 +29,14 @@ import com.linkedin.metadata.query.ListResult;
import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.query.ListUrnsResult;
import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.resources.entity.AspectUtils; import com.linkedin.metadata.entity.AspectUtils;
import com.linkedin.metadata.resources.entity.EntityResource;
import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.EntitySearchService;
import com.linkedin.metadata.search.LineageSearchResult; import com.linkedin.metadata.search.LineageSearchResult;
import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.LineageSearchService;
import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchResult;
import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.SearchService;
import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.search.client.CachingEntitySearchService;
import com.linkedin.metadata.shared.ValidationUtils;
import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.timeseries.TimeseriesAspectService;
import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.MetadataChangeProposal;
import com.linkedin.mxe.PlatformEvent; import com.linkedin.mxe.PlatformEvent;
@ -56,8 +58,8 @@ import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import static com.linkedin.metadata.resources.entity.ResourceUtils.*;
import static com.linkedin.metadata.search.utils.QueryUtils.*; import static com.linkedin.metadata.search.utils.QueryUtils.*;
import static com.linkedin.metadata.search.utils.SearchUtils.*;
@Slf4j @Slf4j
@ -181,7 +183,7 @@ public class JavaEntityClient implements EntityClient {
int start, int start,
int limit, int limit,
@Nonnull final Authentication authentication) throws RemoteInvocationException { @Nonnull final Authentication authentication) throws RemoteInvocationException {
return validateBrowseResult( return ValidationUtils.validateBrowseResult(
_entitySearchService.browse(entityType, path, newFilter(requestFilters), start, limit), _entityService); _entitySearchService.browse(entityType, path, newFilter(requestFilters), start, limit), _entityService);
} }
@ -245,7 +247,7 @@ public class JavaEntityClient implements EntityClient {
int count, int count,
@Nonnull final Authentication authentication) @Nonnull final Authentication authentication)
throws RemoteInvocationException { throws RemoteInvocationException {
return validateSearchResult( return ValidationUtils.validateSearchResult(
_entitySearchService.search(entity, input, newFilter(requestFilters), null, start, count), _entityService); _entitySearchService.search(entity, input, newFilter(requestFilters), null, start, count), _entityService);
} }
@ -269,7 +271,7 @@ public class JavaEntityClient implements EntityClient {
int count, int count,
@Nonnull final Authentication authentication) @Nonnull final Authentication authentication)
throws RemoteInvocationException { throws RemoteInvocationException {
return validateListResult(EntityResource.toListResult( return ValidationUtils.validateListResult(toListResult(
_entitySearchService.filter(entity, newFilter(requestFilters), null, start, count)), _entityService); _entitySearchService.filter(entity, newFilter(requestFilters), null, start, count)), _entityService);
} }
@ -294,7 +296,7 @@ public class JavaEntityClient implements EntityClient {
int count, int count,
@Nonnull final Authentication authentication) @Nonnull final Authentication authentication)
throws RemoteInvocationException { throws RemoteInvocationException {
return validateSearchResult(_entitySearchService.search(entity, input, filter, sortCriterion, start, count), return ValidationUtils.validateSearchResult(_entitySearchService.search(entity, input, filter, sortCriterion, start, count),
_entityService); _entityService);
} }
@ -317,7 +319,7 @@ public class JavaEntityClient implements EntityClient {
int start, int start,
int count, int count,
@Nonnull final Authentication authentication) throws RemoteInvocationException { @Nonnull final Authentication authentication) throws RemoteInvocationException {
return validateSearchResult( return ValidationUtils.validateSearchResult(
_searchService.searchAcrossEntities(entities, input, filter, null, start, count, null), _entityService); _searchService.searchAcrossEntities(entities, input, filter, null, start, count, null), _entityService);
} }
@ -327,7 +329,7 @@ public class JavaEntityClient implements EntityClient {
@Nonnull List<String> entities, @Nullable String input, @Nullable Integer maxHops, @Nullable Filter filter, @Nonnull List<String> entities, @Nullable String input, @Nullable Integer maxHops, @Nullable Filter filter,
@Nullable SortCriterion sortCriterion, int start, int count, @Nonnull final Authentication authentication) @Nullable SortCriterion sortCriterion, int start, int count, @Nonnull final Authentication authentication)
throws RemoteInvocationException { throws RemoteInvocationException {
return validateLineageSearchResult( return ValidationUtils.validateLineageSearchResult(
_lineageSearchService.searchAcrossLineage(sourceUrn, direction, entities, input, maxHops, filter, _lineageSearchService.searchAcrossLineage(sourceUrn, direction, entities, input, maxHops, filter,
sortCriterion, start, count), _entityService); sortCriterion, start, count), _entityService);
} }
@ -380,7 +382,7 @@ public class JavaEntityClient implements EntityClient {
@Override @Override
public SearchResult filter(@Nonnull String entity, @Nonnull Filter filter, @Nullable SortCriterion sortCriterion, public SearchResult filter(@Nonnull String entity, @Nonnull Filter filter, @Nullable SortCriterion sortCriterion,
int start, int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { int start, int count, @Nonnull final Authentication authentication) throws RemoteInvocationException {
return validateSearchResult(_entitySearchService.filter(entity, filter, sortCriterion, start, count), return ValidationUtils.validateSearchResult(_entitySearchService.filter(entity, filter, sortCriterion, start, count),
_entityService); _entityService);
} }

View File

@ -1,10 +1,9 @@
package com.linkedin.metadata.resources.entity; package com.linkedin.metadata.entity;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.RecordTemplate;
import com.linkedin.events.metadata.ChangeType; import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.EntityKeyUtils;
import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.utils.GenericRecordUtils;
import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.GenericAspect;

View File

@ -1,7 +1,9 @@
package com.linkedin.metadata.search.utils; package com.linkedin.metadata.search.utils;
import com.linkedin.common.UrnArray;
import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.LongMap; import com.linkedin.data.template.LongMap;
import com.linkedin.metadata.query.ListResult;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
import com.linkedin.metadata.query.filter.Criterion; import com.linkedin.metadata.query.filter.Criterion;
@ -9,6 +11,7 @@ import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.search.AggregationMetadata; import com.linkedin.metadata.search.AggregationMetadata;
import com.linkedin.metadata.search.FilterValueArray; import com.linkedin.metadata.search.FilterValueArray;
import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchResult;
import com.linkedin.metadata.search.SearchResultMetadata; import com.linkedin.metadata.search.SearchResultMetadata;
@ -137,4 +140,17 @@ public class SearchUtils {
.setAggregations(new LongMap(mergedMap)) .setAggregations(new LongMap(mergedMap))
.setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(mergedMap))); .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(mergedMap)));
} }
public static ListResult toListResult(final SearchResult searchResult) {
if (searchResult == null) {
return null;
}
final ListResult listResult = new ListResult();
listResult.setStart(searchResult.getFrom());
listResult.setCount(searchResult.getPageSize());
listResult.setTotal(searchResult.getNumEntities());
listResult.setEntities(
new UrnArray(searchResult.getEntities().stream().map(SearchEntity::getEntity).collect(Collectors.toList())));
return listResult;
}
} }

View File

@ -0,0 +1,107 @@
package com.linkedin.metadata.shared;
import com.linkedin.common.UrnArray;
import com.linkedin.metadata.browse.BrowseResult;
import com.linkedin.metadata.browse.BrowseResultEntityArray;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.query.ListResult;
import com.linkedin.metadata.search.LineageSearchEntityArray;
import com.linkedin.metadata.search.LineageSearchResult;
import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
public class ValidationUtils {
public static SearchResult validateSearchResult(final SearchResult searchResult,
@Nonnull final EntityService entityService) {
if (searchResult == null) {
return null;
}
Objects.requireNonNull(entityService, "entityService must not be null");
SearchResult validatedSearchResult = new SearchResult().setFrom(searchResult.getFrom())
.setMetadata(searchResult.getMetadata())
.setPageSize(searchResult.getPageSize())
.setNumEntities(searchResult.getNumEntities());
SearchEntityArray validatedEntities = searchResult.getEntities()
.stream()
.filter(searchEntity -> entityService.exists(searchEntity.getEntity()))
.collect(Collectors.toCollection(SearchEntityArray::new));
validatedSearchResult.setEntities(validatedEntities);
return validatedSearchResult;
}
public static BrowseResult validateBrowseResult(final BrowseResult browseResult,
@Nonnull final EntityService entityService) {
if (browseResult == null) {
return null;
}
Objects.requireNonNull(entityService, "entityService must not be null");
BrowseResult validatedBrowseResult = new BrowseResult().setGroups(browseResult.getGroups())
.setMetadata(browseResult.getMetadata())
.setFrom(browseResult.getFrom())
.setPageSize(browseResult.getPageSize())
.setNumGroups(browseResult.getNumGroups())
.setNumEntities(browseResult.getNumEntities())
.setNumElements(browseResult.getNumElements());
BrowseResultEntityArray validatedEntities = browseResult.getEntities()
.stream()
.filter(browseResultEntity -> entityService.exists(browseResultEntity.getUrn()))
.collect(Collectors.toCollection(BrowseResultEntityArray::new));
validatedBrowseResult.setEntities(validatedEntities);
return validatedBrowseResult;
}
public static ListResult validateListResult(final ListResult listResult, @Nonnull final EntityService entityService) {
if (listResult == null) {
return null;
}
Objects.requireNonNull(entityService, "entityService must not be null");
ListResult validatedListResult = new ListResult().setStart(listResult.getStart())
.setCount(listResult.getCount())
.setTotal(listResult.getTotal());
UrnArray validatedEntities =
listResult.getEntities().stream().filter(entityService::exists).collect(Collectors.toCollection(UrnArray::new));
validatedListResult.setEntities(validatedEntities);
return validatedListResult;
}
public static LineageSearchResult validateLineageSearchResult(final LineageSearchResult lineageSearchResult,
@Nonnull final EntityService entityService) {
if (lineageSearchResult == null) {
return null;
}
Objects.requireNonNull(entityService, "entityService must not be null");
LineageSearchResult validatedLineageSearchResult =
new LineageSearchResult().setMetadata(lineageSearchResult.getMetadata())
.setFrom(lineageSearchResult.getFrom())
.setPageSize(lineageSearchResult.getPageSize())
.setNumEntities(lineageSearchResult.getNumEntities());
LineageSearchEntityArray validatedEntities = lineageSearchResult.getEntities()
.stream()
.filter(entity -> entityService.exists(entity.getEntity()))
.collect(Collectors.toCollection(LineageSearchEntityArray::new));
validatedLineageSearchResult.setEntities(validatedEntities);
return validatedLineageSearchResult;
}
private ValidationUtils() {
}
}

View File

@ -7,8 +7,8 @@ compileJava {
dependencies { dependencies {
compile project(path: ':metadata-models') compile project(path: ':metadata-models')
compile project(path: ':metadata-service:restli-client')
compile project(path: ':metadata-service:auth-api') compile project(path: ':metadata-service:auth-api')
compile project(path: ':metadata-io')
implementation 'io.jsonwebtoken:jjwt-api:0.11.2' implementation 'io.jsonwebtoken:jjwt-api:0.11.2'
runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.11.2', runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.11.2',

View File

@ -10,9 +10,9 @@ import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils; import com.linkedin.common.urn.UrnUtils;
import com.linkedin.events.metadata.ChangeType; import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.Constants; import com.linkedin.metadata.Constants;
import com.linkedin.metadata.entity.AspectUtils;
import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.key.DataHubAccessTokenKey; import com.linkedin.metadata.key.DataHubAccessTokenKey;
import com.linkedin.metadata.resources.entity.AspectUtils;
import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.AuditStampUtils;
import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.utils.GenericRecordUtils;
import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.MetadataChangeProposal;

View File

@ -8,7 +8,7 @@ import com.datahub.authorization.Authorizer;
import com.datahub.authorization.AuthorizerChain; import com.datahub.authorization.AuthorizerChain;
import com.datahub.authorization.DefaultResourceSpecResolver; import com.datahub.authorization.DefaultResourceSpecResolver;
import com.datahub.authorization.ResourceSpecResolver; import com.datahub.authorization.ResourceSpecResolver;
import com.linkedin.entity.client.JavaEntityClient; import com.linkedin.metadata.client.JavaEntityClient;
import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.config.ConfigurationProvider;
import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory;
import java.util.ArrayList; import java.util.ArrayList;

View File

@ -2,7 +2,7 @@ package com.linkedin.gms.factory.auth;
import com.datahub.authentication.Authentication; import com.datahub.authentication.Authentication;
import com.datahub.authorization.DataHubAuthorizer; import com.datahub.authorization.DataHubAuthorizer;
import com.linkedin.entity.client.JavaEntityClient; import com.linkedin.metadata.client.JavaEntityClient;
import com.linkedin.gms.factory.entity.RestliEntityClientFactory; import com.linkedin.gms.factory.entity.RestliEntityClientFactory;
import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;

View File

@ -3,7 +3,7 @@
package com.linkedin.gms.factory.auth; package com.linkedin.gms.factory.auth;
import com.datahub.authentication.group.GroupService; import com.datahub.authentication.group.GroupService;
import com.linkedin.entity.client.JavaEntityClient; import com.linkedin.metadata.client.JavaEntityClient;
import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory;
import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.graph.GraphClient;

View File

@ -3,7 +3,7 @@
package com.linkedin.gms.factory.auth; package com.linkedin.gms.factory.auth;
import com.datahub.authentication.user.NativeUserService; import com.datahub.authentication.user.NativeUserService;
import com.linkedin.entity.client.JavaEntityClient; import com.linkedin.metadata.client.JavaEntityClient;
import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory;
import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.secret.SecretService; import com.linkedin.metadata.secret.SecretService;

View File

@ -1,6 +1,6 @@
package com.linkedin.gms.factory.entity; package com.linkedin.gms.factory.entity;
import com.linkedin.entity.client.JavaEntityClient; import com.linkedin.metadata.client.JavaEntityClient;
import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.entity.client.RestliEntityClient;
import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory;
import com.linkedin.metadata.entity.DeleteEntityService; import com.linkedin.metadata.entity.DeleteEntityService;

View File

@ -6,7 +6,7 @@ import com.datahub.authentication.user.NativeUserService;
import com.linkedin.datahub.graphql.GmsGraphQLEngine; import com.linkedin.datahub.graphql.GmsGraphQLEngine;
import com.linkedin.datahub.graphql.GraphQLEngine; import com.linkedin.datahub.graphql.GraphQLEngine;
import com.linkedin.datahub.graphql.analytics.service.AnalyticsService; import com.linkedin.datahub.graphql.analytics.service.AnalyticsService;
import com.linkedin.entity.client.JavaEntityClient; import com.linkedin.metadata.client.JavaEntityClient;
import com.linkedin.gms.factory.auth.DataHubTokenServiceFactory; import com.linkedin.gms.factory.auth.DataHubTokenServiceFactory;
import com.linkedin.gms.factory.common.GitVersionFactory; import com.linkedin.gms.factory.common.GitVersionFactory;
import com.linkedin.gms.factory.common.IndexConventionFactory; import com.linkedin.gms.factory.common.IndexConventionFactory;

View File

@ -18,7 +18,7 @@ import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.entity.RollbackRunResult; import com.linkedin.metadata.entity.RollbackRunResult;
import com.linkedin.metadata.entity.ValidationException; import com.linkedin.metadata.entity.ValidationException;
import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.EntitySpec;
import com.linkedin.metadata.resources.entity.AspectUtils; import com.linkedin.metadata.entity.AspectUtils;
import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.metadata.utils.metrics.MetricUtils;
import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.GenericAspect;
import com.linkedin.mxe.SystemMetadata; import com.linkedin.mxe.SystemMetadata;

View File

@ -3,11 +3,9 @@ apply plugin: 'pegasus'
dependencies { dependencies {
compile project(':metadata-service:restli-api') compile project(':metadata-service:restli-api')
compile project(':metadata-service:auth-api') compile project(':metadata-service:auth-api')
compile project(':metadata-service:restli-servlet-impl')
compile project(path: ':metadata-service:restli-api', configuration: 'restClient') compile project(path: ':metadata-service:restli-api', configuration: 'restClient')
compile project(':metadata-events:mxe-schemas') compile project(':metadata-events:mxe-schemas')
compile project(':metadata-utils')
dataModel project(':li-utils')
compile externalDependency.lombok compile externalDependency.lombok
annotationProcessor externalDependency.lombok annotationProcessor externalDependency.lombok

View File

@ -2,6 +2,7 @@ package com.linkedin.entity.client;
import com.datahub.authentication.Authentication; import com.datahub.authentication.Authentication;
import com.datahub.util.RecordUtils; import com.datahub.util.RecordUtils;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.VersionedUrn; import com.linkedin.common.VersionedUrn;
import com.linkedin.common.client.BaseClient; import com.linkedin.common.client.BaseClient;
import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.Urn;
@ -48,6 +49,11 @@ import com.linkedin.metadata.graph.LineageDirection;
import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.AutoCompleteResult;
import com.linkedin.metadata.query.ListResult; import com.linkedin.metadata.query.ListResult;
import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.query.ListUrnsResult;
import com.linkedin.metadata.query.filter.Condition;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortCriterion;
import com.linkedin.metadata.search.LineageSearchResult; import com.linkedin.metadata.search.LineageSearchResult;
@ -66,6 +72,7 @@ import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@ -76,8 +83,6 @@ import javax.mail.MethodNotSupportedException;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import static com.linkedin.metadata.search.utils.QueryUtils.newFilter;
@Slf4j @Slf4j
public class RestliEntityClient extends BaseClient implements EntityClient { public class RestliEntityClient extends BaseClient implements EntityClient {
@ -681,4 +686,25 @@ public class RestliEntityClient extends BaseClient implements EntityClient {
final RunsDoRollbackRequestBuilder requestBuilder = RUNS_REQUEST_BUILDERS.actionRollback().runIdParam(runId).dryRunParam(false); final RunsDoRollbackRequestBuilder requestBuilder = RUNS_REQUEST_BUILDERS.actionRollback().runIdParam(runId).dryRunParam(false);
sendClientRequest(requestBuilder, authentication); sendClientRequest(requestBuilder, authentication);
} }
// TODO: Refactor QueryUtils inside of metadata-io to extract these methods into a single shared library location.
// Creates new Filter from a map of Criteria by removing null-valued Criteria and using EQUAL condition (default).
@Nonnull
public static Filter newFilter(@Nullable Map<String, String> params) {
if (params == null) {
return new Filter().setOr(new ConjunctiveCriterionArray());
}
CriterionArray criteria = params.entrySet()
.stream()
.filter(e -> Objects.nonNull(e.getValue()))
.map(e -> newCriterion(e.getKey(), e.getValue(), Condition.EQUAL))
.collect(Collectors.toCollection(CriterionArray::new));
return new Filter().setOr(
new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion().setAnd(criteria))));
}
@Nonnull
public static Criterion newCriterion(@Nonnull String field, @Nonnull String value, @Nonnull Condition condition) {
return new Criterion().setField(field).setValue(value).setCondition(condition);
}
} }

View File

@ -8,6 +8,7 @@ import com.linkedin.common.AuditStamp;
import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.Urn;
import com.linkedin.metadata.aspect.EnvelopedAspectArray; import com.linkedin.metadata.aspect.EnvelopedAspectArray;
import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.aspect.VersionedAspect;
import com.linkedin.metadata.entity.AspectUtils;
import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.entity.ValidationException; import com.linkedin.metadata.entity.ValidationException;
import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs;

View File

@ -5,7 +5,6 @@ import com.datahub.authentication.Authentication;
import com.datahub.authentication.AuthenticationContext; import com.datahub.authentication.AuthenticationContext;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.linkedin.common.AuditStamp; import com.linkedin.common.AuditStamp;
import com.linkedin.common.UrnArray;
import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.LongMap; import com.linkedin.data.template.LongMap;
import com.linkedin.data.template.StringArray; import com.linkedin.data.template.StringArray;
@ -35,7 +34,6 @@ import com.linkedin.metadata.run.RollbackResponse;
import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.EntitySearchService;
import com.linkedin.metadata.search.LineageSearchResult; import com.linkedin.metadata.search.LineageSearchResult;
import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.LineageSearchService;
import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchResult;
import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.SearchService;
import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.metadata.search.utils.ESUtils;
@ -77,6 +75,8 @@ import org.apache.maven.artifact.versioning.ComparableVersion;
import static com.linkedin.metadata.entity.ValidationUtils.*; import static com.linkedin.metadata.entity.ValidationUtils.*;
import static com.linkedin.metadata.resources.entity.ResourceUtils.*; import static com.linkedin.metadata.resources.entity.ResourceUtils.*;
import static com.linkedin.metadata.resources.restli.RestliConstants.*; import static com.linkedin.metadata.resources.restli.RestliConstants.*;
import static com.linkedin.metadata.search.utils.SearchUtils.*;
import static com.linkedin.metadata.shared.ValidationUtils.*;
import static com.linkedin.metadata.utils.PegasusUtils.*; import static com.linkedin.metadata.utils.PegasusUtils.*;
@ -144,19 +144,6 @@ public class EntityResource extends CollectionResourceTaskTemplate<String, Entit
@Named("timeseriesAspectService") @Named("timeseriesAspectService")
private TimeseriesAspectService _timeseriesAspectService; private TimeseriesAspectService _timeseriesAspectService;
public static ListResult toListResult(final SearchResult searchResult) {
if (searchResult == null) {
return null;
}
final ListResult listResult = new ListResult();
listResult.setStart(searchResult.getFrom());
listResult.setCount(searchResult.getPageSize());
listResult.setTotal(searchResult.getNumEntities());
listResult.setEntities(
new UrnArray(searchResult.getEntities().stream().map(SearchEntity::getEntity).collect(Collectors.toList())));
return listResult;
}
/** /**
* Retrieves the value for an entity that is made up of latest versions of specified aspects. * Retrieves the value for an entity that is made up of latest versions of specified aspects.
*/ */

View File

@ -1,21 +1,10 @@
package com.linkedin.metadata.resources.entity; package com.linkedin.metadata.resources.entity;
import com.linkedin.common.UrnArray;
import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.Urn;
import com.linkedin.metadata.browse.BrowseResult;
import com.linkedin.metadata.browse.BrowseResultEntityArray;
import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.query.ListResult;
import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.EntitySearchService;
import com.linkedin.metadata.search.LineageSearchEntityArray;
import com.linkedin.metadata.search.LineageSearchResult;
import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchResult;
import com.linkedin.mxe.SystemMetadata; import com.linkedin.mxe.SystemMetadata;
import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -34,89 +23,4 @@ public class ResourceUtils {
entitySearchService.appendRunId(urn.getEntityType(), urn, systemMetadata.getRunId()); entitySearchService.appendRunId(urn.getEntityType(), urn, systemMetadata.getRunId());
} }
} }
public static SearchResult validateSearchResult(final SearchResult searchResult,
@Nonnull final EntityService entityService) {
if (searchResult == null) {
return null;
}
Objects.requireNonNull(entityService, "entityService must not be null");
SearchResult validatedSearchResult = new SearchResult().setFrom(searchResult.getFrom())
.setMetadata(searchResult.getMetadata())
.setPageSize(searchResult.getPageSize())
.setNumEntities(searchResult.getNumEntities());
SearchEntityArray validatedEntities = searchResult.getEntities()
.stream()
.filter(searchEntity -> entityService.exists(searchEntity.getEntity()))
.collect(Collectors.toCollection(SearchEntityArray::new));
validatedSearchResult.setEntities(validatedEntities);
return validatedSearchResult;
}
public static BrowseResult validateBrowseResult(final BrowseResult browseResult,
@Nonnull final EntityService entityService) {
if (browseResult == null) {
return null;
}
Objects.requireNonNull(entityService, "entityService must not be null");
BrowseResult validatedBrowseResult = new BrowseResult().setGroups(browseResult.getGroups())
.setMetadata(browseResult.getMetadata())
.setFrom(browseResult.getFrom())
.setPageSize(browseResult.getPageSize())
.setNumGroups(browseResult.getNumGroups())
.setNumEntities(browseResult.getNumEntities())
.setNumElements(browseResult.getNumElements());
BrowseResultEntityArray validatedEntities = browseResult.getEntities()
.stream()
.filter(browseResultEntity -> entityService.exists(browseResultEntity.getUrn()))
.collect(Collectors.toCollection(BrowseResultEntityArray::new));
validatedBrowseResult.setEntities(validatedEntities);
return validatedBrowseResult;
}
public static ListResult validateListResult(final ListResult listResult, @Nonnull final EntityService entityService) {
if (listResult == null) {
return null;
}
Objects.requireNonNull(entityService, "entityService must not be null");
ListResult validatedListResult = new ListResult().setStart(listResult.getStart())
.setCount(listResult.getCount())
.setTotal(listResult.getTotal());
UrnArray validatedEntities =
listResult.getEntities().stream().filter(entityService::exists).collect(Collectors.toCollection(UrnArray::new));
validatedListResult.setEntities(validatedEntities);
return validatedListResult;
}
public static LineageSearchResult validateLineageSearchResult(final LineageSearchResult lineageSearchResult,
@Nonnull final EntityService entityService) {
if (lineageSearchResult == null) {
return null;
}
Objects.requireNonNull(entityService, "entityService must not be null");
LineageSearchResult validatedLineageSearchResult =
new LineageSearchResult().setMetadata(lineageSearchResult.getMetadata())
.setFrom(lineageSearchResult.getFrom())
.setPageSize(lineageSearchResult.getPageSize())
.setNumEntities(lineageSearchResult.getNumEntities());
LineageSearchEntityArray validatedEntities = lineageSearchResult.getEntities()
.stream()
.filter(entity -> entityService.exists(entity.getEntity()))
.collect(Collectors.toCollection(LineageSearchEntityArray::new));
validatedLineageSearchResult.setEntities(validatedEntities);
return validatedLineageSearchResult;
}
} }