diff --git a/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java b/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java index 1e0362d1b6..3397d07d69 100644 --- a/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java +++ b/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java @@ -9,7 +9,7 @@ import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; -import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.ingestion.DataHubIngestionSourceConfig; import com.linkedin.ingestion.DataHubIngestionSourceInfo; diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index c223f896d5..9ee3d558fe 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -13,6 +13,7 @@ dependencies { compile project(':metadata-events:mxe-registration') compile project(':metadata-events:mxe-utils-avro-1.7') compile project(path: ':metadata-models') + compile project(':metadata-service:restli-client') compile spec.product.pegasus.data compile spec.product.pegasus.generator diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java similarity index 95% rename from metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java rename to metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java index 5bd5a5f1dd..8af042320c 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java @@ -1,4 +1,4 @@ -package com.linkedin.entity.client; +package com.linkedin.metadata.client; import com.datahub.authentication.Authentication; import com.datahub.util.RecordUtils; @@ -13,6 +13,8 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringArray; import com.linkedin.entity.Entity; import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.aspect.EnvelopedAspectArray; @@ -27,14 +29,14 @@ import com.linkedin.metadata.query.ListResult; import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; -import com.linkedin.metadata.resources.entity.AspectUtils; -import com.linkedin.metadata.resources.entity.EntityResource; +import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.LineageSearchResult; import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.client.CachingEntitySearchService; +import com.linkedin.metadata.shared.ValidationUtils; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.PlatformEvent; @@ -56,8 +58,8 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.resources.entity.ResourceUtils.*; import static com.linkedin.metadata.search.utils.QueryUtils.*; +import static com.linkedin.metadata.search.utils.SearchUtils.*; @Slf4j @@ -181,7 +183,7 @@ public class JavaEntityClient implements EntityClient { int start, int limit, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return validateBrowseResult( + return ValidationUtils.validateBrowseResult( _entitySearchService.browse(entityType, path, newFilter(requestFilters), start, limit), _entityService); } @@ -245,7 +247,7 @@ public class JavaEntityClient implements EntityClient { int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return validateSearchResult( + return ValidationUtils.validateSearchResult( _entitySearchService.search(entity, input, newFilter(requestFilters), null, start, count), _entityService); } @@ -269,7 +271,7 @@ public class JavaEntityClient implements EntityClient { int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return validateListResult(EntityResource.toListResult( + return ValidationUtils.validateListResult(toListResult( _entitySearchService.filter(entity, newFilter(requestFilters), null, start, count)), _entityService); } @@ -294,7 +296,7 @@ public class JavaEntityClient implements EntityClient { int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return validateSearchResult(_entitySearchService.search(entity, input, filter, sortCriterion, start, count), + return ValidationUtils.validateSearchResult(_entitySearchService.search(entity, input, filter, sortCriterion, start, count), _entityService); } @@ -317,7 +319,7 @@ public class JavaEntityClient implements EntityClient { int start, int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return validateSearchResult( + return ValidationUtils.validateSearchResult( _searchService.searchAcrossEntities(entities, input, filter, null, start, count, null), _entityService); } @@ -327,7 +329,7 @@ public class JavaEntityClient implements EntityClient { @Nonnull List entities, @Nullable String input, @Nullable Integer maxHops, @Nullable Filter filter, @Nullable SortCriterion sortCriterion, int start, int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return validateLineageSearchResult( + return ValidationUtils.validateLineageSearchResult( _lineageSearchService.searchAcrossLineage(sourceUrn, direction, entities, input, maxHops, filter, sortCriterion, start, count), _entityService); } @@ -380,7 +382,7 @@ public class JavaEntityClient implements EntityClient { @Override public SearchResult filter(@Nonnull String entity, @Nonnull Filter filter, @Nullable SortCriterion sortCriterion, int start, int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return validateSearchResult(_entitySearchService.filter(entity, filter, sortCriterion, start, count), + return ValidationUtils.validateSearchResult(_entitySearchService.filter(entity, filter, sortCriterion, start, count), _entityService); } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectUtils.java similarity index 95% rename from metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectUtils.java rename to metadata-io/src/main/java/com/linkedin/metadata/entity/AspectUtils.java index 3d309a9477..ea80e44a37 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectUtils.java @@ -1,10 +1,9 @@ -package com.linkedin.metadata.resources.entity; +package com.linkedin.metadata.entity; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.GenericAspect; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java index 8d5895d120..b80503740c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/SearchUtils.java @@ -1,7 +1,9 @@ package com.linkedin.metadata.search.utils; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.LongMap; +import com.linkedin.metadata.query.ListResult; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; import com.linkedin.metadata.query.filter.Criterion; @@ -9,6 +11,7 @@ import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.AggregationMetadata; import com.linkedin.metadata.search.FilterValueArray; +import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchResultMetadata; @@ -137,4 +140,17 @@ public class SearchUtils { .setAggregations(new LongMap(mergedMap)) .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(mergedMap))); } + + public static ListResult toListResult(final SearchResult searchResult) { + if (searchResult == null) { + return null; + } + final ListResult listResult = new ListResult(); + listResult.setStart(searchResult.getFrom()); + listResult.setCount(searchResult.getPageSize()); + listResult.setTotal(searchResult.getNumEntities()); + listResult.setEntities( + new UrnArray(searchResult.getEntities().stream().map(SearchEntity::getEntity).collect(Collectors.toList()))); + return listResult; + } } \ No newline at end of file diff --git a/metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java new file mode 100644 index 0000000000..074436fb6f --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java @@ -0,0 +1,107 @@ +package com.linkedin.metadata.shared; + +import com.linkedin.common.UrnArray; +import com.linkedin.metadata.browse.BrowseResult; +import com.linkedin.metadata.browse.BrowseResultEntityArray; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.query.ListResult; +import com.linkedin.metadata.search.LineageSearchEntityArray; +import com.linkedin.metadata.search.LineageSearchResult; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import java.util.Objects; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + + +public class ValidationUtils { + + + public static SearchResult validateSearchResult(final SearchResult searchResult, + @Nonnull final EntityService entityService) { + if (searchResult == null) { + return null; + } + Objects.requireNonNull(entityService, "entityService must not be null"); + + SearchResult validatedSearchResult = new SearchResult().setFrom(searchResult.getFrom()) + .setMetadata(searchResult.getMetadata()) + .setPageSize(searchResult.getPageSize()) + .setNumEntities(searchResult.getNumEntities()); + + SearchEntityArray validatedEntities = searchResult.getEntities() + .stream() + .filter(searchEntity -> entityService.exists(searchEntity.getEntity())) + .collect(Collectors.toCollection(SearchEntityArray::new)); + validatedSearchResult.setEntities(validatedEntities); + + return validatedSearchResult; + } + + public static BrowseResult validateBrowseResult(final BrowseResult browseResult, + @Nonnull final EntityService entityService) { + if (browseResult == null) { + return null; + } + Objects.requireNonNull(entityService, "entityService must not be null"); + + BrowseResult validatedBrowseResult = new BrowseResult().setGroups(browseResult.getGroups()) + .setMetadata(browseResult.getMetadata()) + .setFrom(browseResult.getFrom()) + .setPageSize(browseResult.getPageSize()) + .setNumGroups(browseResult.getNumGroups()) + .setNumEntities(browseResult.getNumEntities()) + .setNumElements(browseResult.getNumElements()); + + BrowseResultEntityArray validatedEntities = browseResult.getEntities() + .stream() + .filter(browseResultEntity -> entityService.exists(browseResultEntity.getUrn())) + .collect(Collectors.toCollection(BrowseResultEntityArray::new)); + validatedBrowseResult.setEntities(validatedEntities); + + + return validatedBrowseResult; + } + + public static ListResult validateListResult(final ListResult listResult, @Nonnull final EntityService entityService) { + if (listResult == null) { + return null; + } + Objects.requireNonNull(entityService, "entityService must not be null"); + + ListResult validatedListResult = new ListResult().setStart(listResult.getStart()) + .setCount(listResult.getCount()) + .setTotal(listResult.getTotal()); + + UrnArray validatedEntities = + listResult.getEntities().stream().filter(entityService::exists).collect(Collectors.toCollection(UrnArray::new)); + validatedListResult.setEntities(validatedEntities); + + return validatedListResult; + } + + public static LineageSearchResult validateLineageSearchResult(final LineageSearchResult lineageSearchResult, + @Nonnull final EntityService entityService) { + if (lineageSearchResult == null) { + return null; + } + Objects.requireNonNull(entityService, "entityService must not be null"); + + LineageSearchResult validatedLineageSearchResult = + new LineageSearchResult().setMetadata(lineageSearchResult.getMetadata()) + .setFrom(lineageSearchResult.getFrom()) + .setPageSize(lineageSearchResult.getPageSize()) + .setNumEntities(lineageSearchResult.getNumEntities()); + + LineageSearchEntityArray validatedEntities = lineageSearchResult.getEntities() + .stream() + .filter(entity -> entityService.exists(entity.getEntity())) + .collect(Collectors.toCollection(LineageSearchEntityArray::new)); + validatedLineageSearchResult.setEntities(validatedEntities); + + return validatedLineageSearchResult; + } + + private ValidationUtils() { + } +} diff --git a/metadata-service/auth-impl/build.gradle b/metadata-service/auth-impl/build.gradle index 0887d7199f..76b18f314e 100644 --- a/metadata-service/auth-impl/build.gradle +++ b/metadata-service/auth-impl/build.gradle @@ -7,8 +7,8 @@ compileJava { dependencies { compile project(path: ':metadata-models') - compile project(path: ':metadata-service:restli-client') compile project(path: ':metadata-service:auth-api') + compile project(path: ':metadata-io') implementation 'io.jsonwebtoken:jjwt-api:0.11.2' runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.11.2', diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java index 98b53a797c..7ff4bff3b5 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java @@ -10,9 +10,9 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.key.DataHubAccessTokenKey; -import com.linkedin.metadata.resources.entity.AspectUtils; import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java index f9be50fc36..e2d814b7e2 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java @@ -8,7 +8,7 @@ import com.datahub.authorization.Authorizer; import com.datahub.authorization.AuthorizerChain; import com.datahub.authorization.DefaultResourceSpecResolver; import com.datahub.authorization.ResourceSpecResolver; -import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import java.util.ArrayList; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java index e5e377b577..30e03d87a8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java @@ -2,7 +2,7 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.Authentication; import com.datahub.authorization.DataHubAuthorizer; -import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.gms.factory.entity.RestliEntityClientFactory; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java index fd77cb606c..9d29b8e77d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java @@ -3,7 +3,7 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.group.GroupService; -import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java index 9f510444a5..ca52420b44 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java @@ -3,7 +3,7 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.user.NativeUserService; -import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.secret.SecretService; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java index 27a15e0281..c9c3953f4d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java @@ -1,6 +1,6 @@ package com.linkedin.gms.factory.entity; -import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; import com.linkedin.metadata.entity.DeleteEntityService; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index a860c1b467..5c4a1e1b9d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -6,7 +6,7 @@ import com.datahub.authentication.user.NativeUserService; import com.linkedin.datahub.graphql.GmsGraphQLEngine; import com.linkedin.datahub.graphql.GraphQLEngine; import com.linkedin.datahub.graphql.analytics.service.AnalyticsService; -import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.gms.factory.auth.DataHubTokenServiceFactory; import com.linkedin.gms.factory.common.GitVersionFactory; import com.linkedin.gms.factory.common.IndexConventionFactory; diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java index 15a90afa36..b87e5d7006 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java @@ -18,7 +18,7 @@ import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.RollbackRunResult; import com.linkedin.metadata.entity.ValidationException; import com.linkedin.metadata.models.EntitySpec; -import com.linkedin.metadata.resources.entity.AspectUtils; +import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.SystemMetadata; diff --git a/metadata-service/restli-client/build.gradle b/metadata-service/restli-client/build.gradle index 1f079ae622..8f25cf7294 100644 --- a/metadata-service/restli-client/build.gradle +++ b/metadata-service/restli-client/build.gradle @@ -3,11 +3,9 @@ apply plugin: 'pegasus' dependencies { compile project(':metadata-service:restli-api') compile project(':metadata-service:auth-api') - compile project(':metadata-service:restli-servlet-impl') compile project(path: ':metadata-service:restli-api', configuration: 'restClient') compile project(':metadata-events:mxe-schemas') - - dataModel project(':li-utils') + compile project(':metadata-utils') compile externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index 6c7d681cf7..ef03ba6d34 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -2,6 +2,7 @@ package com.linkedin.entity.client; import com.datahub.authentication.Authentication; import com.datahub.util.RecordUtils; +import com.google.common.collect.ImmutableList; import com.linkedin.common.VersionedUrn; import com.linkedin.common.client.BaseClient; import com.linkedin.common.urn.Urn; @@ -48,6 +49,11 @@ import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.ListResult; import com.linkedin.metadata.query.ListUrnsResult; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.search.LineageSearchResult; @@ -66,6 +72,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -76,8 +83,6 @@ import javax.mail.MethodNotSupportedException; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.search.utils.QueryUtils.newFilter; - @Slf4j public class RestliEntityClient extends BaseClient implements EntityClient { @@ -681,4 +686,25 @@ public class RestliEntityClient extends BaseClient implements EntityClient { final RunsDoRollbackRequestBuilder requestBuilder = RUNS_REQUEST_BUILDERS.actionRollback().runIdParam(runId).dryRunParam(false); sendClientRequest(requestBuilder, authentication); } + + // TODO: Refactor QueryUtils inside of metadata-io to extract these methods into a single shared library location. + // Creates new Filter from a map of Criteria by removing null-valued Criteria and using EQUAL condition (default). + @Nonnull + public static Filter newFilter(@Nullable Map params) { + if (params == null) { + return new Filter().setOr(new ConjunctiveCriterionArray()); + } + CriterionArray criteria = params.entrySet() + .stream() + .filter(e -> Objects.nonNull(e.getValue())) + .map(e -> newCriterion(e.getKey(), e.getValue(), Condition.EQUAL)) + .collect(Collectors.toCollection(CriterionArray::new)); + return new Filter().setOr( + new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion().setAnd(criteria)))); + } + + @Nonnull + public static Criterion newCriterion(@Nonnull String field, @Nonnull String value, @Nonnull Condition condition) { + return new Criterion().setField(field).setValue(value).setCondition(condition); + } } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 71b1c6f7f7..2eb0035220 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -8,6 +8,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.EnvelopedAspectArray; import com.linkedin.metadata.aspect.VersionedAspect; +import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ValidationException; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 642fca4377..6356ef9ab0 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -5,7 +5,6 @@ import com.datahub.authentication.Authentication; import com.datahub.authentication.AuthenticationContext; import com.google.common.collect.ImmutableList; import com.linkedin.common.AuditStamp; -import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.LongMap; import com.linkedin.data.template.StringArray; @@ -35,7 +34,6 @@ import com.linkedin.metadata.run.RollbackResponse; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.LineageSearchResult; import com.linkedin.metadata.search.LineageSearchService; -import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.utils.ESUtils; @@ -77,6 +75,8 @@ import org.apache.maven.artifact.versioning.ComparableVersion; import static com.linkedin.metadata.entity.ValidationUtils.*; import static com.linkedin.metadata.resources.entity.ResourceUtils.*; import static com.linkedin.metadata.resources.restli.RestliConstants.*; +import static com.linkedin.metadata.search.utils.SearchUtils.*; +import static com.linkedin.metadata.shared.ValidationUtils.*; import static com.linkedin.metadata.utils.PegasusUtils.*; @@ -144,19 +144,6 @@ public class EntityResource extends CollectionResourceTaskTemplate entityService.exists(searchEntity.getEntity())) - .collect(Collectors.toCollection(SearchEntityArray::new)); - validatedSearchResult.setEntities(validatedEntities); - - return validatedSearchResult; - } - - public static BrowseResult validateBrowseResult(final BrowseResult browseResult, - @Nonnull final EntityService entityService) { - if (browseResult == null) { - return null; - } - Objects.requireNonNull(entityService, "entityService must not be null"); - - BrowseResult validatedBrowseResult = new BrowseResult().setGroups(browseResult.getGroups()) - .setMetadata(browseResult.getMetadata()) - .setFrom(browseResult.getFrom()) - .setPageSize(browseResult.getPageSize()) - .setNumGroups(browseResult.getNumGroups()) - .setNumEntities(browseResult.getNumEntities()) - .setNumElements(browseResult.getNumElements()); - - BrowseResultEntityArray validatedEntities = browseResult.getEntities() - .stream() - .filter(browseResultEntity -> entityService.exists(browseResultEntity.getUrn())) - .collect(Collectors.toCollection(BrowseResultEntityArray::new)); - validatedBrowseResult.setEntities(validatedEntities); - - - return validatedBrowseResult; - } - - public static ListResult validateListResult(final ListResult listResult, @Nonnull final EntityService entityService) { - if (listResult == null) { - return null; - } - Objects.requireNonNull(entityService, "entityService must not be null"); - - ListResult validatedListResult = new ListResult().setStart(listResult.getStart()) - .setCount(listResult.getCount()) - .setTotal(listResult.getTotal()); - - UrnArray validatedEntities = - listResult.getEntities().stream().filter(entityService::exists).collect(Collectors.toCollection(UrnArray::new)); - validatedListResult.setEntities(validatedEntities); - - return validatedListResult; - } - - public static LineageSearchResult validateLineageSearchResult(final LineageSearchResult lineageSearchResult, - @Nonnull final EntityService entityService) { - if (lineageSearchResult == null) { - return null; - } - Objects.requireNonNull(entityService, "entityService must not be null"); - - LineageSearchResult validatedLineageSearchResult = - new LineageSearchResult().setMetadata(lineageSearchResult.getMetadata()) - .setFrom(lineageSearchResult.getFrom()) - .setPageSize(lineageSearchResult.getPageSize()) - .setNumEntities(lineageSearchResult.getNumEntities()); - - LineageSearchEntityArray validatedEntities = lineageSearchResult.getEntities() - .stream() - .filter(entity -> entityService.exists(entity.getEntity())) - .collect(Collectors.toCollection(LineageSearchEntityArray::new)); - validatedLineageSearchResult.setEntities(validatedEntities); - - return validatedLineageSearchResult; - } }