From 1b15dd484fbe42e538a00bc6e8a0fde1eeaae051 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:53:04 -0500 Subject: [PATCH] lint(test): remove unused imports, other test fixes (#7659) --- .../metadata/entity/EntityService.java | 2 +- .../metadata/entity/RetentionService.java | 1 + .../metadata/AspectGenerationUtils.java | 2 - .../metadata/entity/EntityServiceTest.java | 55 ++++++++++--------- .../kafka/hook/UpdateIndicesHook.java | 4 +- 5 files changed, 33 insertions(+), 31 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index 66db564d17..887c1533d3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -1084,7 +1084,7 @@ public class EntityService { logger.accept(String.format( "Reading rows %s through %s from the aspects table completed.", args.start, args.start + args.batchSize)); - for (EbeanAspectV2 aspect : rows.getList()) { + for (EbeanAspectV2 aspect : rows != null ? rows.getList() : List.of()) { // 1. Extract an Entity type from the entity Urn result.timeGetRowMs = System.currentTimeMillis() - startTime; startTime = System.currentTimeMillis(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/RetentionService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/RetentionService.java index 50c3ca986b..e4cdb5f531 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/RetentionService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/RetentionService.java @@ -107,6 +107,7 @@ public abstract class RetentionService { GenericAspect retentionAspect = GenericRecordUtils.serializeAspect(retentionConfig); aspectProposal.setAspect(retentionAspect); aspectProposal.setAspectName(Constants.DATAHUB_RETENTION_ASPECT); + aspectProposal.setChangeType(ChangeType.UPSERT); return getEntityService().ingestProposal(aspectProposal, auditStamp, false).isDidUpdate(); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java index a7f4b1a34a..58b5d9ab2d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java @@ -6,7 +6,6 @@ import com.linkedin.common.ChangeAuditStamps; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.dataset.Upstream; import com.linkedin.dataset.UpstreamArray; import com.linkedin.dataset.UpstreamLineage; import com.linkedin.identity.CorpUserInfo; @@ -14,7 +13,6 @@ import com.linkedin.metadata.key.CorpUserKey; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.SystemMetadata; -import java.util.Collections; import javax.annotation.Nonnull; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index 0ea62a487e..e088c78933 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -1153,38 +1153,41 @@ abstract public class EntityServiceTest> pairToIngest = new ArrayList<>(); + if (this instanceof EbeanEntityServiceTest) { + String urnStr = "urn:li:dataset:(urn:li:dataPlatform:looker,sample_dataset_unique,PROD)"; + Urn entityUrn = UrnUtils.getUrn(urnStr); + List> pairToIngest = new ArrayList<>(); - final UpstreamLineage upstreamLineage = AspectGenerationUtils.createUpstreamLineage(); - String aspectName1 = AspectGenerationUtils.getAspectName(upstreamLineage); - pairToIngest.add(getAspectRecordPair(upstreamLineage, UpstreamLineage.class)); + final UpstreamLineage upstreamLineage = AspectGenerationUtils.createUpstreamLineage(); + String aspectName1 = AspectGenerationUtils.getAspectName(upstreamLineage); + pairToIngest.add(getAspectRecordPair(upstreamLineage, UpstreamLineage.class)); - SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); + _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); - clearInvocations(_mockProducer); + clearInvocations(_mockProducer); - RestoreIndicesArgs args = new RestoreIndicesArgs(); - args.setAspectName(UPSTREAM_LINEAGE_ASPECT_NAME); - args.setBatchSize(1); - args.setStart(0); - args.setBatchDelayMs(1L); - args.setNumThreads(1); - args.setUrn(urnStr); - _entityService.restoreIndices(args, obj -> { }); + RestoreIndicesArgs args = new RestoreIndicesArgs(); + args.setAspectName(UPSTREAM_LINEAGE_ASPECT_NAME); + args.setBatchSize(1); + args.setStart(0); + args.setBatchDelayMs(1L); + args.setNumThreads(1); + args.setUrn(urnStr); + _entityService.restoreIndices(args, obj -> { + }); - ArgumentCaptor mclCaptor = ArgumentCaptor.forClass(MetadataChangeLog.class); - verify(_mockProducer, times(1)).produceMetadataChangeLog( - Mockito.eq(entityUrn), Mockito.any(), mclCaptor.capture()); - MetadataChangeLog mcl = mclCaptor.getValue(); - assertEquals(mcl.getEntityType(), "dataset"); - assertNull(mcl.getPreviousAspectValue()); - assertNull(mcl.getPreviousSystemMetadata()); - assertEquals(mcl.getChangeType(), ChangeType.RESTATE); - assertEquals(mcl.getSystemMetadata().getProperties().get(FORCE_INDEXING_KEY), "true"); + ArgumentCaptor mclCaptor = ArgumentCaptor.forClass(MetadataChangeLog.class); + verify(_mockProducer, times(1)).produceMetadataChangeLog( + Mockito.eq(entityUrn), Mockito.any(), mclCaptor.capture()); + MetadataChangeLog mcl = mclCaptor.getValue(); + assertEquals(mcl.getEntityType(), "dataset"); + assertNull(mcl.getPreviousAspectValue()); + assertNull(mcl.getPreviousSystemMetadata()); + assertEquals(mcl.getChangeType(), ChangeType.RESTATE); + assertEquals(mcl.getSystemMetadata().getProperties().get(FORCE_INDEXING_KEY), "true"); + } } @Nonnull diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java index b088a39fac..171bee9f75 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java @@ -22,7 +22,7 @@ import com.linkedin.gms.factory.timeseries.TimeseriesAspectServiceFactory; import com.linkedin.metadata.Constants; import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.GraphService; -import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; +import com.linkedin.metadata.graph.dgraph.DgraphGraphService; import com.linkedin.metadata.key.SchemaFieldKey; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; @@ -179,7 +179,7 @@ public class UpdateIndicesHook implements MetadataChangeLogHook { // Step 2. For all aspects, attempt to update Graph SystemMetadata systemMetadata = event.getSystemMetadata(); - if (_graphDiffMode && !(_graphService instanceof DGraphGraphService) + if (_graphDiffMode && !(_graphService instanceof DgraphGraphService) && (systemMetadata == null || systemMetadata.getProperties() == null || !Boolean.parseBoolean(systemMetadata.getProperties().get(FORCE_INDEXING_KEY)))) { updateGraphServiceDiff(urn, aspectSpec, previousAspect, aspect, event);