yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-18 12:27:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

lint(test): remove unused imports, other test fixes (#7659)

Browse Source
This commit is contained in:
david-leifker 2023-03-21 16:53:04 -05:00 committed by GitHub
parent 8963e53bbd
commit 1b15dd484f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 33 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java
View File

@ -1084,7 +1084,7 @@ public class EntityService {
logger.accept(String.format( logger.accept(String.format(
"Reading rows %s through %s from the aspects table completed.", args.start, args.start + args.batchSize)); "Reading rows %s through %s from the aspects table completed.", args.start, args.start + args.batchSize));
for (EbeanAspectV2 aspect : rows.getList()) { for (EbeanAspectV2 aspect : rows != null ? rows.getList() : List.<EbeanAspectV2>of()) {
// 1. Extract an Entity type from the entity Urn // 1. Extract an Entity type from the entity Urn
result.timeGetRowMs = System.currentTimeMillis() - startTime; result.timeGetRowMs = System.currentTimeMillis() - startTime;
startTime = System.currentTimeMillis(); startTime = System.currentTimeMillis();

1
metadata-io/src/main/java/com/linkedin/metadata/entity/RetentionService.java
View File

@ -107,6 +107,7 @@ public abstract class RetentionService {
GenericAspect retentionAspect = GenericRecordUtils.serializeAspect(retentionConfig); GenericAspect retentionAspect = GenericRecordUtils.serializeAspect(retentionConfig);
aspectProposal.setAspect(retentionAspect); aspectProposal.setAspect(retentionAspect);
aspectProposal.setAspectName(Constants.DATAHUB_RETENTION_ASPECT); aspectProposal.setAspectName(Constants.DATAHUB_RETENTION_ASPECT);
aspectProposal.setChangeType(ChangeType.UPSERT);
return getEntityService().ingestProposal(aspectProposal, auditStamp, false).isDidUpdate(); return getEntityService().ingestProposal(aspectProposal, auditStamp, false).isDidUpdate();
} }

2
metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java
View File

@ -6,7 +6,6 @@ import com.linkedin.common.ChangeAuditStamps;
import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils; import com.linkedin.common.urn.UrnUtils;
import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.RecordTemplate;
import com.linkedin.dataset.Upstream;
import com.linkedin.dataset.UpstreamArray; import com.linkedin.dataset.UpstreamArray;
import com.linkedin.dataset.UpstreamLineage; import com.linkedin.dataset.UpstreamLineage;
import com.linkedin.identity.CorpUserInfo; import com.linkedin.identity.CorpUserInfo;
@ -14,7 +13,6 @@ import com.linkedin.metadata.key.CorpUserKey;
import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.EntityKeyUtils;
import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.metadata.utils.PegasusUtils;
import com.linkedin.mxe.SystemMetadata; import com.linkedin.mxe.SystemMetadata;
import java.util.Collections;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;

55
metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java
View File

@ -1153,38 +1153,41 @@ abstract public class EntityServiceTest<T_AD extends AspectDao, T_RS extends Ret
@Test @Test
public void testRestoreIndices() throws Exception { public void testRestoreIndices() throws Exception {
String urnStr = "urn:li:dataset:(urn:li:dataPlatform:looker,sample_dataset_unique,PROD)"; if (this instanceof EbeanEntityServiceTest) {
Urn entityUrn = UrnUtils.getUrn(urnStr); String urnStr = "urn:li:dataset:(urn:li:dataPlatform:looker,sample_dataset_unique,PROD)";
List<Pair<String, RecordTemplate>> pairToIngest = new ArrayList<>(); Urn entityUrn = UrnUtils.getUrn(urnStr);
List<Pair<String, RecordTemplate>> pairToIngest = new ArrayList<>();
final UpstreamLineage upstreamLineage = AspectGenerationUtils.createUpstreamLineage(); final UpstreamLineage upstreamLineage = AspectGenerationUtils.createUpstreamLineage();
String aspectName1 = AspectGenerationUtils.getAspectName(upstreamLineage); String aspectName1 = AspectGenerationUtils.getAspectName(upstreamLineage);
pairToIngest.add(getAspectRecordPair(upstreamLineage, UpstreamLineage.class)); pairToIngest.add(getAspectRecordPair(upstreamLineage, UpstreamLineage.class));
SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata();
_entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1);
clearInvocations(_mockProducer); clearInvocations(_mockProducer);
RestoreIndicesArgs args = new RestoreIndicesArgs(); RestoreIndicesArgs args = new RestoreIndicesArgs();
args.setAspectName(UPSTREAM_LINEAGE_ASPECT_NAME); args.setAspectName(UPSTREAM_LINEAGE_ASPECT_NAME);
args.setBatchSize(1); args.setBatchSize(1);
args.setStart(0); args.setStart(0);
args.setBatchDelayMs(1L); args.setBatchDelayMs(1L);
args.setNumThreads(1); args.setNumThreads(1);
args.setUrn(urnStr); args.setUrn(urnStr);
_entityService.restoreIndices(args, obj -> { }); _entityService.restoreIndices(args, obj -> {
});
ArgumentCaptor<MetadataChangeLog> mclCaptor = ArgumentCaptor.forClass(MetadataChangeLog.class); ArgumentCaptor<MetadataChangeLog> mclCaptor = ArgumentCaptor.forClass(MetadataChangeLog.class);
verify(_mockProducer, times(1)).produceMetadataChangeLog( verify(_mockProducer, times(1)).produceMetadataChangeLog(
Mockito.eq(entityUrn), Mockito.any(), mclCaptor.capture()); Mockito.eq(entityUrn), Mockito.any(), mclCaptor.capture());
MetadataChangeLog mcl = mclCaptor.getValue(); MetadataChangeLog mcl = mclCaptor.getValue();
assertEquals(mcl.getEntityType(), "dataset"); assertEquals(mcl.getEntityType(), "dataset");
assertNull(mcl.getPreviousAspectValue()); assertNull(mcl.getPreviousAspectValue());
assertNull(mcl.getPreviousSystemMetadata()); assertNull(mcl.getPreviousSystemMetadata());
assertEquals(mcl.getChangeType(), ChangeType.RESTATE); assertEquals(mcl.getChangeType(), ChangeType.RESTATE);
assertEquals(mcl.getSystemMetadata().getProperties().get(FORCE_INDEXING_KEY), "true"); assertEquals(mcl.getSystemMetadata().getProperties().get(FORCE_INDEXING_KEY), "true");
}
} }
@Nonnull @Nonnull

4
metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java
View File

@ -22,7 +22,7 @@ import com.linkedin.gms.factory.timeseries.TimeseriesAspectServiceFactory;
import com.linkedin.metadata.Constants; import com.linkedin.metadata.Constants;
import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.Edge;
import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.GraphService;
import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; import com.linkedin.metadata.graph.dgraph.DgraphGraphService;
import com.linkedin.metadata.key.SchemaFieldKey; import com.linkedin.metadata.key.SchemaFieldKey;
import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.AspectSpec;
import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.EntitySpec;
@ -179,7 +179,7 @@ public class UpdateIndicesHook implements MetadataChangeLogHook {
// Step 2. For all aspects, attempt to update Graph // Step 2. For all aspects, attempt to update Graph
SystemMetadata systemMetadata = event.getSystemMetadata(); SystemMetadata systemMetadata = event.getSystemMetadata();
if (_graphDiffMode && !(_graphService instanceof DGraphGraphService) if (_graphDiffMode && !(_graphService instanceof DgraphGraphService)
&& (systemMetadata == null || systemMetadata.getProperties() == null && (systemMetadata == null || systemMetadata.getProperties() == null
|| !Boolean.parseBoolean(systemMetadata.getProperties().get(FORCE_INDEXING_KEY)))) { || !Boolean.parseBoolean(systemMetadata.getProperties().get(FORCE_INDEXING_KEY)))) {
updateGraphServiceDiff(urn, aspectSpec, previousAspect, aspect, event); updateGraphServiceDiff(urn, aspectSpec, previousAspect, aspect, event);