diff --git a/metadata-service/openapi-servlet/build.gradle b/metadata-service/openapi-servlet/build.gradle index 139ab038d7..7cd022f972 100644 --- a/metadata-service/openapi-servlet/build.gradle +++ b/metadata-service/openapi-servlet/build.gradle @@ -24,6 +24,7 @@ dependencies { annotationProcessor externalDependency.lombok testImplementation externalDependency.springBootTest + testImplementation project(':mock-entity-registry') testCompile externalDependency.springBoot testCompile externalDependency.testContainers testCompile externalDependency.springKafka diff --git a/metadata-service/restli-servlet-impl/build.gradle b/metadata-service/restli-servlet-impl/build.gradle index e1768c18d0..1028f7c3dc 100644 --- a/metadata-service/restli-servlet-impl/build.gradle +++ b/metadata-service/restli-servlet-impl/build.gradle @@ -51,6 +51,7 @@ dependencies { annotationProcessor externalDependency.lombok testCompile project(':test-models') + testImplementation project(':mock-entity-registry') testCompile externalDependency.mockito testCompile externalDependency.testng diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 5a10d5e941..0004c82b43 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -5,6 +5,7 @@ import com.datahub.authentication.Authentication; import com.datahub.authentication.AuthenticationContext; import com.datahub.authorization.ResourceSpec; import com.datahub.plugins.auth.authorization.Authorizer; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.linkedin.aspect.GetTimeseriesAspectValuesResponse; import com.linkedin.metadata.resources.operations.Utils; @@ -78,6 +79,11 @@ public class AspectResource extends CollectionResourceTaskTemplate _entityService.ingestProposal(proposal, auditStamp, asyncBool)); + if (!asyncBool) { + AspectUtils.getAdditionalChanges(metadataChangeProposal, _entityService) + .forEach(proposal -> _entityService.ingestProposal(proposal, auditStamp, asyncBool)); + } if (!result.isQueued()) { tryIndexRunId(responseUrn, metadataChangeProposal.getSystemMetadata(), _entitySearchService); diff --git a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java new file mode 100644 index 0000000000..0d3642b917 --- /dev/null +++ b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java @@ -0,0 +1,85 @@ +package com.linkedin.metadata.resources.entity; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.plugins.auth.authorization.Authorizer; +import com.linkedin.common.FabricType; +import com.linkedin.common.urn.DataPlatformUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.dataset.DatasetProperties; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.config.PreProcessHooks; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.event.EventProducer; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.service.UpdateIndicesService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.MetadataChangeProposal; +import java.net.URISyntaxException; +import mock.MockEntityRegistry; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; + + +public class AspectResourceTest { + private AspectResource _aspectResource; + private EntityService _entityService; + private AspectDao _aspectDao; + private EventProducer _producer; + private EntityRegistry _entityRegistry; + private UpdateIndicesService _updateIndicesService; + private PreProcessHooks _preProcessHooks; + private Authorizer _authorizer; + + @BeforeTest + public void setup() { + _aspectResource = new AspectResource(); + _aspectDao = mock(AspectDao.class); + _producer = mock(EventProducer.class); + _entityRegistry = new MockEntityRegistry(); + _updateIndicesService = mock(UpdateIndicesService.class); + _preProcessHooks = mock(PreProcessHooks.class); + _entityService = new EntityService(_aspectDao, _producer, _entityRegistry, false, _updateIndicesService, _preProcessHooks); + _authorizer = mock(Authorizer.class); + _aspectResource.setAuthorizer(_authorizer); + _aspectResource.setEntityService(_entityService); + } + + @Test + public void testAsyncDefaultAspects() throws URISyntaxException { + MetadataChangeProposal mcp = new MetadataChangeProposal(); + mcp.setEntityType(DATASET_ENTITY_NAME); + Urn urn = new DatasetUrn(new DataPlatformUrn("platform"), "name", FabricType.PROD); + mcp.setEntityUrn(urn); + DatasetProperties properties = new DatasetProperties().setName("name"); + mcp.setAspect(GenericRecordUtils.serializeAspect(properties)); + mcp.setAspectName(DATASET_PROPERTIES_ASPECT_NAME); + mcp.setChangeType(ChangeType.UPSERT); + + Authentication mockAuthentication = mock(Authentication.class); + AuthenticationContext.setAuthentication(mockAuthentication); + Actor actor = new Actor(ActorType.USER, "user"); + when(mockAuthentication.getActor()).thenReturn(actor); + _aspectResource.ingestProposal(mcp, "true"); + verify(_producer, times(1)).produceMetadataChangeProposal(urn, mcp); + verifyNoMoreInteractions(_producer); + verifyNoMoreInteractions(_aspectDao); + + reset(_producer, _aspectDao); + + when(_aspectDao.runInTransactionWithRetry(any(), anyInt())) + .thenReturn(new EntityService.UpdateAspectResult(urn, null, properties, null, null, null, null, 0)); + _aspectResource.ingestProposal(mcp, "false"); + verify(_producer, times(5)).produceMetadataChangeLog(eq(urn), any(AspectSpec.class), any(MetadataChangeLog.class)); + verifyNoMoreInteractions(_producer); + } +} diff --git a/metadata-service/schema-registry-servlet/build.gradle b/metadata-service/schema-registry-servlet/build.gradle index 5e2bc791ac..ec62203ddf 100644 --- a/metadata-service/schema-registry-servlet/build.gradle +++ b/metadata-service/schema-registry-servlet/build.gradle @@ -22,6 +22,7 @@ dependencies { annotationProcessor externalDependency.lombok testImplementation externalDependency.springBootTest + testImplementation project(':mock-entity-registry') testCompile externalDependency.springBoot testCompile externalDependency.testContainers testCompile externalDependency.testContainersKafka diff --git a/metadata-service/schema-registry-servlet/src/test/java/mock/MockAspectSpec.java b/metadata-service/schema-registry-servlet/src/test/java/mock/MockAspectSpec.java deleted file mode 100644 index 594bc583ee..0000000000 --- a/metadata-service/schema-registry-servlet/src/test/java/mock/MockAspectSpec.java +++ /dev/null @@ -1,27 +0,0 @@ -package mock; - -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.models.RelationshipFieldSpec; -import com.linkedin.metadata.models.SearchScoreFieldSpec; -import com.linkedin.metadata.models.SearchableFieldSpec; -import com.linkedin.metadata.models.TimeseriesFieldCollectionSpec; -import com.linkedin.metadata.models.TimeseriesFieldSpec; -import com.linkedin.metadata.models.annotation.AspectAnnotation; -import java.util.List; -import javax.annotation.Nonnull; - - -public class MockAspectSpec extends AspectSpec { - public MockAspectSpec(@Nonnull AspectAnnotation aspectAnnotation, - @Nonnull List searchableFieldSpecs, - @Nonnull List searchScoreFieldSpecs, - @Nonnull List relationshipFieldSpecs, - @Nonnull List timeseriesFieldSpecs, - @Nonnull List timeseriesFieldCollectionSpecs, RecordDataSchema schema, - Class aspectClass) { - super(aspectAnnotation, searchableFieldSpecs, searchScoreFieldSpecs, relationshipFieldSpecs, timeseriesFieldSpecs, - timeseriesFieldCollectionSpecs, schema, aspectClass); - } -} diff --git a/metadata-service/schema-registry-servlet/src/test/java/mock/MockEntitySpec.java b/metadata-service/schema-registry-servlet/src/test/java/mock/MockEntitySpec.java deleted file mode 100644 index a9845d814b..0000000000 --- a/metadata-service/schema-registry-servlet/src/test/java/mock/MockEntitySpec.java +++ /dev/null @@ -1,118 +0,0 @@ -package mock; - -import com.linkedin.common.GlossaryTerms; -import com.linkedin.common.SubTypes; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.TyperefDataSchema; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.dataset.DatasetProfile; -import com.linkedin.dataset.ViewProperties; -import com.linkedin.metadata.key.CorpUserKey; -import com.linkedin.metadata.key.DataPlatformKey; -import com.linkedin.metadata.key.DatasetKey; -import com.linkedin.metadata.key.GlossaryTermKey; -import com.linkedin.metadata.key.TagKey; -import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.models.EntitySpec; -import com.linkedin.metadata.models.annotation.AspectAnnotation; -import com.linkedin.metadata.models.annotation.EntityAnnotation; -import com.linkedin.schema.SchemaMetadata; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.linkedin.metadata.Constants.*; - - -public class MockEntitySpec implements EntitySpec { - - private String _name; - - public MockEntitySpec(String name) { - _name = name; - } - - @Override - public String getName() { - return _name; - } - - @Override - public EntityAnnotation getEntityAnnotation() { - return null; - } - - @Override - public String getKeyAspectName() { - return null; - } - - @Override - public AspectSpec getKeyAspectSpec() { - if (DATASET_ENTITY_NAME.equals(_name)) { - DatasetKey datasetKey = new DatasetKey(); - return createAspectSpec(datasetKey, DATASET_KEY_ASPECT_NAME); - } else if (DATA_PLATFORM_ENTITY_NAME.equals(_name)) { - DataPlatformKey dataPlatformKey = new DataPlatformKey(); - return createAspectSpec(dataPlatformKey, DATA_PLATFORM_KEY_ASPECT_NAME); - } else if (TAG_ENTITY_NAME.equals(_name)) { - TagKey tagKey = new TagKey(); - return createAspectSpec(tagKey, TAG_KEY_ASPECT_NAME); - } else if (GLOSSARY_TERM_ENTITY_NAME.equals(_name)) { - GlossaryTermKey glossaryTermKey = new GlossaryTermKey(); - return createAspectSpec(glossaryTermKey, GLOSSARY_TERM_KEY_ASPECT_NAME); - } else if (CORP_USER_ENTITY_NAME.equals(_name)) { - CorpUserKey corpUserKey = new CorpUserKey(); - return createAspectSpec(corpUserKey, CORP_USER_KEY_ASPECT_NAME); - } - return null; - } - - public AspectSpec createAspectSpec(T type, String name) { - return new MockAspectSpec(new AspectAnnotation(name, false, false, null), - Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), - Collections.emptyList(), type.schema(), (Class) type.getClass().asSubclass(RecordTemplate.class)); - } - - @Override - public List getAspectSpecs() { - return Collections.emptyList(); - } - - @Override - public Map getAspectSpecMap() { - return Collections.emptyMap(); - } - - @Override - public Boolean hasAspect(String name) { - return false; - } - - private static final Map ASPECT_TYPE_MAP; - - static { - ASPECT_TYPE_MAP = new HashMap<>(); - ASPECT_TYPE_MAP.put(DATASET_KEY_ASPECT_NAME, new DatasetKey()); - ASPECT_TYPE_MAP.put(VIEW_PROPERTIES_ASPECT_NAME, new ViewProperties()); - ASPECT_TYPE_MAP.put(SCHEMA_METADATA_ASPECT_NAME, new SchemaMetadata()); - ASPECT_TYPE_MAP.put(SUB_TYPES_ASPECT_NAME, new SubTypes()); - ASPECT_TYPE_MAP.put("datasetProfile", new DatasetProfile()); - ASPECT_TYPE_MAP.put(GLOSSARY_TERMS_ASPECT_NAME, new GlossaryTerms()); - } - @Override - public AspectSpec getAspectSpec(String name) { - return createAspectSpec(ASPECT_TYPE_MAP.get(name), name); - } - - @Override - public RecordDataSchema getSnapshotSchema() { - return null; - } - - @Override - public TyperefDataSchema getAspectTyperefSchema() { - return null; - } -} diff --git a/mock-entity-registry/build.gradle b/mock-entity-registry/build.gradle new file mode 100644 index 0000000000..12d7e58eee --- /dev/null +++ b/mock-entity-registry/build.gradle @@ -0,0 +1,5 @@ +apply plugin: 'java' + +dependencies { + implementation project(':entity-registry') +} \ No newline at end of file diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockAspectSpec.java b/mock-entity-registry/src/main/java/mock/MockAspectSpec.java similarity index 100% rename from metadata-service/openapi-servlet/src/test/java/mock/MockAspectSpec.java rename to mock-entity-registry/src/main/java/mock/MockAspectSpec.java diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntityRegistry.java b/mock-entity-registry/src/main/java/mock/MockEntityRegistry.java similarity index 100% rename from metadata-service/openapi-servlet/src/test/java/mock/MockEntityRegistry.java rename to mock-entity-registry/src/main/java/mock/MockEntityRegistry.java diff --git a/metadata-service/openapi-servlet/src/test/java/mock/MockEntitySpec.java b/mock-entity-registry/src/main/java/mock/MockEntitySpec.java similarity index 79% rename from metadata-service/openapi-servlet/src/test/java/mock/MockEntitySpec.java rename to mock-entity-registry/src/main/java/mock/MockEntitySpec.java index a9845d814b..097e084550 100644 --- a/metadata-service/openapi-servlet/src/test/java/mock/MockEntitySpec.java +++ b/mock-entity-registry/src/main/java/mock/MockEntitySpec.java @@ -1,11 +1,15 @@ package mock; +import com.linkedin.common.BrowsePaths; +import com.linkedin.common.BrowsePathsV2; +import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.GlossaryTerms; import com.linkedin.common.SubTypes; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.schema.TyperefDataSchema; import com.linkedin.data.template.RecordTemplate; import com.linkedin.dataset.DatasetProfile; +import com.linkedin.dataset.DatasetProperties; import com.linkedin.dataset.ViewProperties; import com.linkedin.metadata.key.CorpUserKey; import com.linkedin.metadata.key.DataPlatformKey; @@ -28,9 +32,16 @@ import static com.linkedin.metadata.Constants.*; public class MockEntitySpec implements EntitySpec { private String _name; + private Map _aspectTypeMap; public MockEntitySpec(String name) { _name = name; + _aspectTypeMap = new HashMap<>(); + if (DATASET_ENTITY_NAME.equals(name)) { + _aspectTypeMap.put(BROWSE_PATHS_ASPECT_NAME, getAspectSpec(BROWSE_PATHS_ASPECT_NAME)); + _aspectTypeMap.put(BROWSE_PATHS_V2_ASPECT_NAME, getAspectSpec(BROWSE_PATHS_V2_ASPECT_NAME)); + _aspectTypeMap.put(DATA_PLATFORM_INSTANCE_ASPECT_NAME, getAspectSpec(DATA_PLATFORM_INSTANCE_ASPECT_NAME)); + } } @Override @@ -82,7 +93,7 @@ public class MockEntitySpec implements EntitySpec { @Override public Map getAspectSpecMap() { - return Collections.emptyMap(); + return _aspectTypeMap; } @Override @@ -100,6 +111,10 @@ public class MockEntitySpec implements EntitySpec { ASPECT_TYPE_MAP.put(SUB_TYPES_ASPECT_NAME, new SubTypes()); ASPECT_TYPE_MAP.put("datasetProfile", new DatasetProfile()); ASPECT_TYPE_MAP.put(GLOSSARY_TERMS_ASPECT_NAME, new GlossaryTerms()); + ASPECT_TYPE_MAP.put(DATASET_PROPERTIES_ASPECT_NAME, new DatasetProperties()); + ASPECT_TYPE_MAP.put(BROWSE_PATHS_ASPECT_NAME, new BrowsePaths()); + ASPECT_TYPE_MAP.put(BROWSE_PATHS_V2_ASPECT_NAME, new BrowsePathsV2()); + ASPECT_TYPE_MAP.put(DATA_PLATFORM_INSTANCE_ASPECT_NAME, new DataPlatformInstance()); } @Override public AspectSpec getAspectSpec(String name) { diff --git a/settings.gradle b/settings.gradle index 84ee306ff5..e4d1702829 100644 --- a/settings.gradle +++ b/settings.gradle @@ -59,3 +59,5 @@ include 'metadata-auth:auth-api' include 'metadata-service:schema-registry-api' include 'metadata-service:schema-registry-servlet' include 'metadata-integration:java:examples' +include 'mock-entity-registry' +