From f36a597b171abcf7b624bdcd5a77cd94fa460ca0 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:50:41 -0500 Subject: [PATCH] fix(spring): refactor spring configuration (#10290) --- build.gradle | 8 +- .../app/config/ConfigurationProvider.java | 5 +- .../upgrade/UpgradeCliApplication.java | 2 - .../upgrade/config/SystemUpdateConfig.java | 12 +- .../DatahubUpgradeNoSchemaRegistryTest.java | 33 +++- .../DatahubUpgradeNonBlockingTest.java | 30 +++- ...pgradeCliApplicationTestConfiguration.java | 11 +- docker/build.gradle | 14 ++ docs/authentication/concepts.md | 2 +- ...oducing-metadata-service-authentication.md | 6 +- docs/deploy/aws.md | 2 +- docs/how/updating-datahub.md | 2 +- docs/plugins.md | 6 +- .../com/linkedin/metadata/EventUtils.java | 4 +- .../kafka/MaeConsumerApplication.java | 3 + .../metadata/kafka/MclConsumerConfig.java | 22 +-- metadata-jobs/mae-consumer/build.gradle | 4 +- .../kafka/hook/UpdateIndicesHook.java | 2 +- .../kafka/hook/spring/MCLSpringTest.java | 2 +- .../spring/MCLSpringTestConfiguration.java | 9 +- .../kafka/MceConsumerApplication.java | 2 +- .../metadata/restli/EbeanServerConfig.java | 2 +- metadata-jobs/mce-consumer/build.gradle | 3 +- .../kafka/MetadataChangeEventsProcessor.java | 8 +- .../MetadataChangeProposalsProcessor.java | 8 +- metadata-jobs/pe-consumer/build.gradle | 3 +- .../AuthenticationConfiguration.java | 2 +- .../AuthenticatorConfiguration.java | 2 +- .../AuthorizationConfiguration.java | 2 +- .../AuthorizerConfiguration.java | 2 +- .../filter/AuthenticationFilter.java | 12 +- .../authentication/AuthServiceController.java | 39 +++-- .../AuthServiceTestConfiguration.java | 7 + .../metadata/config/DataHubConfiguration.java | 2 +- .../config/IngestionConfiguration.java | 2 +- .../metadata/config/TestsConfiguration.java | 2 +- .../metadata/config/ViewsConfiguration.java | 2 +- .../metadata/config/VisualConfiguration.java | 2 +- .../config/search/CustomConfiguration.java | 2 +- .../telemetry/TelemetryConfiguration.java | 2 +- .../spring/YamlPropertySourceFactory.java | 2 +- .../{application.yml => application.yaml} | 0 metadata-service/factories/build.gradle | 1 + .../factory/auth/AuthorizerChainFactory.java | 3 - .../auth/DataHubAuthorizerFactory.java | 3 - .../auth/DataHubTokenServiceFactory.java | 3 - .../gms/factory/auth/GroupServiceFactory.java | 3 - .../auth/InviteTokenServiceFactory.java | 3 - .../auth/NativeUserServiceFactory.java | 3 - .../gms/factory/auth/PostServiceFactory.java | 3 - .../gms/factory/auth/RoleServiceFactory.java | 3 - .../auth/SystemAuthenticationFactory.java | 3 - .../ElasticSearchGraphServiceFactory.java | 3 - ...ticSearchSystemMetadataServiceFactory.java | 3 - .../ElasticsearchSSLContextFactory.java | 3 - .../factory/common/GraphServiceFactory.java | 3 - .../common/IndexConventionFactory.java | 3 - ...tory.java => LocalEbeanConfigFactory.java} | 15 +- .../factory/common/Neo4jDriverFactory.java | 3 - .../factory/common/ObjectMapperFactory.java | 25 +++ .../common/RestHighLevelClientFactory.java | 3 - .../factory/config/ConfigurationProvider.java | 11 +- .../services/RestrictedServiceFactory.java | 3 - .../DataProductServiceFactory.java | 3 - ...Factory.java => EbeanDatabaseFactory.java} | 17 +- .../entity/EntityAspectDaoFactory.java | 5 +- .../EntityAspectMigrationsDaoFactory.java | 5 +- .../entity/RetentionServiceFactory.java | 8 +- .../EntityClientConfigFactory.java | 3 - .../entityclient/JavaEntityClientFactory.java | 3 - .../RestliEntityClientFactory.java | 3 - .../ConfigEntityRegistryFactory.java | 3 - .../ERModelRelationshipServiceFactory.java | 3 - .../gms/factory/form/FormServiceFactory.java | 3 - .../incident/IncidentServiceFactory.java | 3 - .../ingestion/IngestionSchedulerFactory.java | 3 - .../DataHubKafkaEventProducerFactory.java | 7 +- .../kafka/DataHubKafkaProducerFactory.java | 26 +-- .../kafka/KafkaEventConsumerFactory.java | 9 - .../kafka/SimpleKafkaConsumerFactory.java | 4 +- .../common/TopicConventionFactory.java | 2 +- .../AwsGlueSchemaRegistryFactory.java | 3 - .../KafkaSchemaRegistryFactory.java | 7 +- .../SchemaRegistryServiceFactory.java | 2 +- .../lineage/LineageServiceFactory.java | 3 - .../OwnershipTypeServiceFactory.java | 3 - .../factory/query/QueryServiceFactory.java | 3 - .../RecentlyViewedCandidateSourceFactory.java | 8 - .../BaseElasticSearchComponentsFactory.java | 3 - .../CachingEntitySearchServiceFactory.java | 3 - .../ElasticSearchBulkProcessorFactory.java | 3 - .../ElasticSearchIndexBuilderFactory.java | 3 - .../search/ElasticSearchServiceFactory.java | 3 - .../search/EntityIndexBuildersFactory.java | 3 - .../search/LineageSearchServiceFactory.java | 3 - .../SearchDocumentTransformerFactory.java | 3 - .../factory/search/SearchServiceFactory.java | 3 - .../search/SettingsBuilderFactory.java | 3 - .../search/views/ViewServiceFactory.java | 3 - .../settings/SettingsServiceFactory.java | 3 - .../factory/telemetry/MixpanelApiFactory.java | 3 - .../MixpanelMessageBuilderFactory.java | 3 - .../telemetry/TrackingServiceFactory.java | 3 - .../timeline/TimelineServiceFactory.java | 3 - ...cSearchTimeseriesAspectServiceFactory.java | 3 - .../gms/factory/usage/UsageClientFactory.java | 3 - .../IngestRetentionPoliciesStepFactory.java | 3 - .../kafka/MockSystemUpdateDeserializer.java | 3 +- .../kafka/MockSystemUpdateSerializer.java | 15 +- .../restli/server/RestliHandlerServlet.java | 4 + ...ElasticSearchBulkProcessorFactoryTest.java | 2 +- ...SearchIndexBuilderFactoryDefaultsTest.java | 2 +- .../ElasticSearchIndexBuilderFactoryTest.java | 2 +- .../secret/SecretServiceFactoryTest.java | 2 +- .../openapi/util/OpenApiEntitiesUtilTest.java | 3 - .../delegates/EntityApiDelegateImplTest.java | 2 + .../openapi/config/SpringWebConfig.java | 2 - .../schema-registry-api/build.gradle | 7 +- .../openapi/schema/registry/Constants.java | 7 + .../schema-registry-servlet/build.gradle | 1 + .../registry/SchemaRegistryController.java | 9 +- .../config/SpringWebSchemaRegistryConfig.java | 2 - ...maRegistryControllerTestConfiguration.java | 6 +- metadata-service/war/build.gradle | 26 +-- .../linkedin/gms/CommonApplicationConfig.java | 47 +++++ .../gms/SpringApplicationInitializer.java | 11 ++ .../gms/WebApplicationInitializer.java | 162 ++++++++++++++++++ .../gms/servlet/AuthServletConfig.java | 10 ++ .../gms/servlet/GraphQLServletConfig.java | 24 +++ .../gms/servlet/OpenAPIServletConfig.java | 22 +++ .../gms/servlet/RestliServletConfig.java | 20 +++ .../servlet/SchemaRegistryServletConfig.java | 10 ++ .../webapp/WEB-INF/apiServlet-servlet.xml | 12 -- .../webapp/WEB-INF/authServlet-servlet.xml | 6 - .../war/src/main/webapp/WEB-INF/beans.xml | 36 ---- .../webapp/WEB-INF/openapiServlet-servlet.xml | 14 -- .../WEB-INF/schemaRegistryServlet-servlet.xml | 13 -- .../war/src/main/webapp/WEB-INF/web.xml | 156 ----------------- 138 files changed, 598 insertions(+), 626 deletions(-) rename metadata-jobs/{mae-consumer => mae-consumer-job}/src/main/java/com/linkedin/metadata/kafka/MclConsumerConfig.java (56%) rename metadata-service/configuration/src/main/resources/{application.yml => application.yaml} (100%) rename metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/{LocalEbeanServerConfigFactory.java => LocalEbeanConfigFactory.java} (87%) create mode 100644 metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ObjectMapperFactory.java rename metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/{EbeanServerFactory.java => EbeanDatabaseFactory.java} (68%) rename metadata-service/factories/src/main/java/com/linkedin/gms/factory/{ => kafka}/common/TopicConventionFactory.java (98%) create mode 100644 metadata-service/schema-registry-api/src/main/java/io/datahubproject/openapi/schema/registry/Constants.java create mode 100644 metadata-service/war/src/main/java/com/linkedin/gms/CommonApplicationConfig.java create mode 100644 metadata-service/war/src/main/java/com/linkedin/gms/SpringApplicationInitializer.java create mode 100644 metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java create mode 100644 metadata-service/war/src/main/java/com/linkedin/gms/servlet/AuthServletConfig.java create mode 100644 metadata-service/war/src/main/java/com/linkedin/gms/servlet/GraphQLServletConfig.java create mode 100644 metadata-service/war/src/main/java/com/linkedin/gms/servlet/OpenAPIServletConfig.java create mode 100644 metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java create mode 100644 metadata-service/war/src/main/java/com/linkedin/gms/servlet/SchemaRegistryServletConfig.java delete mode 100644 metadata-service/war/src/main/webapp/WEB-INF/apiServlet-servlet.xml delete mode 100644 metadata-service/war/src/main/webapp/WEB-INF/authServlet-servlet.xml delete mode 100644 metadata-service/war/src/main/webapp/WEB-INF/beans.xml delete mode 100644 metadata-service/war/src/main/webapp/WEB-INF/openapiServlet-servlet.xml delete mode 100644 metadata-service/war/src/main/webapp/WEB-INF/schemaRegistryServlet-servlet.xml delete mode 100644 metadata-service/war/src/main/webapp/WEB-INF/web.xml diff --git a/build.gradle b/build.gradle index dbbe6118ee..3245ed65a0 100644 --- a/build.gradle +++ b/build.gradle @@ -2,12 +2,14 @@ buildscript { ext.jdkVersionDefault = 17 ext.javaClassVersionDefault = 11 + def springModules = ['mae-consumer', 'mce-consumer', 'pe-consumer'] + ext.jdkVersion = { p -> // If Spring 6 is present, hard dependency on jdk17 if (p.configurations.any { it.getDependencies().any{ (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6.")) || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test")) - }}) { + }} || springModules.contains(p.name)) { return 17 } else { // otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault @@ -20,7 +22,7 @@ buildscript { if (p.configurations.any { it.getDependencies().any { (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6.")) || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test")) - }}) { + }} || springModules.contains(p.name)) { return 17 } else { // otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault @@ -158,6 +160,7 @@ project.ext.externalDependency = [ 'javatuples': 'org.javatuples:javatuples:1.2', 'javaxInject' : 'javax.inject:javax.inject:1', 'javaxValidation' : 'javax.validation:validation-api:2.0.1.Final', + 'jakartaValidation': 'jakarta.validation:jakarta.validation-api:3.1.0-M2', 'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.41', 'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1', 'jettyJaas': "org.eclipse.jetty:jetty-jaas:$jettyVersion", @@ -266,6 +269,7 @@ project.ext.externalDependency = [ 'jline':'jline:jline:1.4.1', 'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0', 'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2', + 'jakartaAnnotationApi': 'jakarta.annotation:jakarta.annotation-api:3.0.0', 'classGraph': 'io.github.classgraph:classgraph:4.8.168', ] diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java index 9f548b104e..d447b28cdc 100644 --- a/datahub-frontend/app/config/ConfigurationProvider.java +++ b/datahub-frontend/app/config/ConfigurationProvider.java @@ -6,16 +6,19 @@ import com.linkedin.metadata.config.cache.CacheConfiguration; import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import lombok.Data; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Component; /** * Minimal sharing between metadata-service and frontend Does not use the factories module to avoid * transitive dependencies. */ @EnableConfigurationProperties -@PropertySource(value = "application.yml", factory = YamlPropertySourceFactory.class) +@PropertySource(value = "classpath:/application.yaml", factory = YamlPropertySourceFactory.class) @ConfigurationProperties @Data public class ConfigurationProvider { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java index 50847da07b..e17ac6be79 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java @@ -5,7 +5,6 @@ import com.linkedin.gms.factory.auth.DataHubAuthorizerFactory; import com.linkedin.gms.factory.graphql.GraphQLEngineFactory; import com.linkedin.gms.factory.kafka.KafkaEventConsumerFactory; import com.linkedin.gms.factory.kafka.SimpleKafkaConsumerFactory; -import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; import com.linkedin.gms.factory.telemetry.ScheduledAnalyticsFactory; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -31,7 +30,6 @@ import org.springframework.context.annotation.FilterType; DataHubAuthorizerFactory.class, SimpleKafkaConsumerFactory.class, KafkaEventConsumerFactory.class, - InternalSchemaRegistryFactory.class, GraphQLEngineFactory.class }) }) diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java index ec944c2c6f..cac9b5f948 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java @@ -6,9 +6,9 @@ import com.linkedin.datahub.upgrade.system.SystemUpdate; import com.linkedin.datahub.upgrade.system.SystemUpdateBlocking; import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking; import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep; -import com.linkedin.gms.factory.common.TopicConventionFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; +import com.linkedin.gms.factory.kafka.common.TopicConventionFactory; import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; import com.linkedin.metadata.config.kafka.KafkaConfiguration; @@ -110,4 +110,14 @@ public class SystemUpdateConfig { @Qualifier("duheKafkaEventProducer") KafkaEventProducer kafkaEventProducer) { return kafkaEventProducer; } + + @Primary + @Bean(name = "schemaRegistryConfig") + @ConditionalOnProperty( + name = "kafka.schemaRegistry.type", + havingValue = InternalSchemaRegistryFactory.TYPE) + protected SchemaRegistryConfig schemaRegistryConfig( + @Qualifier("duheSchemaRegistryConfig") SchemaRegistryConfig duheSchemaRegistryConfig) { + return duheSchemaRegistryConfig; + } } diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java index 51e82e6237..8c9b72b0d8 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNoSchemaRegistryTest.java @@ -1,13 +1,22 @@ package com.linkedin.datahub.upgrade; +import static com.linkedin.metadata.EventUtils.RENAMED_MCL_AVRO_SCHEMA; +import static com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer.topicToSubjectName; import static org.mockito.Mockito.mock; -import static org.testng.AssertJUnit.assertEquals; -import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import com.linkedin.datahub.upgrade.system.SystemUpdate; +import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; +import com.linkedin.metadata.boot.kafka.MockSystemUpdateDeserializer; +import com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer; import com.linkedin.metadata.dao.producer.KafkaEventProducer; import com.linkedin.metadata.entity.EntityServiceImpl; +import com.linkedin.mxe.Topics; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import io.datahubproject.metadata.context.OperationContext; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Optional; @@ -23,8 +32,8 @@ import org.testng.annotations.Test; classes = {UpgradeCliApplication.class, UpgradeCliApplicationTestConfiguration.class}, properties = { "kafka.schemaRegistry.type=INTERNAL", - "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=test_due_topic", - "METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=test_mcl_versioned_topic" + "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=" + Topics.DATAHUB_UPGRADE_HISTORY_TOPIC_NAME, + "METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=" + Topics.METADATA_CHANGE_LOG_VERSIONED, }, args = {"-u", "SystemUpdate"}) public class DatahubUpgradeNoSchemaRegistryTest extends AbstractTestNGSpringContextTests { @@ -43,15 +52,29 @@ public class DatahubUpgradeNoSchemaRegistryTest extends AbstractTestNGSpringCont @Autowired private EntityServiceImpl entityService; + @Autowired + @Named("schemaRegistryConfig") + private SchemaRegistryConfig schemaRegistryConfig; + @Test public void testSystemUpdateInit() { assertNotNull(systemUpdate); } @Test - public void testSystemUpdateKafkaProducerOverride() { + public void testSystemUpdateKafkaProducerOverride() throws RestClientException, IOException { + assertEquals(schemaRegistryConfig.getDeserializer(), MockSystemUpdateDeserializer.class); + assertEquals(schemaRegistryConfig.getSerializer(), MockSystemUpdateSerializer.class); assertEquals(kafkaEventProducer, duheKafkaEventProducer); assertEquals(entityService.getProducer(), duheKafkaEventProducer); + + MockSystemUpdateSerializer serializer = new MockSystemUpdateSerializer(); + serializer.configure(schemaRegistryConfig.getProperties(), false); + SchemaRegistryClient registry = serializer.getSchemaRegistryClient(); + assertEquals( + registry.getId( + topicToSubjectName(Topics.METADATA_CHANGE_LOG_VERSIONED), RENAMED_MCL_AVRO_SCHEMA), + 2); } @Test diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java index 3fa254fc15..c28ff7fd29 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java @@ -4,14 +4,21 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; import static org.testng.AssertJUnit.assertNotNull; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager; import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking; import com.linkedin.datahub.upgrade.system.vianodes.ReindexDataJobViaNodesCLL; +import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; +import com.linkedin.metadata.boot.kafka.MockSystemUpdateDeserializer; +import com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer; +import com.linkedin.metadata.dao.producer.KafkaEventProducer; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; +import com.linkedin.mxe.Topics; import io.datahubproject.test.metadata.context.TestOperationContexts; import java.util.List; import javax.inject.Named; @@ -27,8 +34,8 @@ import org.testng.annotations.Test; properties = { "BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_ENABLED=true", "kafka.schemaRegistry.type=INTERNAL", - "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=test_due_topic", - "METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=test_mcl_versioned_topic" + "DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=" + Topics.DATAHUB_UPGRADE_HISTORY_TOPIC_NAME, + "METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=" + Topics.METADATA_CHANGE_LOG_VERSIONED, }, args = {"-u", "SystemUpdateNonBlocking"}) public class DatahubUpgradeNonBlockingTest extends AbstractTestNGSpringContextTests { @@ -38,9 +45,28 @@ public class DatahubUpgradeNonBlockingTest extends AbstractTestNGSpringContextTe private SystemUpdateNonBlocking systemUpdateNonBlocking; @Autowired + @Named("schemaRegistryConfig") + private SchemaRegistryConfig schemaRegistryConfig; + + @Autowired + @Named("duheKafkaEventProducer") + private KafkaEventProducer duheKafkaEventProducer; + + @Autowired + @Named("kafkaEventProducer") + private KafkaEventProducer kafkaEventProducer; + + @Autowired private EntityServiceImpl entityService; + @Test public void testSystemUpdateNonBlockingInit() { assertNotNull(systemUpdateNonBlocking); + + // Expected system update configuration and producer + assertEquals(schemaRegistryConfig.getDeserializer(), MockSystemUpdateDeserializer.class); + assertEquals(schemaRegistryConfig.getSerializer(), MockSystemUpdateSerializer.class); + assertEquals(duheKafkaEventProducer, kafkaEventProducer); + assertEquals(entityService.getProducer(), duheKafkaEventProducer); } @Test diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java index 5c2d6fff0f..81d883d8ce 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java @@ -1,18 +1,15 @@ package com.linkedin.datahub.upgrade; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.registry.SchemaRegistryService; +import com.linkedin.metadata.registry.SchemaRegistryServiceImpl; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; +import com.linkedin.mxe.TopicConventionImpl; import io.ebean.Database; -import java.util.Optional; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.Bean; @@ -38,8 +35,6 @@ public class UpgradeCliApplicationTestConfiguration { @Bean public SchemaRegistryService schemaRegistryService() { - SchemaRegistryService mockService = mock(SchemaRegistryService.class); - when(mockService.getSchemaIdForTopic(anyString())).thenReturn(Optional.of(0)); - return mockService; + return new SchemaRegistryServiceImpl(new TopicConventionImpl()); } } diff --git a/docker/build.gradle b/docker/build.gradle index b14739104a..20608bd857 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -42,6 +42,7 @@ tasks.register('quickstart') {} tasks.register('quickstartSlim') {} tasks.register('quickstartDebug') {} tasks.register('quickstartPg') {} +tasks.register('quickstartStorage') {} tasks.register('quickstartNuke') { doFirst { @@ -133,6 +134,19 @@ dockerCompose { stopContainers = false removeVolumes = false } + + quickstartStorage { + isRequiredBy(tasks.named('quickstartStorage')) + composeAdditionalArgs = ['--profile', 'quickstart-storage'] + + useComposeFiles = ['profiles/docker-compose.yml'] + projectName = 'datahub' + projectNamePrefix = '' + buildBeforeUp = false + buildBeforePull = false + stopContainers = false + removeVolumes = false + } } tasks.getByName('quickstartComposeUp').dependsOn( quickstart_modules.collect { it + ':dockerTag' }) diff --git a/docs/authentication/concepts.md b/docs/authentication/concepts.md index 0940f86a80..39117bf512 100644 --- a/docs/authentication/concepts.md +++ b/docs/authentication/concepts.md @@ -75,7 +75,7 @@ for configuring multiple ways to authenticate a given request, for example via L Only if each Authenticator within the chain fails to authenticate a request will it be rejected. -The Authenticator Chain can be configured in the `application.yml` file under `authentication.authenticators`: +The Authenticator Chain can be configured in the `application.yaml` file under `authentication.authenticators`: ``` authentication: diff --git a/docs/authentication/introducing-metadata-service-authentication.md b/docs/authentication/introducing-metadata-service-authentication.md index 30b4cd0757..863f1f85e7 100644 --- a/docs/authentication/introducing-metadata-service-authentication.md +++ b/docs/authentication/introducing-metadata-service-authentication.md @@ -53,7 +53,7 @@ To enable Metadata Service Authentication: OR -- change the Metadata Service `application.yml` configuration file to set `authentication.enabled` to "true" AND +- change the Metadata Service `application.yaml` configuration file to set `authentication.enabled` to "true" AND - change the Frontend Proxy Service `application.config` configuration file to set `metadataService.auth.enabled` to "true" After setting the configuration flag, simply restart the Metadata Service to start enforcing Authentication. @@ -116,7 +116,7 @@ These changes represent the first milestone in Metadata Service Authentication. That's perfectly fine, for now. Metadata Service Authentication is disabled by default, only enabled if you provide the environment variable `METADATA_SERVICE_AUTH_ENABLED` to the `datahub-gms` container or change the `authentication.enabled` to "true" -inside your DataHub Metadata Service configuration (`application.yml`). +inside your DataHub Metadata Service configuration (`application.yaml`). That being said, we will be recommending that you enable Authentication for production use cases, to prevent arbitrary actors from ingesting metadata into DataHub. @@ -141,7 +141,7 @@ the root "datahub" user account. ### I want to authenticate requests using a custom Authenticator? How do I do this? -You can configure DataHub to add your custom **Authenticator** to the **Authentication Chain** by changing the `application.yml` configuration file for the Metadata Service: +You can configure DataHub to add your custom **Authenticator** to the **Authentication Chain** by changing the `application.yaml` configuration file for the Metadata Service: ```yml authentication: diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index 355ed41467..d1003077e2 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -332,7 +332,7 @@ and [here](../../metadata-service/factories/src/main/java/com/linkedin/gms/facto . A mapping between the property name used in the above two files and the name used in docker/env file can be -found [here](../../metadata-service/configuration/src/main/resources/application.yml). +found [here](../../metadata-service/configuration/src/main/resources/application.yaml). ### Managed Streaming for Apache Kafka (MSK) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index d880c17709..8fe33f45c6 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -328,7 +328,7 @@ Helm with `--atomic`: In general, it is recommended to not use the `--atomic` se - #6243 apache-ranger authorizer is no longer the core part of DataHub GMS, and it is shifted as plugin. Please refer updated documentation [Configuring Authorization with Apache Ranger](./configuring-authorization-with-apache-ranger.md#configuring-your-datahub-deployment) for configuring `apache-ranger-plugin` in DataHub GMS. - #6243 apache-ranger authorizer as plugin is not supported in DataHub Kubernetes deployment. -- #6243 Authentication and Authorization plugins configuration are removed from [application.yml](../../metadata-service/configuration/src/main/resources/application.yml). Refer documentation [Migration Of Plugins From application.yml](../plugins.md#migration-of-plugins-from-applicationyml) for migrating any existing custom plugins. +- #6243 Authentication and Authorization plugins configuration are removed from [application.yaml](../../metadata-service/configuration/src/main/resources/application.yaml). Refer documentation [Migration Of Plugins From application.yaml](../plugins.md#migration-of-plugins-from-applicationyml) for migrating any existing custom plugins. - `datahub check graph-consistency` command has been removed. It was a beta API that we had considered but decided there are better solutions for this. So removing this. - `graphql_url` option of `powerbi-report-server` source deprecated as the options is not used. - #6789 BigQuery ingestion: If `enable_legacy_sharded_table_support` is set to False, sharded table names will be suffixed with \_yyyymmdd to make sure they don't clash with non-sharded tables. This means if stateful ingestion is enabled then old sharded tables will be recreated with a new id and attached tags/glossary terms/etc will need to be added again. _This behavior is not enabled by default yet, but will be enabled by default in a future release._ diff --git a/docs/plugins.md b/docs/plugins.md index 1d49d99ffc..12c192b5b5 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -252,11 +252,11 @@ All other access are forbidden for the plugin. > Disclaimer: In BETA version your plugin can access any port and can read/write to any location on file system, however you should implement the plugin as per above access permission to keep your plugin compatible with upcoming release of DataHub. -## Migration Of Plugins From application.yml -If you have any custom Authentication or Authorization plugin define in `authorization` or `authentication` section of [application.yml](../metadata-service/configuration/src/main/resources/application.yml) then migrate them as per below steps. +## Migration Of Plugins From application.yaml +If you have any custom Authentication or Authorization plugin define in `authorization` or `authentication` section of [application.yaml](../metadata-service/configuration/src/main/resources/application.yaml) then migrate them as per below steps. 1. Implement Plugin: For Authentication Plugin follow steps of [Implementing an Authentication Plugin](#implementing-an-authentication-plugin) and for Authorization Plugin follow steps of [Implementing an Authorization Plugin](#implementing-an-authorization-plugin) -2. Install Plugin: Install the plugins as per steps mentioned in [Plugin Installation](#plugin-installation). Here you need to map the configuration from [application.yml](../metadata-service/configuration/src/main/resources/application.yml) to configuration in `config.yml`. This mapping from `application.yml` to `config.yml` is described below +2. Install Plugin: Install the plugins as per steps mentioned in [Plugin Installation](#plugin-installation). Here you need to map the configuration from [application.yaml](../metadata-service/configuration/src/main/resources/application.yaml) to configuration in `config.yml`. This mapping from `application.yaml` to `config.yml` is described below **Mapping for Authenticators** diff --git a/metadata-events/mxe-utils-avro/src/main/java/com/linkedin/metadata/EventUtils.java b/metadata-events/mxe-utils-avro/src/main/java/com/linkedin/metadata/EventUtils.java index adff32d5d3..8e92fd70b9 100644 --- a/metadata-events/mxe-utils-avro/src/main/java/com/linkedin/metadata/EventUtils.java +++ b/metadata-events/mxe-utils-avro/src/main/java/com/linkedin/metadata/EventUtils.java @@ -57,7 +57,7 @@ public class EventUtils { private static final Schema ORIGINAL_MCP_AVRO_SCHEMA = getAvroSchemaFromResource("avro/com/linkedin/mxe/MetadataChangeProposal.avsc"); - public static final Schema ORIGINAL_MCL_AVRO_SCHEMA = + private static final Schema ORIGINAL_MCL_AVRO_SCHEMA = getAvroSchemaFromResource("avro/com/linkedin/mxe/MetadataChangeLog.avsc"); private static final Schema ORIGINAL_FMCL_AVRO_SCHEMA = @@ -84,7 +84,7 @@ public class EventUtils { private static final Schema RENAMED_MCP_AVRO_SCHEMA = com.linkedin.pegasus2avro.mxe.MetadataChangeProposal.SCHEMA$; - private static final Schema RENAMED_MCL_AVRO_SCHEMA = + public static final Schema RENAMED_MCL_AVRO_SCHEMA = com.linkedin.pegasus2avro.mxe.MetadataChangeLog.SCHEMA$; private static final Schema RENAMED_FMCP_AVRO_SCHEMA = diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java index 2b2643f088..35693ec96b 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java @@ -14,7 +14,10 @@ import org.springframework.context.annotation.FilterType; exclude = {ElasticsearchRestClientAutoConfiguration.class, CassandraAutoConfiguration.class}) @ComponentScan( basePackages = { + "com.linkedin.gms.factory.common", "com.linkedin.gms.factory.kafka", + "com.linkedin.gms.factory.kafka.common", + "com.linkedin.gms.factory.kafka.schemaregistry", "com.linkedin.metadata.boot.kafka", "com.linkedin.metadata.kafka", "com.linkedin.metadata.dao.producer", diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MclConsumerConfig.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MclConsumerConfig.java similarity index 56% rename from metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MclConsumerConfig.java rename to metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MclConsumerConfig.java index 280ca87d1c..16df598788 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MclConsumerConfig.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MclConsumerConfig.java @@ -1,10 +1,6 @@ package com.linkedin.metadata.kafka; -import static com.linkedin.metadata.Constants.*; - -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.gms.factory.common.GitVersionFactory; import com.linkedin.metadata.version.GitVersion; @@ -21,27 +17,15 @@ public class MclConsumerConfig { private final Map config; private final String configJson; - private static final ObjectMapper OBJECT_MAPPER = - new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL); - - static { - int maxSize = - Integer.parseInt( - System.getenv() - .getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); - OBJECT_MAPPER - .getFactory() - .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); - } - - public MclConsumerConfig(GitVersion gitVersion) throws JsonProcessingException { + public MclConsumerConfig(final GitVersion gitVersion, final ObjectMapper objectMapper) + throws JsonProcessingException { config = new HashMap<>(); config.put("noCode", "true"); Map versionConfig = new HashMap<>(); versionConfig.put("acryldata/datahub", gitVersion.toConfig()); config.put("versions", versionConfig); - configJson = OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(config); + configJson = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(config); } @GetMapping("/config") diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle index 4fc1944388..6afc12c30e 100644 --- a/metadata-jobs/mae-consumer/build.gradle +++ b/metadata-jobs/mae-consumer/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' } apply plugin: 'pegasus' @@ -32,7 +32,7 @@ dependencies { implementation externalDependency.neo4jJavaDriver implementation externalDependency.springKafka - implementation externalDependency.springActuator + implementation externalDependency.annotationApi implementation externalDependency.slf4jApi compileOnly externalDependency.lombok diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java index 35107d00ab..e30fea976f 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java @@ -11,7 +11,7 @@ import com.linkedin.gms.factory.timeseries.TimeseriesAspectServiceFactory; import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.mxe.MetadataChangeLog; import io.datahubproject.metadata.context.OperationContext; -import jakarta.annotation.Nonnull; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Import; diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTest.java index 44536cd4fc..0b0f7c7bce 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTest.java @@ -24,7 +24,7 @@ import org.testng.annotations.Test; "kafka.schemaRegistry.type=INTERNAL" }) @TestPropertySource( - locations = "classpath:/application.yml", + locations = "classpath:/application.yaml", properties = {"MCL_CONSUMER_ENABLED=true"}) @EnableAutoConfiguration(exclude = {CassandraAutoConfiguration.class}) public class MCLSpringTest extends AbstractTestNGSpringContextTests { diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java index 3c1ba61cf3..dac4e98b62 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java @@ -6,12 +6,10 @@ import static org.mockito.Mockito.when; import com.datahub.authentication.Authentication; import com.datahub.metadata.ingestion.IngestionScheduler; import com.linkedin.entity.client.SystemEntityClient; -import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; import com.linkedin.metadata.boot.kafka.DataHubUpgradeKafkaListener; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.registry.SchemaRegistryService; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; @@ -36,6 +34,8 @@ import org.springframework.kafka.core.DefaultKafkaConsumerFactory; @ComponentScan( basePackages = { "com.linkedin.metadata.kafka", + "com.linkedin.gms.factory.kafka.common", + "com.linkedin.gms.factory.kafka.schemaregistry", "com.linkedin.gms.factory.entity.update.indices", "com.linkedin.gms.factory.timeline.eventgenerator" }) @@ -68,14 +68,9 @@ public class MCLSpringTestConfiguration { @MockBean(name = "dataHubUpgradeKafkaListener") public DataHubUpgradeKafkaListener dataHubUpgradeKafkaListener; - @MockBean(name = "duheSchemaRegistryConfig") - public SchemaRegistryConfig schemaRegistryConfig; - @MockBean(name = "duheKafkaConsumerFactory") public DefaultKafkaConsumerFactory defaultKafkaConsumerFactory; - @MockBean public SchemaRegistryService schemaRegistryService; - @MockBean public EntityIndexBuilders entityIndexBuilders; @Bean(name = "systemOperationContext") diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java index 0b0c8f622e..af3caecba8 100644 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java +++ b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java @@ -40,7 +40,7 @@ import org.springframework.context.annotation.PropertySource; type = FilterType.ASSIGNABLE_TYPE, classes = {ScheduledAnalyticsFactory.class}) }) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +@PropertySource(value = "classpath:/application.yaml", factory = YamlPropertySourceFactory.class) public class MceConsumerApplication { public static void main(String[] args) { diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/EbeanServerConfig.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/EbeanServerConfig.java index 990e0df102..e57aac6ab2 100644 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/EbeanServerConfig.java +++ b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/EbeanServerConfig.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.restli; -import static com.linkedin.gms.factory.common.LocalEbeanServerConfigFactory.getListenerToTrackCounts; +import static com.linkedin.gms.factory.common.LocalEbeanConfigFactory.getListenerToTrackCounts; import io.ebean.datasource.DataSourceConfig; import java.util.HashMap; diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle index 17ec825712..b062547724 100644 --- a/metadata-jobs/mce-consumer/build.gradle +++ b/metadata-jobs/mce-consumer/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' id 'pegasus' } @@ -28,7 +28,6 @@ dependencies { implementation externalDependency.protobuf implementation externalDependency.springKafka - implementation externalDependency.springActuator implementation externalDependency.slf4jApi compileOnly externalDependency.lombok diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java index b231d9a10e..1b3d19915b 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java @@ -6,8 +6,6 @@ import com.codahale.metrics.Timer; import com.linkedin.entity.Entity; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory; -import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; -import com.linkedin.gms.factory.kafka.KafkaEventConsumerFactory; import com.linkedin.metadata.EventUtils; import com.linkedin.metadata.kafka.config.MetadataChangeProposalProcessorCondition; import com.linkedin.metadata.snapshot.Snapshot; @@ -38,11 +36,7 @@ import org.springframework.stereotype.Component; @Slf4j @Component @Conditional(MetadataChangeProposalProcessorCondition.class) -@Import({ - RestliEntityClientFactory.class, - KafkaEventConsumerFactory.class, - DataHubKafkaProducerFactory.class -}) +@Import({RestliEntityClientFactory.class}) @EnableKafka @RequiredArgsConstructor public class MetadataChangeEventsProcessor { diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java index c570dcb297..60d9c7496d 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java @@ -5,8 +5,6 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory; -import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; -import com.linkedin.gms.factory.kafka.KafkaEventConsumerFactory; import com.linkedin.metadata.EventUtils; import com.linkedin.metadata.kafka.config.MetadataChangeProposalProcessorCondition; import com.linkedin.metadata.utils.metrics.MetricUtils; @@ -33,11 +31,7 @@ import org.springframework.stereotype.Component; @Slf4j @Component -@Import({ - RestliEntityClientFactory.class, - KafkaEventConsumerFactory.class, - DataHubKafkaProducerFactory.class -}) +@Import({RestliEntityClientFactory.class}) @Conditional(MetadataChangeProposalProcessorCondition.class) @EnableKafka @RequiredArgsConstructor diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle index 3c9e916a96..5c031dafba 100644 --- a/metadata-jobs/pe-consumer/build.gradle +++ b/metadata-jobs/pe-consumer/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' id 'pegasus' } @@ -17,7 +17,6 @@ dependencies { exclude group: 'org.neo4j.test' } implementation externalDependency.springKafka - implementation externalDependency.springActuator implementation externalDependency.slf4jApi compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java b/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java index b8553235b3..442263bbd6 100644 --- a/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java +++ b/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticationConfiguration.java @@ -3,7 +3,7 @@ package com.datahub.authentication; import java.util.List; import lombok.Data; -/** POJO representing the "authentication" configuration block in application.yml. */ +/** POJO representing the "authentication" configuration block in application.yaml. */ @Data public class AuthenticationConfiguration { /** Whether authentication is enabled */ diff --git a/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticatorConfiguration.java b/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticatorConfiguration.java index 36814ee380..bb044b875e 100644 --- a/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticatorConfiguration.java +++ b/metadata-service/auth-config/src/main/java/com/datahub/authentication/AuthenticatorConfiguration.java @@ -5,7 +5,7 @@ import lombok.Data; /** * POJO representing {@link com.datahub.plugins.auth.authentication.Authenticator} configurations - * provided in the application.yml. + * provided in the application.yaml. */ @Data public class AuthenticatorConfiguration { diff --git a/metadata-service/auth-config/src/main/java/com/datahub/authorization/AuthorizationConfiguration.java b/metadata-service/auth-config/src/main/java/com/datahub/authorization/AuthorizationConfiguration.java index 139be5cca8..07fa512f5b 100644 --- a/metadata-service/auth-config/src/main/java/com/datahub/authorization/AuthorizationConfiguration.java +++ b/metadata-service/auth-config/src/main/java/com/datahub/authorization/AuthorizationConfiguration.java @@ -5,7 +5,7 @@ import com.datahub.plugins.auth.authorization.Authorizer; import java.util.List; import lombok.Data; -/** POJO representing the "authentication" configuration block in application.yml. */ +/** POJO representing the "authentication" configuration block in application.yaml. */ @Data public class AuthorizationConfiguration { /** Configuration for the default DataHub Policies-based authorizer. */ diff --git a/metadata-service/auth-config/src/main/java/com/datahub/authorization/AuthorizerConfiguration.java b/metadata-service/auth-config/src/main/java/com/datahub/authorization/AuthorizerConfiguration.java index c4a26a1cd6..1f584b7d61 100644 --- a/metadata-service/auth-config/src/main/java/com/datahub/authorization/AuthorizerConfiguration.java +++ b/metadata-service/auth-config/src/main/java/com/datahub/authorization/AuthorizerConfiguration.java @@ -4,7 +4,7 @@ import com.datahub.plugins.auth.authorization.Authorizer; import java.util.Map; import lombok.Data; -/** POJO representing {@link Authorizer} configurations provided in the application.yml. */ +/** POJO representing {@link Authorizer} configurations provided in the application.yaml. */ @Data public class AuthorizerConfiguration { /** Whether to enable this authorizer */ diff --git a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java index c4afabd3a9..dd68e4d36f 100644 --- a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java +++ b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java @@ -28,7 +28,6 @@ import com.datahub.plugins.loader.PluginPermissionManagerImpl; import com.google.common.collect.ImmutableMap; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.entity.EntityService; -import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.servlet.Filter; import jakarta.servlet.FilterChain; @@ -48,6 +47,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.web.context.support.SpringBeanAutowiringSupport; @@ -58,13 +58,13 @@ import org.springframework.web.context.support.SpringBeanAutowiringSupport; @Slf4j public class AuthenticationFilter implements Filter { - @Inject private ConfigurationProvider configurationProvider; + @Autowired private ConfigurationProvider configurationProvider; - @Inject + @Autowired @Named("entityService") - private EntityService _entityService; + private EntityService _entityService; - @Inject + @Autowired @Named("dataHubTokenService") private StatefulTokenService _tokenService; @@ -252,7 +252,7 @@ public class AuthenticationFilter implements Filter { authenticatorChain.register( systemAuthenticator); // Always register authenticator for internal system. - // Register authenticator define in application.yml + // Register authenticator define in application.yaml final List authenticatorConfigurations = this.configurationProvider.getAuthentication().getAuthenticators(); for (AuthenticatorConfiguration internalAuthenticatorConfig : authenticatorConfigurations) { diff --git a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java index 1d91f4680b..bb6aa1ed23 100644 --- a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java +++ b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java @@ -23,13 +23,12 @@ import com.linkedin.settings.global.OidcSettings; import com.linkedin.settings.global.SsoSettings; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.services.SecretService; -import jakarta.inject.Inject; -import jakarta.inject.Named; import java.util.Objects; import java.util.concurrent.CompletableFuture; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; @@ -78,27 +77,29 @@ public class AuthServiceController { private static final String PREFERRED_JWS_ALGORITHM = "preferredJwsAlgorithm"; private static final String PREFERRED_JWS_ALGORITHM_2 = "preferredJwsAlgorithm2"; - @Inject StatelessTokenService _statelessTokenService; + @Autowired private StatelessTokenService _statelessTokenService; - @Inject Authentication _systemAuthentication; + @Autowired private Authentication _systemAuthentication; - @Inject + @Autowired @Qualifier("configurationProvider") - ConfigurationProvider _configProvider; + private ConfigurationProvider _configProvider; - @Inject NativeUserService _nativeUserService; + @Autowired private NativeUserService _nativeUserService; - @Inject EntityService _entityService; + @Autowired private EntityService _entityService; - @Inject SecretService _secretService; + @Autowired private SecretService _secretService; - @Inject InviteTokenService _inviteTokenService; + @Autowired private InviteTokenService _inviteTokenService; - @Inject @Nullable TrackingService _trackingService; + @Autowired @Nullable private TrackingService _trackingService; - @Inject - @Named("systemOperationContext") - OperationContext systemOperationContext; + @Autowired private ObjectMapper mapper; + + @Autowired + @Qualifier("systemOperationContext") + private OperationContext systemOperationContext; /** * Generates a JWT access token for as user UI session, provided a unique "user id" to generate @@ -117,7 +118,7 @@ public class AuthServiceController { CompletableFuture> generateSessionTokenForUser( final HttpEntity httpEntity) { String jsonStr = httpEntity.getBody(); - ObjectMapper mapper = new ObjectMapper(); + JsonNode bodyJson = null; try { bodyJson = mapper.readTree(jsonStr); @@ -183,7 +184,7 @@ public class AuthServiceController { @PostMapping(value = "/signUp", produces = "application/json;charset=utf-8") CompletableFuture> signUp(final HttpEntity httpEntity) { String jsonStr = httpEntity.getBody(); - ObjectMapper mapper = new ObjectMapper(); + JsonNode bodyJson; try { bodyJson = mapper.readTree(jsonStr); @@ -273,7 +274,7 @@ public class AuthServiceController { CompletableFuture> resetNativeUserCredentials( final HttpEntity httpEntity) { String jsonStr = httpEntity.getBody(); - ObjectMapper mapper = new ObjectMapper(); + JsonNode bodyJson; try { bodyJson = mapper.readTree(jsonStr); @@ -332,7 +333,7 @@ public class AuthServiceController { CompletableFuture> verifyNativeUserCredentials( final HttpEntity httpEntity) { String jsonStr = httpEntity.getBody(); - ObjectMapper mapper = new ObjectMapper(); + JsonNode bodyJson; try { bodyJson = mapper.readTree(jsonStr); @@ -377,7 +378,7 @@ public class AuthServiceController { @PostMapping(value = "/track", produces = "application/json;charset=utf-8") CompletableFuture> track(final HttpEntity httpEntity) { String jsonStr = httpEntity.getBody(); - ObjectMapper mapper = new ObjectMapper(); + JsonNode bodyJson; try { bodyJson = mapper.readTree(jsonStr); diff --git a/metadata-service/auth-servlet-impl/src/test/java/com/datahub/auth/authentication/AuthServiceTestConfiguration.java b/metadata-service/auth-servlet-impl/src/test/java/com/datahub/auth/authentication/AuthServiceTestConfiguration.java index 6eda80b418..ff42173c25 100644 --- a/metadata-service/auth-servlet-impl/src/test/java/com/datahub/auth/authentication/AuthServiceTestConfiguration.java +++ b/metadata-service/auth-servlet-impl/src/test/java/com/datahub/auth/authentication/AuthServiceTestConfiguration.java @@ -5,12 +5,14 @@ import com.datahub.authentication.invite.InviteTokenService; import com.datahub.authentication.token.StatelessTokenService; import com.datahub.authentication.user.NativeUserService; import com.datahub.telemetry.TrackingService; +import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.entity.EntityService; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.services.SecretService; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; @TestConfiguration public class AuthServiceTestConfiguration { @@ -33,4 +35,9 @@ public class AuthServiceTestConfiguration { @MockBean InviteTokenService _inviteTokenService; @MockBean TrackingService _trackingService; + + @Bean + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } } diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java index 72dfbf84a4..2ba1842ca8 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/DataHubConfiguration.java @@ -2,7 +2,7 @@ package com.linkedin.metadata.config; import lombok.Data; -/** POJO representing the "datahub" configuration block in application.yml. */ +/** POJO representing the "datahub" configuration block in application.yaml. */ @Data public class DataHubConfiguration { /** diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java index 2f3c3dc3bd..9bdcfe61d1 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/IngestionConfiguration.java @@ -2,7 +2,7 @@ package com.linkedin.metadata.config; import lombok.Data; -/** POJO representing the "ingestion" configuration block in application.yml. */ +/** POJO representing the "ingestion" configuration block in application.yaml. */ @Data public class IngestionConfiguration { /** Whether managed ingestion is enabled */ diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java index 738a9684cc..5e97641601 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/TestsConfiguration.java @@ -2,7 +2,7 @@ package com.linkedin.metadata.config; import lombok.Data; -/** POJO representing the "tests" configuration block in application.yml.on.yml */ +/** POJO representing the "tests" configuration block in application.yaml.on.yml */ @Data public class TestsConfiguration { /** Whether tests are enabled */ diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java index 670a412ec2..4067a357eb 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java @@ -2,7 +2,7 @@ package com.linkedin.metadata.config; import lombok.Data; -/** POJO representing the "views" configuration block in application.yml.on.yml */ +/** POJO representing the "views" configuration block in application.yaml.on.yml */ @Data public class ViewsConfiguration { /** Whether Views are enabled */ diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java index eb5243c0e5..e597421e12 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java @@ -2,7 +2,7 @@ package com.linkedin.metadata.config; import lombok.Data; -/** POJO representing visualConfig block in the application.yml. */ +/** POJO representing visualConfig block in the application.yaml. */ @Data public class VisualConfiguration { /** Asset related configurations */ diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java index 82e3868fa3..2a492c746c 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/CustomConfiguration.java @@ -17,7 +17,7 @@ public class CustomConfiguration { private String file; /** - * Materialize the search configuration from a location external to main application.yml + * Materialize the search configuration from a location external to main application.yaml * * @param mapper yaml enabled jackson mapper * @return search configuration class diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/telemetry/TelemetryConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/telemetry/TelemetryConfiguration.java index aa6825360a..af3ed761d8 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/telemetry/TelemetryConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/telemetry/TelemetryConfiguration.java @@ -2,7 +2,7 @@ package com.linkedin.metadata.config.telemetry; import lombok.Data; -/** POJO representing the "telemetry" configuration block in application.yml. */ +/** POJO representing the "telemetry" configuration block in application.yaml. */ @Data public class TelemetryConfiguration { /** Whether cli telemetry is enabled */ diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/spring/YamlPropertySourceFactory.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/spring/YamlPropertySourceFactory.java index f84ac9ec8b..b1cddd931c 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/spring/YamlPropertySourceFactory.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/spring/YamlPropertySourceFactory.java @@ -8,7 +8,7 @@ import org.springframework.core.env.PropertySource; import org.springframework.core.io.support.EncodedResource; import org.springframework.core.io.support.PropertySourceFactory; -/** Required for Spring to parse the application.yml provided by this module */ +/** Required for Spring to parse the application.yaml provided by this module */ public class YamlPropertySourceFactory implements PropertySourceFactory { @Override diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yaml similarity index 100% rename from metadata-service/configuration/src/main/resources/application.yml rename to metadata-service/configuration/src/main/resources/application.yaml diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle index 8bce521a17..76e4de7007 100644 --- a/metadata-service/factories/build.gradle +++ b/metadata-service/factories/build.gradle @@ -14,6 +14,7 @@ dependencies { implementation project(':metadata-service:restli-servlet-impl') implementation project(':metadata-dao-impl:kafka-producer') implementation project(':ingestion-scheduler') + implementation project(':metadata-service:schema-registry-api') implementation (externalDependency.awsGlueSchemaRegistrySerde) { exclude group: 'org.json', module: 'json' diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java index 0d599b57db..d94d970316 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java @@ -19,7 +19,6 @@ import com.datahub.plugins.loader.PluginPermissionManagerImpl; import com.google.common.collect.ImmutableMap; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.datahubproject.metadata.context.OperationContext; import jakarta.annotation.Nonnull; import java.nio.file.Path; @@ -35,12 +34,10 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Slf4j @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({DataHubAuthorizerFactory.class}) public class AuthorizerChainFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java index d29b770cbf..f2a875206d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java @@ -2,18 +2,15 @@ package com.linkedin.gms.factory.auth; import com.datahub.authorization.DataHubAuthorizer; import com.linkedin.entity.client.SystemEntityClient; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.datahubproject.metadata.context.OperationContext; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class DataHubAuthorizerFactory { @Value("${authorization.defaultAuthorizer.cacheRefreshIntervalSecs}") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java index f3a455b8da..a4ff18e0f6 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java @@ -2,7 +2,6 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.token.StatefulTokenService; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.datahubproject.metadata.context.OperationContext; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; @@ -10,11 +9,9 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class DataHubTokenServiceFactory { @Value("${authentication.tokenService.signingKey:}") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java index 47af58a8d8..a32d9d6ba6 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java @@ -4,17 +4,14 @@ import com.datahub.authentication.group.GroupService; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class GroupServiceFactory { @Autowired @Qualifier("entityService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/InviteTokenServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/InviteTokenServiceFactory.java index 5e486f0bdb..ccdc5dc44a 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/InviteTokenServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/InviteTokenServiceFactory.java @@ -2,18 +2,15 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.invite.InviteTokenService; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.datahubproject.metadata.services.SecretService; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class InviteTokenServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java index 92b0667f53..53d762a616 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java @@ -4,18 +4,15 @@ import com.datahub.authentication.user.NativeUserService; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.datahubproject.metadata.services.SecretService; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class NativeUserServiceFactory { @Autowired @Qualifier("entityService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/PostServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/PostServiceFactory.java index 317d8583ef..90805cf5f4 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/PostServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/PostServiceFactory.java @@ -2,16 +2,13 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.post.PostService; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class PostServiceFactory { @Bean(name = "postService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/RoleServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/RoleServiceFactory.java index 9321e2544a..c21bd1abc4 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/RoleServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/RoleServiceFactory.java @@ -2,16 +2,13 @@ package com.linkedin.gms.factory.auth; import com.datahub.authorization.role.RoleService; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class RoleServiceFactory { @Bean(name = "roleService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/SystemAuthenticationFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/SystemAuthenticationFactory.java index efe688ceee..649e34ae48 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/SystemAuthenticationFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/SystemAuthenticationFactory.java @@ -3,14 +3,12 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.Actor; import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; /** @@ -19,7 +17,6 @@ import org.springframework.context.annotation.Scope; */ @Configuration @ConfigurationProperties -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Data public class SystemAuthenticationFactory { diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticSearchGraphServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticSearchGraphServiceFactory.java index aa47a35f3d..eb56e8d42c 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticSearchGraphServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticSearchGraphServiceFactory.java @@ -8,17 +8,14 @@ import com.linkedin.metadata.graph.elastic.ESGraphWriteDAO; import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.LineageRegistry; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({BaseElasticSearchComponentsFactory.class, EntityRegistryFactory.class}) public class ElasticSearchGraphServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticSearchSystemMetadataServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticSearchSystemMetadataServiceFactory.java index 20c3e92767..d560fba399 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticSearchSystemMetadataServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticSearchSystemMetadataServiceFactory.java @@ -1,7 +1,6 @@ package com.linkedin.gms.factory.common; import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.systemmetadata.ESSystemMetadataDAO; import com.linkedin.metadata.systemmetadata.ElasticSearchSystemMetadataService; import javax.annotation.Nonnull; @@ -10,10 +9,8 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({BaseElasticSearchComponentsFactory.class}) public class ElasticSearchSystemMetadataServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticsearchSSLContextFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticsearchSSLContextFactory.java index 483251644b..a713f82090 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticsearchSSLContextFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticsearchSSLContextFactory.java @@ -1,6 +1,5 @@ package com.linkedin.gms.factory.common; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -17,10 +16,8 @@ import org.apache.http.ssl.SSLContextBuilder; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class ElasticsearchSSLContextFactory { @Value("${elasticsearch.sslContext.protocol}") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/GraphServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/GraphServiceFactory.java index d98dfcb617..a731e8bcb8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/GraphServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/GraphServiceFactory.java @@ -3,7 +3,6 @@ package com.linkedin.gms.factory.common; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; import com.linkedin.metadata.graph.neo4j.Neo4jGraphService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -13,10 +12,8 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({Neo4jGraphServiceFactory.class, ElasticSearchGraphServiceFactory.class}) public class GraphServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/IndexConventionFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/IndexConventionFactory.java index b268bb0937..5b76a3f2cb 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/IndexConventionFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/IndexConventionFactory.java @@ -1,12 +1,10 @@ package com.linkedin.gms.factory.common; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; /** * Creates a {@link IndexConvention} to generate search index names. @@ -14,7 +12,6 @@ import org.springframework.context.annotation.PropertySource; *

This allows you to easily add prefixes to the index names. */ @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class IndexConventionFactory { public static final String INDEX_CONVENTION_BEAN = "searchIndexConvention"; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanConfigFactory.java similarity index 87% rename from metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java rename to metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanConfigFactory.java index 08787cdb89..6edd5189cf 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanServerConfigFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/LocalEbeanConfigFactory.java @@ -1,8 +1,7 @@ package com.linkedin.gms.factory.common; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.utils.metrics.MetricUtils; -import io.ebean.config.ServerConfig; +import io.ebean.config.DatabaseConfig; import io.ebean.datasource.DataSourceConfig; import io.ebean.datasource.DataSourcePoolListener; import java.sql.Connection; @@ -13,12 +12,10 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Slf4j @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) -public class LocalEbeanServerConfigFactory { +public class LocalEbeanConfigFactory { @Value("${ebean.username}") private String ebeanDatasourceUsername; @@ -91,11 +88,11 @@ public class LocalEbeanServerConfigFactory { return dataSourceConfig; } - @Bean(name = "gmsEbeanServiceConfig") - protected ServerConfig createInstance( + @Bean(name = "gmsEbeanDatabaseConfig") + protected DatabaseConfig createInstance( @Qualifier("ebeanDataSourceConfig") DataSourceConfig config) { - ServerConfig serverConfig = new ServerConfig(); - serverConfig.setName("gmsEbeanServiceConfig"); + DatabaseConfig serverConfig = new DatabaseConfig(); + serverConfig.setName("gmsEbeanDatabaseConfig"); serverConfig.setDataSourceConfig(config); serverConfig.setDdlGenerate(ebeanAutoCreate); serverConfig.setDdlRun(ebeanAutoCreate); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jDriverFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jDriverFactory.java index 04ed294075..712fb44db2 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jDriverFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jDriverFactory.java @@ -1,6 +1,5 @@ package com.linkedin.gms.factory.common; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import java.util.concurrent.TimeUnit; import org.neo4j.driver.AuthTokens; import org.neo4j.driver.Config; @@ -9,10 +8,8 @@ import org.neo4j.driver.GraphDatabase; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class Neo4jDriverFactory { @Value("${neo4j.username}") private String username; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ObjectMapperFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ObjectMapperFactory.java new file mode 100644 index 0000000000..ee0aa281e4 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ObjectMapperFactory.java @@ -0,0 +1,25 @@ +package com.linkedin.gms.factory.common; + +import static com.linkedin.metadata.Constants.INGESTION_MAX_SERIALIZED_STRING_LENGTH; +import static com.linkedin.metadata.Constants.MAX_JACKSON_STRING_SIZE; + +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ObjectMapperFactory { + @Bean + public ObjectMapper objectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + int maxSize = + Integer.parseInt( + System.getenv() + .getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); + objectMapper + .getFactory() + .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); + return objectMapper; + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java index ddd31f2692..06b0229d7d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java @@ -1,7 +1,6 @@ package com.linkedin.gms.factory.common; import com.linkedin.gms.factory.auth.AwsRequestSigningApacheInterceptor; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import java.io.IOException; import javax.annotation.Nonnull; import javax.net.ssl.HostnameVerifier; @@ -38,13 +37,11 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.signer.Aws4Signer; @Slf4j @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({ElasticsearchSSLContextFactory.class}) public class RestHighLevelClientFactory { diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java index 89a4275c27..9381e24fab 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java @@ -15,15 +15,16 @@ import com.linkedin.metadata.config.cache.CacheConfiguration; import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.config.telemetry.TelemetryConfiguration; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import lombok.Data; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.stereotype.Component; -@Configuration +@Component +// Include extra kafka properties +@EnableConfigurationProperties(KafkaProperties.class) @ConfigurationProperties -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Data public class ConfigurationProvider { /** Authentication related configs */ diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/services/RestrictedServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/services/RestrictedServiceFactory.java index 1fbcb7ff5a..9313eb67f2 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/services/RestrictedServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/context/services/RestrictedServiceFactory.java @@ -1,6 +1,5 @@ package com.linkedin.gms.factory.context.services; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.datahubproject.metadata.services.RestrictedService; import io.datahubproject.metadata.services.SecretService; import javax.annotation.Nonnull; @@ -8,11 +7,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class RestrictedServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/dataproduct/DataProductServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/dataproduct/DataProductServiceFactory.java index 39d42b6fb7..1bcc278178 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/dataproduct/DataProductServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/dataproduct/DataProductServiceFactory.java @@ -3,17 +3,14 @@ package com.linkedin.gms.factory.dataproduct; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.service.DataProductService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class DataProductServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EbeanServerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EbeanDatabaseFactory.java similarity index 68% rename from metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EbeanServerFactory.java rename to metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EbeanDatabaseFactory.java index 2bfe7bff1b..efd3b1e9b7 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EbeanServerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EbeanDatabaseFactory.java @@ -2,29 +2,24 @@ package com.linkedin.gms.factory.entity; import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import io.ebean.Database; -import io.ebean.config.ServerConfig; +import io.ebean.config.DatabaseConfig; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; @Configuration @Slf4j -public class EbeanServerFactory { +public class EbeanDatabaseFactory { public static final String EBEAN_MODEL_PACKAGE = EbeanAspectV2.class.getPackage().getName(); - @Autowired ApplicationContext applicationContext; - - @Bean(name = "ebeanServer") - @DependsOn({"gmsEbeanServiceConfig"}) + @Bean("ebeanServer") @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull - protected Database createServer() { - ServerConfig serverConfig = applicationContext.getBean(ServerConfig.class); + protected Database createServer( + @Qualifier("gmsEbeanDatabaseConfig") DatabaseConfig serverConfig) { // Make sure that the serverConfig includes the package that contains DAO's Ebean model. if (!serverConfig.getPackages().contains(EBEAN_MODEL_PACKAGE)) { serverConfig.getPackages().add(EBEAN_MODEL_PACKAGE); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityAspectDaoFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityAspectDaoFactory.java index 22eced4fd5..a6a9fc4809 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityAspectDaoFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityAspectDaoFactory.java @@ -7,6 +7,7 @@ import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import io.ebean.Database; import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -16,11 +17,11 @@ import org.springframework.context.annotation.DependsOn; public class EntityAspectDaoFactory { @Bean(name = "entityAspectDao") - @DependsOn({"gmsEbeanServiceConfig"}) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull protected AspectDao createEbeanInstance( - Database server, final ConfigurationProvider configurationProvider) { + @Qualifier("ebeanServer") final Database server, + final ConfigurationProvider configurationProvider) { return new EbeanAspectDao(server, configurationProvider.getEbean()); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityAspectMigrationsDaoFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityAspectMigrationsDaoFactory.java index 2bf9804030..6b6c7d29b4 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityAspectMigrationsDaoFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityAspectMigrationsDaoFactory.java @@ -7,6 +7,7 @@ import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import io.ebean.Database; import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -16,11 +17,11 @@ import org.springframework.context.annotation.DependsOn; public class EntityAspectMigrationsDaoFactory { @Bean(name = "entityAspectMigrationsDao") - @DependsOn({"gmsEbeanServiceConfig"}) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull protected AspectMigrationsDao createEbeanInstance( - Database server, final ConfigurationProvider configurationProvider) { + @Qualifier("ebeanServer") final Database server, + final ConfigurationProvider configurationProvider) { return new EbeanAspectDao(server, configurationProvider.getEbean()); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RetentionServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RetentionServiceFactory.java index db9d9c8e65..08c9b477a8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RetentionServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RetentionServiceFactory.java @@ -6,7 +6,6 @@ import com.linkedin.metadata.entity.RetentionService; import com.linkedin.metadata.entity.cassandra.CassandraRetentionService; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.ebean.Database; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; @@ -16,10 +15,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class RetentionServiceFactory { @Autowired @@ -41,10 +38,11 @@ public class RetentionServiceFactory { } @Bean(name = "retentionService") - @DependsOn({"ebeanServer", "entityService"}) + @DependsOn("entityService") @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull - protected RetentionService createEbeanInstance(Database server) { + protected RetentionService createEbeanInstance( + @Qualifier("ebeanServer") final Database server) { RetentionService retentionService = new EbeanRetentionService<>(_entityService, server, _batchSize); _entityService.setRetentionService(retentionService); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java index c6fe0d6e95..c6ed9c625d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java @@ -2,14 +2,11 @@ package com.linkedin.gms.factory.entityclient; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class EntityClientConfigFactory { @Bean diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java index 5591d0b573..2f92f0ad5b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java @@ -13,18 +13,15 @@ import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.service.RollbackService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import javax.inject.Singleton; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; /** The *Java* Entity Client should be preferred if executing within the GMS service. */ @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @ConditionalOnProperty(name = "entityClient.impl", havingValue = "java", matchIfMissing = true) public class JavaEntityClientFactory { diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java index db02c33d40..9da7fc706d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java @@ -6,7 +6,6 @@ import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.entity.client.SystemRestliEntityClient; import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig; import com.linkedin.metadata.restli.DefaultRestliClientFactory; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.parseq.retry.backoff.ExponentialBackoff; import com.linkedin.restli.client.Client; import java.net.URI; @@ -15,11 +14,9 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; /** The Java Entity Client should be preferred if executing within the GMS service. */ @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @ConditionalOnProperty(name = "entityClient.impl", havingValue = "restli") public class RestliEntityClientFactory { diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityregistry/ConfigEntityRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityregistry/ConfigEntityRegistryFactory.java index 356fb22693..f78dcf1b80 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityregistry/ConfigEntityRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityregistry/ConfigEntityRegistryFactory.java @@ -2,17 +2,14 @@ package com.linkedin.gms.factory.entityregistry; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistryException; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import java.io.IOException; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.core.io.Resource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class ConfigEntityRegistryFactory { @Value("${configEntityRegistry.path}") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationshipServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationshipServiceFactory.java index fd2467b708..f5345c8397 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationshipServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ermodelrelation/ERModelRelationshipServiceFactory.java @@ -2,16 +2,13 @@ package com.linkedin.gms.factory.ermodelrelation; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.ERModelRelationshipService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class ERModelRelationshipServiceFactory { @Bean(name = "erModelRelationshipService") @Scope("singleton") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/form/FormServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/form/FormServiceFactory.java index 195dbfbc61..c1db920d83 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/form/FormServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/form/FormServiceFactory.java @@ -2,15 +2,12 @@ package com.linkedin.gms.factory.form; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.FormService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class FormServiceFactory { @Bean(name = "formService") @Scope("singleton") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/incident/IncidentServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/incident/IncidentServiceFactory.java index 569d554f71..bb33a62bba 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/incident/IncidentServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/incident/IncidentServiceFactory.java @@ -3,16 +3,13 @@ package com.linkedin.gms.factory.incident; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; import com.linkedin.metadata.service.IncidentService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({SystemAuthenticationFactory.class}) public class IncidentServiceFactory { @Bean(name = "incidentService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ingestion/IngestionSchedulerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ingestion/IngestionSchedulerFactory.java index 10833109e6..2337d5e672 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ingestion/IngestionSchedulerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ingestion/IngestionSchedulerFactory.java @@ -4,7 +4,6 @@ import com.datahub.metadata.ingestion.IngestionScheduler; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.datahubproject.metadata.context.OperationContext; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; @@ -12,11 +11,9 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Import({SystemAuthenticationFactory.class}) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class IngestionSchedulerFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaEventProducerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaEventProducerFactory.java index 41807d0daa..fcc4ee2527 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaEventProducerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaEventProducerFactory.java @@ -1,9 +1,8 @@ package com.linkedin.gms.factory.kafka; -import com.linkedin.gms.factory.common.TopicConventionFactory; +import com.linkedin.gms.factory.kafka.common.TopicConventionFactory; import com.linkedin.metadata.dao.producer.KafkaEventProducer; import com.linkedin.metadata.dao.producer.KafkaHealthChecker; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.mxe.TopicConvention; import org.apache.avro.generic.IndexedRecord; import org.apache.kafka.clients.producer.Producer; @@ -11,12 +10,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) -@Import({DataHubKafkaProducerFactory.class, TopicConventionFactory.class, KafkaHealthChecker.class}) public class DataHubKafkaEventProducerFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java index 0b331ffc40..5844dc4a8f 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/DataHubKafkaProducerFactory.java @@ -1,12 +1,8 @@ package com.linkedin.gms.factory.kafka; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.gms.factory.kafka.schemaregistry.AwsGlueSchemaRegistryFactory; -import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; -import com.linkedin.gms.factory.kafka.schemaregistry.KafkaSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; import com.linkedin.metadata.config.kafka.KafkaConfiguration; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import java.util.Arrays; import java.util.Map; import org.apache.avro.generic.IndexedRecord; @@ -14,36 +10,24 @@ import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; +import org.springframework.context.annotation.DependsOn; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) -@EnableConfigurationProperties({KafkaProperties.class}) -@Import({ - KafkaSchemaRegistryFactory.class, - AwsGlueSchemaRegistryFactory.class, - InternalSchemaRegistryFactory.class -}) +@DependsOn("configurationProvider") public class DataHubKafkaProducerFactory { - @Autowired - @Qualifier("schemaRegistryConfig") - private SchemaRegistryConfig _schemaRegistryConfig; - @Bean(name = "kafkaProducer") protected Producer createInstance( @Qualifier("configurationProvider") ConfigurationProvider provider, - KafkaProperties properties) { + final KafkaProperties properties, + @Qualifier("schemaRegistryConfig") final SchemaRegistryConfig schemaRegistryConfig) { KafkaConfiguration kafkaConfiguration = provider.getKafka(); return new KafkaProducer<>( - buildProducerProperties(_schemaRegistryConfig, kafkaConfiguration, properties)); + buildProducerProperties(schemaRegistryConfig, kafkaConfiguration, properties)); } public static Map buildProducerProperties( diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java index 0d00218d19..d521021318 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java @@ -1,9 +1,6 @@ package com.linkedin.gms.factory.kafka; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.gms.factory.kafka.schemaregistry.AwsGlueSchemaRegistryFactory; -import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory; -import com.linkedin.gms.factory.kafka.schemaregistry.KafkaSchemaRegistryFactory; import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; import com.linkedin.metadata.config.kafka.KafkaConfiguration; import java.time.Duration; @@ -17,7 +14,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.config.KafkaListenerContainerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; @@ -29,11 +25,6 @@ import org.springframework.kafka.support.serializer.ErrorHandlingDeserializer; @Slf4j @Configuration -@Import({ - KafkaSchemaRegistryFactory.class, - AwsGlueSchemaRegistryFactory.class, - InternalSchemaRegistryFactory.class -}) public class KafkaEventConsumerFactory { private int kafkaEventConsumerConcurrency; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java index 58cb311c52..3a6c9770fd 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/SimpleKafkaConsumerFactory.java @@ -11,16 +11,16 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.config.KafkaListenerContainerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; @Slf4j @Configuration -@EnableConfigurationProperties({KafkaProperties.class}) +@DependsOn("configurationProvider") public class SimpleKafkaConsumerFactory { @Bean(name = "simpleKafkaConsumer") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/TopicConventionFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/common/TopicConventionFactory.java similarity index 98% rename from metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/TopicConventionFactory.java rename to metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/common/TopicConventionFactory.java index fac0bf0c46..39a8b9427f 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/TopicConventionFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/common/TopicConventionFactory.java @@ -1,4 +1,4 @@ -package com.linkedin.gms.factory.common; +package com.linkedin.gms.factory.kafka.common; import com.linkedin.mxe.TopicConvention; import com.linkedin.mxe.TopicConventionImpl; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/AwsGlueSchemaRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/AwsGlueSchemaRegistryFactory.java index c06ebae27f..91dce49e5b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/AwsGlueSchemaRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/AwsGlueSchemaRegistryFactory.java @@ -5,7 +5,6 @@ import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistryKafka import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; import com.amazonaws.services.schemaregistry.utils.AvroRecordType; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -15,11 +14,9 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Slf4j @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @ConditionalOnProperty( name = "kafka.schemaRegistry.type", havingValue = AwsGlueSchemaRegistryFactory.TYPE) diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/KafkaSchemaRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/KafkaSchemaRegistryFactory.java index e6c255b99a..e72fbf97e5 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/KafkaSchemaRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/KafkaSchemaRegistryFactory.java @@ -1,7 +1,6 @@ package com.linkedin.gms.factory.kafka.schemaregistry; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClientConfig; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; import io.confluent.kafka.serializers.KafkaAvroDeserializer; @@ -16,14 +15,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Slf4j @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) -@ConditionalOnProperty( - name = "kafka.schemaRegistry.type", - havingValue = KafkaSchemaRegistryFactory.TYPE) +@ConditionalOnProperty(name = "kafka.schemaRegistry.type", havingValue = "KAFKA") public class KafkaSchemaRegistryFactory { public static final String TYPE = "KAFKA"; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/SchemaRegistryServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/SchemaRegistryServiceFactory.java index a6869321d7..4fa5a55b5b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/SchemaRegistryServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/SchemaRegistryServiceFactory.java @@ -1,6 +1,6 @@ package com.linkedin.gms.factory.kafka.schemaregistry; -import com.linkedin.gms.factory.common.TopicConventionFactory; +import com.linkedin.gms.factory.kafka.common.TopicConventionFactory; import com.linkedin.metadata.registry.SchemaRegistryService; import com.linkedin.metadata.registry.SchemaRegistryServiceImpl; import com.linkedin.mxe.TopicConvention; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/lineage/LineageServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/lineage/LineageServiceFactory.java index d52a0e105d..8660319be2 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/lineage/LineageServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/lineage/LineageServiceFactory.java @@ -2,16 +2,13 @@ package com.linkedin.gms.factory.lineage; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.LineageService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class LineageServiceFactory { @Bean(name = "lineageService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ownership/OwnershipTypeServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ownership/OwnershipTypeServiceFactory.java index dd2efaf2fb..10df7639c0 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ownership/OwnershipTypeServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ownership/OwnershipTypeServiceFactory.java @@ -2,15 +2,12 @@ package com.linkedin.gms.factory.ownership; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.OwnershipTypeService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class OwnershipTypeServiceFactory { @Bean(name = "ownerShipTypeService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/query/QueryServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/query/QueryServiceFactory.java index 49c3c69d22..266173035d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/query/QueryServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/query/QueryServiceFactory.java @@ -2,15 +2,12 @@ package com.linkedin.gms.factory.query; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.QueryService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class QueryServiceFactory { @Bean(name = "queryService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java index 742ed685fd..e4660bc530 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java @@ -1,8 +1,6 @@ package com.linkedin.gms.factory.recommendation.candidatesource; import com.linkedin.gms.factory.common.IndexConventionFactory; -import com.linkedin.gms.factory.common.RestHighLevelClientFactory; -import com.linkedin.gms.factory.entity.EntityServiceFactory; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.recommendation.candidatesource.RecentlyViewedSource; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; @@ -12,14 +10,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; @Configuration -@Import({ - RestHighLevelClientFactory.class, - IndexConventionFactory.class, - EntityServiceFactory.class -}) public class RecentlyViewedCandidateSourceFactory { @Autowired @Qualifier("elasticSearchRestHighLevelClient") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/BaseElasticSearchComponentsFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/BaseElasticSearchComponentsFactory.java index e4e7d04e31..3fb69d12c8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/BaseElasticSearchComponentsFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/BaseElasticSearchComponentsFactory.java @@ -4,7 +4,6 @@ import com.linkedin.gms.factory.common.IndexConventionFactory; import com.linkedin.gms.factory.common.RestHighLevelClientFactory; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import javax.annotation.Nonnull; import org.opensearch.client.RestHighLevelClient; @@ -14,7 +13,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; /** Factory for components required for any services using elasticsearch */ @Configuration @@ -24,7 +22,6 @@ import org.springframework.context.annotation.PropertySource; ElasticSearchBulkProcessorFactory.class, ElasticSearchIndexBuilderFactory.class }) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class BaseElasticSearchComponentsFactory { @lombok.Value public static class BaseElasticSearchComponents { diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingEntitySearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingEntitySearchServiceFactory.java index d2292b215e..f2446b316c 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingEntitySearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingEntitySearchServiceFactory.java @@ -2,7 +2,6 @@ package com.linkedin.gms.factory.search; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.client.CachingEntitySearchService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -11,10 +10,8 @@ import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class CachingEntitySearchServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java index 64b1fcc2f5..2766f7011e 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactory.java @@ -2,7 +2,6 @@ package com.linkedin.gms.factory.search; import com.linkedin.gms.factory.common.RestHighLevelClientFactory; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.opensearch.action.support.WriteRequest; @@ -13,12 +12,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; @Slf4j @Configuration @Import({RestHighLevelClientFactory.class}) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class ElasticSearchBulkProcessorFactory { @Autowired @Qualifier("elasticSearchRestHighLevelClient") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java index 7fde8c4dd5..ef7e8a8c6a 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java @@ -9,7 +9,6 @@ import com.linkedin.gms.factory.common.IndexConventionFactory; import com.linkedin.gms.factory.common.RestHighLevelClientFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.version.GitVersion; import jakarta.annotation.Nonnull; @@ -25,11 +24,9 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; @Configuration @Import({RestHighLevelClientFactory.class, IndexConventionFactory.class, GitVersionFactory.class}) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class ElasticSearchIndexBuilderFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java index 6c6882c8af..7a40c474ac 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java @@ -16,7 +16,6 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder; import com.linkedin.metadata.search.elasticsearch.query.ESBrowseDAO; import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; import com.linkedin.metadata.search.elasticsearch.update.ESWriteDAO; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import java.io.IOException; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -25,11 +24,9 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; @Slf4j @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({EntityRegistryFactory.class, SettingsBuilderFactory.class}) public class ElasticSearchServiceFactory { private static final ObjectMapper YAML_MAPPER = new YAMLMapper(); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/EntityIndexBuildersFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/EntityIndexBuildersFactory.java index 334194b95c..9670d2819c 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/EntityIndexBuildersFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/EntityIndexBuildersFactory.java @@ -3,15 +3,12 @@ package com.linkedin.gms.factory.search; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class EntityIndexBuildersFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/LineageSearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/LineageSearchServiceFactory.java index 0d7d2e9c18..a0df92f6f6 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/LineageSearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/LineageSearchServiceFactory.java @@ -5,18 +5,15 @@ import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.PropertySource; @Configuration @Import({GraphServiceFactory.class}) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class LineageSearchServiceFactory { public static final String LINEAGE_SEARCH_SERVICE_CACHE_NAME = "relationshipSearchService"; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchDocumentTransformerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchDocumentTransformerFactory.java index 9d9018bd31..bb9d763a7d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchDocumentTransformerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchDocumentTransformerFactory.java @@ -1,14 +1,11 @@ package com.linkedin.gms.factory.search; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class SearchDocumentTransformerFactory { @Value("${elasticsearch.index.maxArrayLength}") private int maxArrayLength; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchServiceFactory.java index 1cb905665e..b17d12c719 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchServiceFactory.java @@ -7,17 +7,14 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.cache.EntityDocCountCache; import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.search.ranker.SearchRanker; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class SearchServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SettingsBuilderFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SettingsBuilderFactory.java index ce1d6f12c5..bdfaf0cc4e 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SettingsBuilderFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SettingsBuilderFactory.java @@ -3,18 +3,15 @@ package com.linkedin.gms.factory.search; import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; @Configuration @Import(EntityRegistryFactory.class) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class SettingsBuilderFactory { @Autowired @Qualifier("entityRegistry") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java index 7fdffa5ad5..d9423bec70 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java @@ -2,15 +2,12 @@ package com.linkedin.gms.factory.search.views; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.ViewService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class ViewServiceFactory { @Bean(name = "viewService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java index e31377bb7d..08d90289a3 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java @@ -2,15 +2,12 @@ package com.linkedin.gms.factory.settings; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.SettingsService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class SettingsServiceFactory { @Bean(name = "settingsService") @Scope("singleton") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/MixpanelApiFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/MixpanelApiFactory.java index b9330d5827..854a64c007 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/MixpanelApiFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/MixpanelApiFactory.java @@ -1,15 +1,12 @@ package com.linkedin.gms.factory.telemetry; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.mixpanel.mixpanelapi.MixpanelAPI; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class MixpanelApiFactory { private static final String EVENTS_ENDPOINT = "https://track.datahubproject.io/mp/track"; private static final String PEOPLE_ENDPOINT = "https://track.datahubproject.io/mp/engage"; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/MixpanelMessageBuilderFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/MixpanelMessageBuilderFactory.java index f647665344..1aba93eddc 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/MixpanelMessageBuilderFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/MixpanelMessageBuilderFactory.java @@ -1,15 +1,12 @@ package com.linkedin.gms.factory.telemetry; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.mixpanel.mixpanelapi.MessageBuilder; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class MixpanelMessageBuilderFactory { private static final String MIXPANEL_TOKEN = "5ee83d940754d63cacbf7d34daa6f44a"; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TrackingServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TrackingServiceFactory.java index 1d844e5c5d..e6d6790def 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TrackingServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TrackingServiceFactory.java @@ -2,7 +2,6 @@ package com.linkedin.gms.factory.telemetry; import com.datahub.telemetry.TrackingService; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.version.GitVersion; import com.mixpanel.mixpanelapi.MessageBuilder; import com.mixpanel.mixpanelapi.MixpanelAPI; @@ -12,11 +11,9 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class TrackingServiceFactory { @Autowired(required = false) @Qualifier("mixpanelApi") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/TimelineServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/TimelineServiceFactory.java index bc121da4e4..ddafc84489 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/TimelineServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/TimelineServiceFactory.java @@ -2,7 +2,6 @@ package com.linkedin.gms.factory.timeline; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeline.TimelineServiceImpl; import javax.annotation.Nonnull; @@ -10,10 +9,8 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class TimelineServiceFactory { @Bean(name = "timelineService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeseries/ElasticSearchTimeseriesAspectServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeseries/ElasticSearchTimeseriesAspectServiceFactory.java index 7739ceecf5..c68db9c3d3 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeseries/ElasticSearchTimeseriesAspectServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeseries/ElasticSearchTimeseriesAspectServiceFactory.java @@ -3,7 +3,6 @@ package com.linkedin.gms.factory.timeseries; import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.metadata.timeseries.elastic.ElasticSearchTimeseriesAspectService; import com.linkedin.metadata.timeseries.elastic.indexbuilder.TimeseriesAspectIndexBuilders; import javax.annotation.Nonnull; @@ -12,10 +11,8 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({BaseElasticSearchComponentsFactory.class, EntityRegistryFactory.class}) public class ElasticSearchTimeseriesAspectServiceFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/usage/UsageClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/usage/UsageClientFactory.java index 9c0519610e..83e14992c9 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/usage/UsageClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/usage/UsageClientFactory.java @@ -2,7 +2,6 @@ package com.linkedin.gms.factory.usage; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.restli.DefaultRestliClientFactory; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.parseq.retry.backoff.ExponentialBackoff; import com.linkedin.r2.transport.http.client.HttpClientFactory; import com.linkedin.restli.client.Client; @@ -14,10 +13,8 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Configuration -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class UsageClientFactory { @Value("${DATAHUB_GMS_HOST:localhost}") diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestRetentionPoliciesStepFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestRetentionPoliciesStepFactory.java index f13037c1e2..56a857ee45 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestRetentionPoliciesStepFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestRetentionPoliciesStepFactory.java @@ -4,7 +4,6 @@ import com.linkedin.gms.factory.entity.RetentionServiceFactory; import com.linkedin.metadata.boot.steps.IngestRetentionPoliciesStep; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.RetentionService; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -12,12 +11,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration @Import({RetentionServiceFactory.class}) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class IngestRetentionPoliciesStepFactory { @Autowired diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockSystemUpdateDeserializer.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockSystemUpdateDeserializer.java index 74a20cdacb..3d0980475d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockSystemUpdateDeserializer.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockSystemUpdateDeserializer.java @@ -3,6 +3,7 @@ package com.linkedin.metadata.boot.kafka; import static com.linkedin.gms.factory.kafka.schemaregistry.SystemUpdateSchemaRegistryFactory.DUHE_SCHEMA_REGISTRY_TOPIC_KEY; import static com.linkedin.gms.factory.kafka.schemaregistry.SystemUpdateSchemaRegistryFactory.SYSTEM_UPDATE_TOPIC_KEY_ID_SUFFIX; import static com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer.topicToSubjectName; +import static io.datahubproject.openapi.schema.registry.Constants.FIXED_SCHEMA_VERSION; import com.linkedin.metadata.EventUtils; import io.confluent.kafka.schemaregistry.ParsedSchema; @@ -42,7 +43,7 @@ public class MockSystemUpdateDeserializer extends KafkaAvroDeserializer { schemaRegistry.register( topicToSubjectName(topicName), new AvroSchema(EventUtils.ORIGINAL_DUHE_AVRO_SCHEMA), - 0, + FIXED_SCHEMA_VERSION, schemaId); return schemaRegistry; } catch (IOException | RestClientException e) { diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockSystemUpdateSerializer.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockSystemUpdateSerializer.java index 14aac2758a..8699695fff 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockSystemUpdateSerializer.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/kafka/MockSystemUpdateSerializer.java @@ -4,11 +4,14 @@ import static com.linkedin.gms.factory.kafka.schemaregistry.SystemUpdateSchemaRe import static com.linkedin.gms.factory.kafka.schemaregistry.SystemUpdateSchemaRegistryFactory.MCL_VERSIONED_SCHEMA_REGISTRY_TOPIC_KEY; import static com.linkedin.gms.factory.kafka.schemaregistry.SystemUpdateSchemaRegistryFactory.SYSTEM_UPDATE_TOPIC_KEY_ID_SUFFIX; import static com.linkedin.gms.factory.kafka.schemaregistry.SystemUpdateSchemaRegistryFactory.SYSTEM_UPDATE_TOPIC_KEY_PREFIX; +import static io.datahubproject.openapi.schema.registry.Constants.FIXED_SCHEMA_VERSION; +import com.google.common.annotations.VisibleForTesting; import com.linkedin.metadata.EventUtils; import com.linkedin.util.Pair; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import io.confluent.kafka.serializers.KafkaAvroSerializer; import java.io.IOException; @@ -26,7 +29,7 @@ public class MockSystemUpdateSerializer extends KafkaAvroSerializer { Map.of( DUHE_SCHEMA_REGISTRY_TOPIC_KEY, new AvroSchema(EventUtils.ORIGINAL_DUHE_AVRO_SCHEMA), MCL_VERSIONED_SCHEMA_REGISTRY_TOPIC_KEY, - new AvroSchema(EventUtils.ORIGINAL_MCL_AVRO_SCHEMA)); + new AvroSchema(EventUtils.RENAMED_MCL_AVRO_SCHEMA)); private Map> topicNameToAvroSchemaMap; @@ -60,7 +63,10 @@ public class MockSystemUpdateSerializer extends KafkaAvroSerializer { (topicName, schemaId) -> { try { schemaRegistry.register( - topicToSubjectName(topicName), schemaId.getFirst(), 0, schemaId.getSecond()); + topicToSubjectName(topicName), + schemaId.getFirst(), + FIXED_SCHEMA_VERSION, + schemaId.getSecond()); } catch (IOException | RestClientException e) { throw new RuntimeException(e); } @@ -70,6 +76,11 @@ public class MockSystemUpdateSerializer extends KafkaAvroSerializer { return schemaRegistry; } + @VisibleForTesting + public SchemaRegistryClient getSchemaRegistryClient() { + return schemaRegistry; + } + public static String topicToSubjectName(String topicName) { return topicName + DATAHUB_SYSTEM_UPDATE_SUBJECT_SUFFIX; } diff --git a/metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java b/metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java index 18546369c9..7702e8bd6d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java +++ b/metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java @@ -5,11 +5,15 @@ import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.HttpRequestHandler; import org.springframework.web.context.support.HttpRequestHandlerServlet; +@AllArgsConstructor +@NoArgsConstructor @Component public class RestliHandlerServlet extends HttpRequestHandlerServlet implements HttpRequestHandler { @Autowired private RAPJakartaServlet _r2Servlet; diff --git a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactoryTest.java b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactoryTest.java index f910f7981b..c8ee7b25f3 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactoryTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchBulkProcessorFactoryTest.java @@ -13,7 +13,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; -@TestPropertySource(locations = "classpath:/application.yml") +@TestPropertySource(locations = "classpath:/application.yaml") @SpringBootTest(classes = {ElasticSearchBulkProcessorFactory.class}) @EnableConfigurationProperties(ConfigurationProvider.class) public class ElasticSearchBulkProcessorFactoryTest extends AbstractTestNGSpringContextTests { diff --git a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryDefaultsTest.java b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryDefaultsTest.java index 87f1546bd9..62fb43d6a9 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryDefaultsTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryDefaultsTest.java @@ -13,7 +13,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; -@TestPropertySource(locations = "classpath:/application.yml") +@TestPropertySource(locations = "classpath:/application.yaml") @SpringBootTest(classes = {ElasticSearchIndexBuilderFactory.class}) @EnableConfigurationProperties(ConfigurationProvider.class) public class ElasticSearchIndexBuilderFactoryDefaultsTest extends AbstractTestNGSpringContextTests { diff --git a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryTest.java b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryTest.java index 2c309cb44b..d13d330a08 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryTest.java @@ -13,7 +13,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; -@TestPropertySource(locations = "classpath:/application.yml") +@TestPropertySource(locations = "classpath:/application.yaml") @SpringBootTest(classes = {ElasticSearchIndexBuilderFactory.class}) @EnableConfigurationProperties(ConfigurationProvider.class) public class ElasticSearchIndexBuilderFactoryTest extends AbstractTestNGSpringContextTests { diff --git a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/secret/SecretServiceFactoryTest.java b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/secret/SecretServiceFactoryTest.java index e3d8522155..e8ee3b998b 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/secret/SecretServiceFactoryTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/secret/SecretServiceFactoryTest.java @@ -16,7 +16,7 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; -@TestPropertySource(locations = "classpath:/application.yml") +@TestPropertySource(locations = "classpath:/application.yaml") @SpringBootTest(classes = {SecretServiceFactory.class}) @EnableConfigurationProperties(ConfigurationProvider.class) public class SecretServiceFactoryTest extends AbstractTestNGSpringContextTests { diff --git a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java index 12596d9410..f78c8597f9 100644 --- a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java +++ b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/util/OpenApiEntitiesUtilTest.java @@ -5,7 +5,6 @@ import static org.testng.AssertJUnit.assertNotNull; import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.spring.YamlPropertySourceFactory; import io.datahubproject.openapi.config.OpenAPIEntityTestConfiguration; import io.datahubproject.openapi.dto.UpsertAspectRequest; import io.datahubproject.openapi.generated.ContainerEntityRequestV2; @@ -14,13 +13,11 @@ import io.datahubproject.openapi.generated.ContainerKeyAspectRequestV2; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.PropertySource; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @Import({OpenAPIEntityTestConfiguration.class}) -@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class OpenApiEntitiesUtilTest extends AbstractTestNGSpringContextTests { @Autowired private EntityRegistry entityRegistry; diff --git a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImplTest.java b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImplTest.java index 2d31e35c48..2beb210e5b 100644 --- a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImplTest.java +++ b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImplTest.java @@ -47,12 +47,14 @@ import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @SpringBootTest(classes = {SpringWebConfig.class}) @ComponentScan(basePackages = {"io.datahubproject.openapi.v2.generated.controller"}) @Import({OpenAPIEntityTestConfiguration.class}) +@EnableWebMvc @AutoConfigureMockMvc public class EntityApiDelegateImplTest extends AbstractTestNGSpringContextTests { @BeforeTest diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java index 01671ccea0..9ea0164843 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java @@ -16,10 +16,8 @@ import org.springframework.http.converter.FormHttpMessageConverter; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; -import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -@EnableWebMvc @OpenAPIDefinition( info = @Info(title = "DataHub OpenAPI", version = "2.0.0"), servers = {@Server(url = "/openapi/", description = "Default Server URL")}) diff --git a/metadata-service/schema-registry-api/build.gradle b/metadata-service/schema-registry-api/build.gradle index 86e7635c7a..201e98beb4 100644 --- a/metadata-service/schema-registry-api/build.gradle +++ b/metadata-service/schema-registry-api/build.gradle @@ -1,13 +1,12 @@ plugins { id 'org.hidetake.swagger.generator' - id 'java' + id 'java-library' } dependencies { // Dependencies for open api implementation externalDependency.reflections - implementation externalDependency.springBoot implementation externalDependency.springCore implementation externalDependency.springWeb implementation externalDependency.springWebMVC @@ -15,9 +14,9 @@ dependencies { implementation externalDependency.springContext implementation externalDependency.antlr4Runtime implementation externalDependency.antlr4 - implementation externalDependency.springBootStarterValidation implementation externalDependency.servletApi - implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' + implementation externalDependency.jakartaValidation + implementation externalDependency.jakartaAnnotationApi implementation externalDependency.jacksonDataBind implementation externalDependency.slf4jApi diff --git a/metadata-service/schema-registry-api/src/main/java/io/datahubproject/openapi/schema/registry/Constants.java b/metadata-service/schema-registry-api/src/main/java/io/datahubproject/openapi/schema/registry/Constants.java new file mode 100644 index 0000000000..b8cc87ae69 --- /dev/null +++ b/metadata-service/schema-registry-api/src/main/java/io/datahubproject/openapi/schema/registry/Constants.java @@ -0,0 +1,7 @@ +package io.datahubproject.openapi.schema.registry; + +public class Constants { + private Constants() {} + + public static final int FIXED_SCHEMA_VERSION = 1; +} diff --git a/metadata-service/schema-registry-servlet/build.gradle b/metadata-service/schema-registry-servlet/build.gradle index dd004a45a7..f19237c32c 100644 --- a/metadata-service/schema-registry-servlet/build.gradle +++ b/metadata-service/schema-registry-servlet/build.gradle @@ -27,6 +27,7 @@ dependencies { testImplementation externalDependency.springBootTest testImplementation project(':mock-entity-registry') + testImplementation project(':metadata-dao-impl:kafka-producer') testImplementation externalDependency.springBoot testImplementation externalDependency.testContainers testImplementation externalDependency.testContainersKafka diff --git a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java index f4228e0d63..b9b4dd1ec0 100644 --- a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java +++ b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/SchemaRegistryController.java @@ -59,8 +59,8 @@ public class SchemaRegistryController private final ObjectMapper objectMapper; private final HttpServletRequest request; - - private static final Set SCHEMA_VERSIONS = ImmutableSet.of("1", "latest"); + private static final Set SCHEMA_VERSIONS = + ImmutableSet.of(String.valueOf(Constants.FIXED_SCHEMA_VERSION), "latest"); @Qualifier("schemaRegistryService") private final SchemaRegistryService _schemaRegistryService; @@ -128,7 +128,7 @@ public class SchemaRegistryController schema -> { Schema result = new Schema(); result.setSubject(subject); - result.setVersion(1); + result.setVersion(Constants.FIXED_SCHEMA_VERSION); result.setId(_schemaRegistryService.getSchemaIdForTopic(topicName).get()); result.setSchema(schema.toString()); return new ResponseEntity<>(result, HttpStatus.OK); @@ -161,7 +161,8 @@ public class SchemaRegistryController .getSchemaForTopic(topicName) .map( schema -> { - return new ResponseEntity<>(Arrays.asList(1), HttpStatus.OK); + return new ResponseEntity<>( + Arrays.asList(Constants.FIXED_SCHEMA_VERSION), HttpStatus.OK); }) .orElseGet( () -> { diff --git a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java index 98163a7d91..5f7d8d9604 100644 --- a/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java +++ b/metadata-service/schema-registry-servlet/src/main/java/io/datahubproject/openapi/schema/registry/config/SpringWebSchemaRegistryConfig.java @@ -10,10 +10,8 @@ import org.springframework.http.converter.FormHttpMessageConverter; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; -import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -@EnableWebMvc @OpenAPIDefinition( info = @Info(title = "DataHub OpenAPI", version = "1.0.0"), servers = {@Server(url = "/schema-registry/", description = "Schema Registry Server URL")}) diff --git a/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTestConfiguration.java b/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTestConfiguration.java index ff5b951092..7ab673b0a4 100644 --- a/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTestConfiguration.java +++ b/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTestConfiguration.java @@ -1,10 +1,14 @@ package io.datahubproject.openapi.test; +import com.linkedin.metadata.dao.producer.KafkaHealthChecker; import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.ComponentScan; import org.springframework.test.context.TestPropertySource; @TestConfiguration @TestPropertySource(value = "classpath:/application.properties") @ComponentScan(basePackages = {"com.linkedin.gms.factory.kafka", "com.linkedin.gms.factory.config"}) -public class SchemaRegistryControllerTestConfiguration {} +public class SchemaRegistryControllerTestConfiguration { + @MockBean KafkaHealthChecker kafkaHealthChecker; +} diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle index d6ef6b603a..80265e8c63 100644 --- a/metadata-service/war/build.gradle +++ b/metadata-service/war/build.gradle @@ -12,15 +12,15 @@ ext { ext.apiProject = project(':metadata-service:restli-api') dependencies { - runtimeOnly project(':metadata-service:factories') - runtimeOnly project(':metadata-service:auth-filter') - runtimeOnly project(':metadata-service:servlet') - runtimeOnly project(':metadata-service:auth-servlet-impl') - runtimeOnly project(':metadata-service:graphql-servlet-impl') - runtimeOnly project(':metadata-service:openapi-servlet') - runtimeOnly project(':metadata-service:openapi-entity-servlet') - runtimeOnly project(':metadata-service:openapi-analytics-servlet') - runtimeOnly project(':metadata-service:schema-registry-servlet') + implementation project(':metadata-service:factories') + implementation project(':metadata-service:auth-filter') + implementation project(':metadata-service:servlet') + implementation project(':metadata-service:auth-servlet-impl') + implementation project(':metadata-service:graphql-servlet-impl') + implementation project(':metadata-service:openapi-servlet') + implementation project(':metadata-service:openapi-entity-servlet') + implementation project(':metadata-service:openapi-analytics-servlet') + implementation project(':metadata-service:schema-registry-servlet') runtimeOnly project(':metadata-jobs:mce-consumer') runtimeOnly project(':metadata-jobs:mae-consumer') runtimeOnly project(':metadata-jobs:pe-consumer') @@ -30,10 +30,12 @@ dependencies { runtimeOnly externalDependency.mariadbConnector runtimeOnly externalDependency.mysqlConnector runtimeOnly externalDependency.postgresql - runtimeOnly externalDependency.springWebMVC + implementation externalDependency.springWebMVC + implementation externalDependency.springBootAutoconfigure + implementation externalDependency.servletApi - runtimeOnly spec.product.pegasus.restliDocgen - runtimeOnly spec.product.pegasus.restliSpringBridge + implementation spec.product.pegasus.restliDocgen + implementation spec.product.pegasus.restliSpringBridge runtimeOnly externalDependency.log4jCore runtimeOnly externalDependency.log4j2Api diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/CommonApplicationConfig.java b/metadata-service/war/src/main/java/com/linkedin/gms/CommonApplicationConfig.java new file mode 100644 index 0000000000..5d956c2c8f --- /dev/null +++ b/metadata-service/war/src/main/java/com/linkedin/gms/CommonApplicationConfig.java @@ -0,0 +1,47 @@ +package com.linkedin.gms; + +import com.datahub.auth.authentication.filter.AuthenticationFilter; +import com.linkedin.metadata.spring.YamlPropertySourceFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + +/** + * Common configuration for all servlets. Generally this list also includes dependencies of the + * embedded MAE/MCE consumers. + */ +@ComponentScan( + basePackages = { + "com.linkedin.metadata.boot", + "com.linkedin.gms.factory.config", + "com.linkedin.gms.factory.entityregistry", + "com.linkedin.gms.factory.common", + "com.linkedin.gms.factory.entity", + "com.linkedin.gms.factory.kafka", + "com.linkedin.gms.factory.kafka.common", + "com.linkedin.gms.factory.kafka.schemaregistry", + "com.linkedin.metadata.boot.kafka", + "com.linkedin.metadata.kafka", + "com.linkedin.metadata.dao.producer", + "com.linkedin.gms.factory.entity.update.indices", + "com.linkedin.gms.factory.entityclient", + "com.linkedin.gms.factory.form", + "com.linkedin.gms.factory.incident", + "com.linkedin.gms.factory.timeline.eventgenerator", + "io.datahubproject.metadata.jobs.common.health.kafka", + "com.linkedin.gms.factory.context", + "com.linkedin.gms.factory.auth", + "com.linkedin.gms.factory.search", + "com.linkedin.gms.factory.secret", + "com.linkedin.gms.factory.timeseries" + }) +@PropertySource(value = "classpath:/application.yaml", factory = YamlPropertySourceFactory.class) +@Configuration +public class CommonApplicationConfig { + + @Bean("authenticationFilter") + public AuthenticationFilter authenticationFilter() { + return new AuthenticationFilter(); + } +} diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/SpringApplicationInitializer.java b/metadata-service/war/src/main/java/com/linkedin/gms/SpringApplicationInitializer.java new file mode 100644 index 0000000000..821d1bf0ad --- /dev/null +++ b/metadata-service/war/src/main/java/com/linkedin/gms/SpringApplicationInitializer.java @@ -0,0 +1,11 @@ +package com.linkedin.gms; + +import javax.annotation.Nonnull; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; + +public class SpringApplicationInitializer + implements ApplicationContextInitializer { + @Override + public void initialize(@Nonnull ConfigurableApplicationContext applicationContext) {} +} diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java b/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java new file mode 100644 index 0000000000..c2a7e99c94 --- /dev/null +++ b/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java @@ -0,0 +1,162 @@ +package com.linkedin.gms; + +import com.datahub.auth.authentication.filter.AuthenticationFilter; +import com.datahub.gms.servlet.Config; +import com.datahub.gms.servlet.ConfigSearchExport; +import com.datahub.gms.servlet.HealthCheck; +import com.linkedin.gms.servlet.AuthServletConfig; +import com.linkedin.gms.servlet.GraphQLServletConfig; +import com.linkedin.gms.servlet.OpenAPIServletConfig; +import com.linkedin.gms.servlet.RestliServletConfig; +import com.linkedin.gms.servlet.SchemaRegistryServletConfig; +import jakarta.servlet.DispatcherType; +import jakarta.servlet.FilterRegistration; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletRegistration; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import org.springframework.web.context.ContextLoaderListener; +import org.springframework.web.context.support.AnnotationConfigWebApplicationContext; +import org.springframework.web.context.support.HttpRequestHandlerServlet; +import org.springframework.web.servlet.DispatcherServlet; + +/** This class is before Spring Context is loaded, previously web.xml based */ +public class WebApplicationInitializer + implements org.springframework.web.WebApplicationInitializer { + @Override + public void onStartup(ServletContext container) { + AnnotationConfigWebApplicationContext rootContext = new AnnotationConfigWebApplicationContext(); + rootContext.register(CommonApplicationConfig.class); + + ContextLoaderListener contextLoaderListener = new ContextLoaderListener(rootContext); + container.addListener(contextLoaderListener); + container.setInitParameter( + "contextInitializerClasses", "com.linkedin.gms.SpringApplicationInitializer"); + + // Auth filter + + // Independent dispatcher + schemaRegistryServlet(container); + + // Non-Spring servlets + healthCheckServlet(container); + configServlet(container); + + List servletNames = new ArrayList<>(); + + // Restli non-Dispatcher + servletNames.add(restliServlet(rootContext, container)); + + // Spring Dispatcher servlets + DispatcherServlet dispatcherServlet = new DispatcherServlet(rootContext); + servletNames.add(authServlet(rootContext, dispatcherServlet, container)); + servletNames.add(graphQLServlet(rootContext, dispatcherServlet, container)); + servletNames.add(openAPIServlet(rootContext, dispatcherServlet, container)); + + FilterRegistration.Dynamic filterRegistration = + container.addFilter("authenticationFilter", AuthenticationFilter.class); + filterRegistration.setAsyncSupported(true); + filterRegistration.addMappingForServletNames( + EnumSet.of(DispatcherType.ASYNC, DispatcherType.REQUEST), + false, + servletNames.toArray(String[]::new)); + } + + /* + * This is a servlet exclusive to DataHub's implementation of the Confluent OpenAPI spec which is not behind + * DataHub's authentication layer as it is not compatible with confluent consumers & producers. + */ + private void schemaRegistryServlet(ServletContext container) { + AnnotationConfigWebApplicationContext webContext = new AnnotationConfigWebApplicationContext(); + webContext.register(SchemaRegistryServletConfig.class); + + DispatcherServlet dispatcherServlet = new DispatcherServlet(webContext); + ServletRegistration.Dynamic registration = + container.addServlet("dispatcher-schema-registry", dispatcherServlet); + registration.addMapping("/schema-registry/*"); + registration.setLoadOnStartup(1); + registration.setAsyncSupported(true); + } + + private String authServlet( + AnnotationConfigWebApplicationContext rootContext, + DispatcherServlet dispatcherServlet, + ServletContext container) { + final String servletName = "dispatcher-auth"; + rootContext.register(AuthServletConfig.class); + + ServletRegistration.Dynamic registration = container.addServlet(servletName, dispatcherServlet); + registration.addMapping("/auth/*"); + registration.setLoadOnStartup(5); + registration.setAsyncSupported(true); + + return servletName; + } + + private String graphQLServlet( + AnnotationConfigWebApplicationContext rootContext, + DispatcherServlet dispatcherServlet, + ServletContext container) { + final String servletName = "dispatcher-graphql"; + rootContext.register(GraphQLServletConfig.class); + + ServletRegistration.Dynamic registration = container.addServlet(servletName, dispatcherServlet); + registration.addMapping("/api/*"); + registration.setLoadOnStartup(5); + registration.setAsyncSupported(true); + + return servletName; + } + + private String openAPIServlet( + AnnotationConfigWebApplicationContext rootContext, + DispatcherServlet dispatcherServlet, + ServletContext container) { + final String servletName = "dispatcher-openapi"; + rootContext.register(OpenAPIServletConfig.class); + + ServletRegistration.Dynamic registration = container.addServlet(servletName, dispatcherServlet); + registration.addMapping("/openapi/*"); + registration.setLoadOnStartup(5); + registration.setAsyncSupported(true); + + return servletName; + } + + private String restliServlet( + AnnotationConfigWebApplicationContext rootContext, ServletContext container) { + final String servletName = "restliRequestHandler"; + + rootContext.register(RestliServletConfig.class); + + ServletRegistration.Dynamic registration = + container.addServlet(servletName, new HttpRequestHandlerServlet()); + registration.addMapping("/*"); + registration.setLoadOnStartup(10); + registration.setAsyncSupported(true); + + return servletName; + } + + private void healthCheckServlet(ServletContext container) { + ServletRegistration.Dynamic registration = + container.addServlet("healthCheck", new HealthCheck()); + registration.addMapping("/health"); + registration.setLoadOnStartup(15); + registration.setAsyncSupported(true); + } + + private void configServlet(ServletContext container) { + ServletRegistration.Dynamic registration = container.addServlet("config", new Config()); + registration.addMapping("/config"); + registration.setLoadOnStartup(15); + registration.setAsyncSupported(true); + + ServletRegistration.Dynamic registration2 = + container.addServlet("config-search-export", new ConfigSearchExport()); + registration2.addMapping("/config/search/export"); + registration2.setLoadOnStartup(15); + registration2.setAsyncSupported(true); + } +} diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/servlet/AuthServletConfig.java b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/AuthServletConfig.java new file mode 100644 index 0000000000..437c8722a7 --- /dev/null +++ b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/AuthServletConfig.java @@ -0,0 +1,10 @@ +package com.linkedin.gms.servlet; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; + +@EnableWebMvc +@ComponentScan(basePackages = {"com.datahub.auth.authentication"}) +@Configuration +public class AuthServletConfig {} diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/servlet/GraphQLServletConfig.java b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/GraphQLServletConfig.java new file mode 100644 index 0000000000..64ec11f58c --- /dev/null +++ b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/GraphQLServletConfig.java @@ -0,0 +1,24 @@ +package com.linkedin.gms.servlet; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; + +@EnableWebMvc +@ComponentScan( + basePackages = { + "com.datahub.graphql", + "com.linkedin.gms.factory.graphql", + "com.linkedin.gms.factory.timeline", + "com.linkedin.gms.factory.usage", + "com.linkedin.gms.factory.recommendation", + "com.linkedin.gms.factory.ownership", + "com.linkedin.gms.factory.settings", + "com.linkedin.gms.factory.lineage", + "com.linkedin.gms.factory.query", + "com.linkedin.gms.factory.ermodelrelation", + "com.linkedin.gms.factory.dataproduct", + "com.linkedin.gms.factory.businessattribute" + }) +@Configuration +public class GraphQLServletConfig {} diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/servlet/OpenAPIServletConfig.java b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/OpenAPIServletConfig.java new file mode 100644 index 0000000000..32e8bfd7f8 --- /dev/null +++ b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/OpenAPIServletConfig.java @@ -0,0 +1,22 @@ +package com.linkedin.gms.servlet; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; + +@EnableWebMvc +@ComponentScan( + basePackages = { + "io.datahubproject.openapi.config", + "io.datahubproject.openapi.health", + "io.datahubproject.openapi.openlineage", + "io.datahubproject.openapi.operations", + "io.datahubproject.openapi.platform", + "io.datahubproject.openapi.relationships", + "io.datahubproject.openapi.timeline", + "io.datahubproject.openapi.entities", + "io.datahubproject.openapi.v2", + "com.linkedin.gms.factory.timeline" + }) +@Configuration +public class OpenAPIServletConfig {} diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java new file mode 100644 index 0000000000..28e2aabe13 --- /dev/null +++ b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java @@ -0,0 +1,20 @@ +package com.linkedin.gms.servlet; + +import com.linkedin.metadata.spring.YamlPropertySourceFactory; +import com.linkedin.r2.transport.http.server.RAPJakartaServlet; +import com.linkedin.restli.server.RestliHandlerServlet; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.web.HttpRequestHandler; + +@ComponentScan(basePackages = {"com.linkedin.restli.server"}) +@PropertySource(value = "classpath:/application.yaml", factory = YamlPropertySourceFactory.class) +@Configuration +public class RestliServletConfig { + @Bean("restliRequestHandler") + public HttpRequestHandler restliHandlerServlet(final RAPJakartaServlet r2Servlet) { + return new RestliHandlerServlet(r2Servlet); + } +} diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/servlet/SchemaRegistryServletConfig.java b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/SchemaRegistryServletConfig.java new file mode 100644 index 0000000000..beb633888b --- /dev/null +++ b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/SchemaRegistryServletConfig.java @@ -0,0 +1,10 @@ +package com.linkedin.gms.servlet; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; + +@EnableWebMvc +@ComponentScan(basePackages = {"io.datahubproject.openapi.schema.registry"}) +@Configuration +public class SchemaRegistryServletConfig {} diff --git a/metadata-service/war/src/main/webapp/WEB-INF/apiServlet-servlet.xml b/metadata-service/war/src/main/webapp/WEB-INF/apiServlet-servlet.xml deleted file mode 100644 index 7547eb8f73..0000000000 --- a/metadata-service/war/src/main/webapp/WEB-INF/apiServlet-servlet.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/metadata-service/war/src/main/webapp/WEB-INF/authServlet-servlet.xml b/metadata-service/war/src/main/webapp/WEB-INF/authServlet-servlet.xml deleted file mode 100644 index 320cba0682..0000000000 --- a/metadata-service/war/src/main/webapp/WEB-INF/authServlet-servlet.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/metadata-service/war/src/main/webapp/WEB-INF/beans.xml b/metadata-service/war/src/main/webapp/WEB-INF/beans.xml deleted file mode 100644 index 10aa36263f..0000000000 --- a/metadata-service/war/src/main/webapp/WEB-INF/beans.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metadata-service/war/src/main/webapp/WEB-INF/openapiServlet-servlet.xml b/metadata-service/war/src/main/webapp/WEB-INF/openapiServlet-servlet.xml deleted file mode 100644 index fb2bc6c033..0000000000 --- a/metadata-service/war/src/main/webapp/WEB-INF/openapiServlet-servlet.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/metadata-service/war/src/main/webapp/WEB-INF/schemaRegistryServlet-servlet.xml b/metadata-service/war/src/main/webapp/WEB-INF/schemaRegistryServlet-servlet.xml deleted file mode 100644 index eebe845f10..0000000000 --- a/metadata-service/war/src/main/webapp/WEB-INF/schemaRegistryServlet-servlet.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/metadata-service/war/src/main/webapp/WEB-INF/web.xml b/metadata-service/war/src/main/webapp/WEB-INF/web.xml deleted file mode 100644 index c1239ed4b7..0000000000 --- a/metadata-service/war/src/main/webapp/WEB-INF/web.xml +++ /dev/null @@ -1,156 +0,0 @@ - - - - Open source GMS - Open source generalized metadata store (GMS) service - - - - - - - contextConfigLocation - /WEB-INF/beans.xml - - - - - - - org.springframework.web.context.ContextLoaderListener - - - - - Health Check Servlet - healthCheck - com.datahub.gms.servlet.HealthCheck - true - - - Config Servlet - config - com.datahub.gms.servlet.Config - true - - - Search Config CSV Servlet - config-search-export - com.datahub.gms.servlet.ConfigSearchExport - true - - - apiServlet - org.springframework.web.servlet.DispatcherServlet - 1 - true - - - authServlet - org.springframework.web.servlet.DispatcherServlet - 1 - true - - - openapiServlet - org.springframework.web.servlet.DispatcherServlet - 1 - true - - - - - schemaRegistryServlet - org.springframework.web.servlet.DispatcherServlet - 1 - true - - - - - Restli Spring Servlet - restliRequestHandler - org.springframework.web.context.support.HttpRequestHandlerServlet - true - - - - - healthCheck - /health - - - openapiServlet - /health/* - - - config - /config - - - config-search-export - /config/search/export - - - apiServlet - /api/* - - - openapiServlet - /openapi/* - - - authServlet - /auth/* - - - schemaRegistryServlet - /schema-registry/* - - - restliRequestHandler - /* - - - - - AuthenticationFilter - com.datahub.auth.authentication.filter.AuthenticationFilter - true - - - - - AuthenticationFilter - apiServlet - REQUEST - - - - - AuthenticationFilter - restliRequestHandler - REQUEST - - - - AuthenticationFilter - authServlet - REQUEST - - - - AuthenticationFilter - openapiServlet - REQUEST - -