2021-08-01 14:27:44 -07:00
|
|
|
/*
|
2021-12-01 12:46:28 +05:30
|
|
|
* Copyright 2021 Collate
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
2021-08-01 14:27:44 -07:00
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2022-09-14 23:14:02 -07:00
|
|
|
package org.openmetadata.service;
|
2021-08-01 14:27:44 -07:00
|
|
|
|
2023-09-08 23:19:44 +05:30
|
|
|
import static org.openmetadata.service.jdbi3.unitofwork.JdbiUnitOfWorkProvider.getWrappedInstanceForDaoClass;
|
2023-03-28 12:51:52 +05:30
|
|
|
import static org.openmetadata.service.util.MicrometerBundleSingleton.webAnalyticEvents;
|
|
|
|
|
2021-10-21 21:13:27 -07:00
|
|
|
import io.dropwizard.Application;
|
2022-01-11 10:13:49 -08:00
|
|
|
import io.dropwizard.configuration.EnvironmentVariableSubstitutor;
|
|
|
|
import io.dropwizard.configuration.SubstitutingSourceProvider;
|
2022-08-25 19:07:44 +05:30
|
|
|
import io.dropwizard.db.DataSourceFactory;
|
2021-08-01 14:27:44 -07:00
|
|
|
import io.dropwizard.health.conf.HealthConfiguration;
|
|
|
|
import io.dropwizard.health.core.HealthCheckBundle;
|
2021-10-20 14:21:24 -07:00
|
|
|
import io.dropwizard.jdbi3.JdbiFactory;
|
2021-08-01 14:27:44 -07:00
|
|
|
import io.dropwizard.jersey.errors.EarlyEofExceptionMapper;
|
|
|
|
import io.dropwizard.jersey.errors.LoggingExceptionMapper;
|
2021-10-21 21:13:27 -07:00
|
|
|
import io.dropwizard.jersey.jackson.JsonProcessingExceptionMapper;
|
2021-08-01 14:27:44 -07:00
|
|
|
import io.dropwizard.lifecycle.Managed;
|
|
|
|
import io.dropwizard.server.DefaultServerFactory;
|
|
|
|
import io.dropwizard.setup.Bootstrap;
|
|
|
|
import io.dropwizard.setup.Environment;
|
|
|
|
import io.federecio.dropwizard.swagger.SwaggerBundle;
|
|
|
|
import io.federecio.dropwizard.swagger.SwaggerBundleConfiguration;
|
2022-06-15 10:14:47 +05:30
|
|
|
import io.socket.engineio.server.EngineIoServerOptions;
|
|
|
|
import io.socket.engineio.server.JettyWebSocketHandler;
|
2022-01-09 21:04:10 -08:00
|
|
|
import java.io.IOException;
|
2021-12-16 07:13:50 -08:00
|
|
|
import java.lang.reflect.InvocationTargetException;
|
2023-03-24 21:56:49 +05:30
|
|
|
import java.security.KeyStoreException;
|
|
|
|
import java.security.NoSuchAlgorithmException;
|
|
|
|
import java.security.cert.CertificateException;
|
2022-01-09 21:04:10 -08:00
|
|
|
import java.time.temporal.ChronoUnit;
|
2022-05-27 03:09:13 -07:00
|
|
|
import java.util.EnumSet;
|
2023-09-08 23:19:44 +05:30
|
|
|
import java.util.HashSet;
|
2023-06-07 01:02:52 +05:30
|
|
|
import java.util.List;
|
2022-02-01 08:31:34 +01:00
|
|
|
import java.util.Optional;
|
2022-10-21 20:49:41 -07:00
|
|
|
import javax.naming.ConfigurationException;
|
2022-05-27 03:09:13 -07:00
|
|
|
import javax.servlet.DispatcherType;
|
|
|
|
import javax.servlet.FilterRegistration;
|
2022-06-15 10:14:47 +05:30
|
|
|
import javax.servlet.ServletException;
|
2023-03-24 21:56:49 +05:30
|
|
|
import javax.servlet.ServletRegistration;
|
2021-12-16 07:13:50 -08:00
|
|
|
import javax.ws.rs.container.ContainerRequestFilter;
|
|
|
|
import javax.ws.rs.container.ContainerResponseFilter;
|
|
|
|
import javax.ws.rs.core.Response;
|
2021-08-01 14:27:44 -07:00
|
|
|
import lombok.SneakyThrows;
|
2022-01-20 02:06:48 +05:30
|
|
|
import lombok.extern.slf4j.Slf4j;
|
2021-08-01 14:27:44 -07:00
|
|
|
import org.apache.commons.lang3.StringUtils;
|
2022-06-15 10:14:47 +05:30
|
|
|
import org.eclipse.jetty.http.pathmap.ServletPathSpec;
|
|
|
|
import org.eclipse.jetty.servlet.FilterHolder;
|
|
|
|
import org.eclipse.jetty.servlet.ServletHolder;
|
2023-04-14 11:31:36 +05:30
|
|
|
import org.eclipse.jetty.websocket.server.NativeWebSocketServletContainerInitializer;
|
2022-06-15 10:14:47 +05:30
|
|
|
import org.eclipse.jetty.websocket.server.WebSocketUpgradeFilter;
|
2021-08-01 14:27:44 -07:00
|
|
|
import org.glassfish.jersey.media.multipart.MultiPartFeature;
|
|
|
|
import org.glassfish.jersey.server.ServerProperties;
|
2021-10-21 21:13:27 -07:00
|
|
|
import org.jdbi.v3.core.Jdbi;
|
2022-01-09 21:04:10 -08:00
|
|
|
import org.jdbi.v3.core.statement.SqlLogger;
|
|
|
|
import org.jdbi.v3.core.statement.StatementContext;
|
2022-04-27 18:58:44 +01:00
|
|
|
import org.jdbi.v3.sqlobject.SqlObjects;
|
2023-05-28 23:44:02 +05:30
|
|
|
import org.openmetadata.schema.api.configuration.extension.Extension;
|
|
|
|
import org.openmetadata.schema.api.configuration.extension.ExtensionConfiguration;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.schema.api.security.AuthenticationConfiguration;
|
|
|
|
import org.openmetadata.schema.api.security.AuthorizerConfiguration;
|
2023-07-26 11:34:58 -07:00
|
|
|
import org.openmetadata.schema.services.connections.metadata.AuthProvider;
|
2023-08-18 17:48:17 +05:30
|
|
|
import org.openmetadata.service.config.OMWebBundle;
|
|
|
|
import org.openmetadata.service.config.OMWebConfiguration;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.events.EventFilter;
|
|
|
|
import org.openmetadata.service.events.EventPubSub;
|
2023-06-22 06:53:56 +02:00
|
|
|
import org.openmetadata.service.events.scheduled.PipelineServiceStatusJobHandler;
|
2023-05-08 16:54:45 +05:30
|
|
|
import org.openmetadata.service.events.scheduled.ReportsHandler;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.exception.CatalogGenericExceptionMapper;
|
|
|
|
import org.openmetadata.service.exception.ConstraintViolationExceptionMapper;
|
|
|
|
import org.openmetadata.service.exception.JsonMappingExceptionMapper;
|
2023-07-13 21:16:05 +05:30
|
|
|
import org.openmetadata.service.exception.OMErrorPageHandler;
|
2023-05-28 23:44:02 +05:30
|
|
|
import org.openmetadata.service.extension.OpenMetadataExtension;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.fernet.Fernet;
|
2022-09-28 22:55:00 +05:30
|
|
|
import org.openmetadata.service.jdbi3.CollectionDAO;
|
2023-08-08 16:42:54 -07:00
|
|
|
import org.openmetadata.service.jdbi3.EntityRepository;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator;
|
2023-08-17 14:39:05 -07:00
|
|
|
import org.openmetadata.service.jdbi3.locator.ConnectionType;
|
2023-09-13 08:58:42 +05:30
|
|
|
import org.openmetadata.service.jdbi3.unitofwork.JdbiTransactionManager;
|
2023-09-08 23:19:44 +05:30
|
|
|
import org.openmetadata.service.jdbi3.unitofwork.JdbiUnitOfWorkApplicationEventListener;
|
|
|
|
import org.openmetadata.service.jdbi3.unitofwork.JdbiUnitOfWorkProvider;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.migration.Migration;
|
2023-08-17 14:39:05 -07:00
|
|
|
import org.openmetadata.service.migration.api.MigrationWorkflow;
|
2022-11-28 17:45:18 +01:00
|
|
|
import org.openmetadata.service.monitoring.EventMonitor;
|
|
|
|
import org.openmetadata.service.monitoring.EventMonitorFactory;
|
|
|
|
import org.openmetadata.service.monitoring.EventMonitorPublisher;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.resources.CollectionRegistry;
|
2023-04-04 17:03:57 +05:30
|
|
|
import org.openmetadata.service.resources.databases.DatasourceConfig;
|
2023-04-05 10:31:51 +05:30
|
|
|
import org.openmetadata.service.resources.settings.SettingsCache;
|
2023-08-09 23:24:10 -07:00
|
|
|
import org.openmetadata.service.search.SearchEventPublisher;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.secrets.SecretsManager;
|
|
|
|
import org.openmetadata.service.secrets.SecretsManagerFactory;
|
2022-11-14 13:48:50 +01:00
|
|
|
import org.openmetadata.service.secrets.SecretsManagerUpdateService;
|
2023-03-03 19:10:01 +01:00
|
|
|
import org.openmetadata.service.secrets.masker.EntityMaskerFactory;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.security.Authorizer;
|
|
|
|
import org.openmetadata.service.security.NoopAuthorizer;
|
|
|
|
import org.openmetadata.service.security.NoopFilter;
|
2022-10-27 22:56:20 +05:30
|
|
|
import org.openmetadata.service.security.auth.AuthenticatorHandler;
|
|
|
|
import org.openmetadata.service.security.auth.BasicAuthenticator;
|
|
|
|
import org.openmetadata.service.security.auth.LdapAuthenticator;
|
|
|
|
import org.openmetadata.service.security.auth.NoopAuthenticator;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.security.jwt.JWTTokenGenerator;
|
2023-03-24 21:56:49 +05:30
|
|
|
import org.openmetadata.service.security.saml.OMMicrometerHttpFilter;
|
|
|
|
import org.openmetadata.service.security.saml.SamlAssertionConsumerServlet;
|
|
|
|
import org.openmetadata.service.security.saml.SamlLoginServlet;
|
|
|
|
import org.openmetadata.service.security.saml.SamlMetadataServlet;
|
|
|
|
import org.openmetadata.service.security.saml.SamlSettingsHolder;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.socket.FeedServlet;
|
2023-01-03 23:13:22 +05:30
|
|
|
import org.openmetadata.service.socket.OpenMetadataAssetServlet;
|
2022-09-14 23:14:02 -07:00
|
|
|
import org.openmetadata.service.socket.SocketAddressFilter;
|
|
|
|
import org.openmetadata.service.socket.WebSocketManager;
|
2022-11-29 19:37:29 +01:00
|
|
|
import org.openmetadata.service.util.MicrometerBundleSingleton;
|
2023-09-27 19:59:24 +03:00
|
|
|
import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory;
|
2023-04-05 18:47:51 +05:30
|
|
|
import org.openmetadata.service.workflows.searchIndex.SearchIndexEvent;
|
2023-05-08 16:54:45 +05:30
|
|
|
import org.quartz.SchedulerException;
|
2021-08-01 14:27:44 -07:00
|
|
|
|
2021-12-16 07:13:50 -08:00
|
|
|
/** Main catalog application */
|
2022-01-20 02:06:48 +05:30
|
|
|
@Slf4j
|
2022-09-17 11:35:45 -07:00
|
|
|
public class OpenMetadataApplication extends Application<OpenMetadataApplicationConfig> {
|
2021-12-29 11:37:47 -08:00
|
|
|
private Authorizer authorizer;
|
2022-10-27 22:56:20 +05:30
|
|
|
private AuthenticatorHandler authenticatorHandler;
|
2023-09-30 22:13:16 -07:00
|
|
|
private static CollectionDAO collectionDAO;
|
2022-10-27 22:56:20 +05:30
|
|
|
|
2021-08-01 14:27:44 -07:00
|
|
|
@Override
|
2022-09-17 11:35:45 -07:00
|
|
|
public void run(OpenMetadataApplicationConfig catalogConfig, Environment environment)
|
2021-12-16 07:13:50 -08:00
|
|
|
throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException,
|
2023-03-24 21:56:49 +05:30
|
|
|
InvocationTargetException, IOException, ConfigurationException, CertificateException, KeyStoreException,
|
|
|
|
NoSuchAlgorithmException {
|
2022-10-21 20:49:41 -07:00
|
|
|
validateConfiguration(catalogConfig);
|
|
|
|
|
2023-06-26 00:33:44 -07:00
|
|
|
// init for dataSourceFactory
|
2023-06-28 12:01:10 -07:00
|
|
|
DatasourceConfig.initialize(catalogConfig.getDataSourceFactory().getDriverClass());
|
2023-06-26 00:33:44 -07:00
|
|
|
|
2023-04-04 17:02:08 +05:30
|
|
|
ChangeEventConfig.initialize(catalogConfig);
|
2022-08-25 19:07:44 +05:30
|
|
|
final Jdbi jdbi = createAndSetupJDBI(environment, catalogConfig.getDataSourceFactory());
|
2023-09-08 23:19:44 +05:30
|
|
|
JdbiUnitOfWorkProvider jdbiUnitOfWorkProvider = JdbiUnitOfWorkProvider.withDefault(jdbi);
|
2023-09-30 22:13:16 -07:00
|
|
|
collectionDAO = (CollectionDAO) getWrappedInstanceForDaoClass(CollectionDAO.class);
|
2023-09-13 08:58:42 +05:30
|
|
|
JdbiTransactionManager.initialize(jdbiUnitOfWorkProvider.getHandleManager());
|
|
|
|
environment.jersey().register(new JdbiUnitOfWorkApplicationEventListener(new HashSet<>()));
|
2023-03-03 19:10:01 +01:00
|
|
|
|
2023-04-12 19:11:10 +05:30
|
|
|
// Configure the Fernet instance
|
|
|
|
Fernet.getInstance().setFernetKey(catalogConfig);
|
|
|
|
|
2023-04-05 10:31:51 +05:30
|
|
|
// Init Settings Cache
|
2023-09-30 22:13:16 -07:00
|
|
|
SettingsCache.initialize(catalogConfig);
|
2023-04-05 10:31:51 +05:30
|
|
|
|
2023-03-03 19:10:01 +01:00
|
|
|
// init Secret Manager
|
2022-07-13 20:49:27 +02:00
|
|
|
final SecretsManager secretsManager =
|
2022-08-09 09:00:43 +02:00
|
|
|
SecretsManagerFactory.createSecretsManager(
|
|
|
|
catalogConfig.getSecretsManagerConfiguration(), catalogConfig.getClusterName());
|
2022-07-13 20:49:27 +02:00
|
|
|
|
2023-03-03 19:10:01 +01:00
|
|
|
// init Entity Masker
|
2023-05-23 23:54:14 -07:00
|
|
|
EntityMaskerFactory.createEntityMasker();
|
2022-02-08 09:50:39 +01:00
|
|
|
|
2022-05-05 03:02:33 -07:00
|
|
|
// Instantiate JWT Token Generator
|
|
|
|
JWTTokenGenerator.getInstance().init(catalogConfig.getJwtTokenConfiguration());
|
|
|
|
|
2022-04-27 18:58:44 +01:00
|
|
|
// Set the Database type for choosing correct queries from annotations
|
|
|
|
jdbi.getConfig(SqlObjects.class)
|
|
|
|
.setSqlLocator(new ConnectionAwareAnnotationSqlLocator(catalogConfig.getDataSourceFactory().getDriverClass()));
|
|
|
|
|
2022-02-01 08:31:34 +01:00
|
|
|
// Validate flyway Migrations
|
2023-08-17 14:39:05 -07:00
|
|
|
validateMigrations(jdbi, catalogConfig);
|
2022-02-01 08:31:34 +01:00
|
|
|
|
2021-08-01 14:27:44 -07:00
|
|
|
// Register Authorizer
|
2022-04-13 23:55:35 -07:00
|
|
|
registerAuthorizer(catalogConfig, environment);
|
2021-08-01 14:27:44 -07:00
|
|
|
|
2022-10-27 22:56:20 +05:30
|
|
|
// Register Authenticator
|
2022-12-16 16:14:40 -08:00
|
|
|
registerAuthenticator(catalogConfig);
|
2022-10-27 22:56:20 +05:30
|
|
|
|
2021-08-01 14:27:44 -07:00
|
|
|
// Unregister dropwizard default exception mappers
|
|
|
|
((DefaultServerFactory) catalogConfig.getServerFactory()).setRegisterDefaultExceptionMappers(false);
|
2021-10-04 21:06:29 -07:00
|
|
|
environment.jersey().property(ServerProperties.RESPONSE_SET_STATUS_OVER_SEND_ERROR, true);
|
2021-08-01 14:27:44 -07:00
|
|
|
environment.jersey().register(MultiPartFeature.class);
|
|
|
|
environment.jersey().register(CatalogGenericExceptionMapper.class);
|
|
|
|
|
|
|
|
// Override constraint violation mapper to catch Json validation errors
|
|
|
|
environment.jersey().register(new ConstraintViolationExceptionMapper());
|
|
|
|
|
|
|
|
// Restore dropwizard default exception mappers
|
2021-10-04 21:06:29 -07:00
|
|
|
environment.jersey().register(new LoggingExceptionMapper<>() {});
|
2021-08-01 14:27:44 -07:00
|
|
|
environment.jersey().register(new JsonProcessingExceptionMapper(true));
|
|
|
|
environment.jersey().register(new EarlyEofExceptionMapper());
|
2021-10-04 21:06:29 -07:00
|
|
|
environment.jersey().register(JsonMappingExceptionMapper.class);
|
2022-05-27 03:09:13 -07:00
|
|
|
environment.healthChecks().register("OpenMetadataServerHealthCheck", new OpenMetadataServerHealthCheck());
|
2022-07-28 23:45:12 +05:30
|
|
|
// start event hub before registering publishers
|
|
|
|
EventPubSub.start();
|
|
|
|
|
2023-09-30 22:13:16 -07:00
|
|
|
registerResources(catalogConfig, environment, jdbi, collectionDAO);
|
2021-08-03 22:40:42 -07:00
|
|
|
|
|
|
|
// Register Event Handler
|
2023-09-08 23:19:44 +05:30
|
|
|
registerEventFilter(catalogConfig, environment, jdbiUnitOfWorkProvider);
|
2021-12-23 20:10:38 -08:00
|
|
|
environment.lifecycle().manage(new ManagedShutdown());
|
2022-01-12 16:35:27 -08:00
|
|
|
// Register Event publishers
|
2023-09-30 22:13:16 -07:00
|
|
|
registerEventPublisher(catalogConfig);
|
2022-05-27 03:09:13 -07:00
|
|
|
|
2022-11-14 13:48:50 +01:00
|
|
|
// update entities secrets if required
|
|
|
|
new SecretsManagerUpdateService(secretsManager, catalogConfig.getClusterName()).updateEntities();
|
|
|
|
|
2022-04-23 19:17:20 -07:00
|
|
|
// start authorizer after event publishers
|
|
|
|
// authorizer creates admin/bot users, ES publisher should start before to index users created by authorizer
|
2023-09-30 22:13:16 -07:00
|
|
|
authorizer.init(catalogConfig, collectionDAO);
|
2022-10-27 22:56:20 +05:30
|
|
|
|
|
|
|
// authenticationHandler Handles auth related activities
|
2023-09-30 22:13:16 -07:00
|
|
|
authenticatorHandler.init(catalogConfig, collectionDAO);
|
2022-10-27 22:56:20 +05:30
|
|
|
|
2023-03-28 12:51:52 +05:30
|
|
|
webAnalyticEvents = MicrometerBundleSingleton.latencyTimer(catalogConfig.getEventMonitorConfiguration());
|
2022-05-27 03:09:13 -07:00
|
|
|
FilterRegistration.Dynamic micrometerFilter =
|
2023-03-24 21:56:49 +05:30
|
|
|
environment.servlets().addFilter("OMMicrometerHttpFilter", new OMMicrometerHttpFilter());
|
2023-03-28 12:51:52 +05:30
|
|
|
micrometerFilter.addMappingForUrlPatterns(
|
|
|
|
EnumSet.allOf(DispatcherType.class), true, catalogConfig.getEventMonitorConfiguration().getPathPattern());
|
2022-10-21 20:49:41 -07:00
|
|
|
initializeWebsockets(catalogConfig, environment);
|
2023-03-24 21:56:49 +05:30
|
|
|
registerSamlHandlers(catalogConfig, environment);
|
2023-01-03 23:13:22 +05:30
|
|
|
|
|
|
|
// Handle Asset Using Servlet
|
2023-08-18 17:48:17 +05:30
|
|
|
OpenMetadataAssetServlet assetServlet =
|
|
|
|
new OpenMetadataAssetServlet("/assets", "/", "index.html", catalogConfig.getWebConfiguration());
|
2023-01-03 23:13:22 +05:30
|
|
|
String pathPattern = "/" + '*';
|
|
|
|
environment.servlets().addServlet("static", assetServlet).addMapping(pathPattern);
|
2023-05-28 23:44:02 +05:30
|
|
|
|
|
|
|
registerExtensions(catalogConfig, environment, jdbi);
|
2023-06-22 06:53:56 +02:00
|
|
|
|
|
|
|
// Handle Pipeline Service Client Status job
|
|
|
|
PipelineServiceStatusJobHandler pipelineServiceStatusJobHandler =
|
|
|
|
PipelineServiceStatusJobHandler.create(
|
|
|
|
catalogConfig.getPipelineServiceClientConfiguration(), catalogConfig.getClusterName());
|
|
|
|
pipelineServiceStatusJobHandler.addPipelineServiceStatusJob();
|
2023-05-28 23:44:02 +05:30
|
|
|
}
|
|
|
|
|
|
|
|
private void registerExtensions(OpenMetadataApplicationConfig catalogConfig, Environment environment, Jdbi jdbi) {
|
|
|
|
ExtensionConfiguration extensionConfiguration = catalogConfig.getExtensionConfiguration();
|
|
|
|
if (extensionConfiguration != null) {
|
|
|
|
for (Extension extension : extensionConfiguration.getExtensions()) {
|
|
|
|
try {
|
|
|
|
OpenMetadataExtension omExtension =
|
|
|
|
Class.forName(extension.getClassName())
|
|
|
|
.asSubclass(OpenMetadataExtension.class)
|
|
|
|
.getConstructor()
|
|
|
|
.newInstance();
|
2023-06-06 00:55:14 +05:30
|
|
|
omExtension.init(extension, catalogConfig, environment, jdbi);
|
2023-05-28 23:44:02 +05:30
|
|
|
LOG.info("[OmExtension] Registering Extension: {}", extension.getClassName());
|
|
|
|
} catch (Exception ex) {
|
|
|
|
LOG.error("[OmExtension] Failed in registering Extension {}", extension.getClassName());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-08-01 14:27:44 -07:00
|
|
|
}
|
|
|
|
|
2023-03-24 21:56:49 +05:30
|
|
|
private void registerSamlHandlers(OpenMetadataApplicationConfig catalogConfig, Environment environment)
|
|
|
|
throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException {
|
|
|
|
if (catalogConfig.getAuthenticationConfiguration() != null
|
2023-07-26 11:34:58 -07:00
|
|
|
&& catalogConfig.getAuthenticationConfiguration().getProvider().equals(AuthProvider.SAML)) {
|
2023-03-24 21:56:49 +05:30
|
|
|
SamlSettingsHolder.getInstance().initDefaultSettings(catalogConfig);
|
|
|
|
ServletRegistration.Dynamic samlRedirectServlet =
|
|
|
|
environment.servlets().addServlet("saml_login", new SamlLoginServlet());
|
|
|
|
samlRedirectServlet.addMapping("/api/v1/saml/login");
|
|
|
|
ServletRegistration.Dynamic samlReceiverServlet =
|
|
|
|
environment.servlets().addServlet("saml_acs", new SamlAssertionConsumerServlet());
|
|
|
|
samlReceiverServlet.addMapping("/api/v1/saml/acs");
|
|
|
|
ServletRegistration.Dynamic samlMetadataServlet =
|
|
|
|
environment.servlets().addServlet("saml_metadata", new SamlMetadataServlet());
|
|
|
|
samlMetadataServlet.addMapping("/api/v1/saml/metadata");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-25 19:07:44 +05:30
|
|
|
private Jdbi createAndSetupJDBI(Environment environment, DataSourceFactory dbFactory) {
|
2023-09-27 19:59:24 +03:00
|
|
|
// Check for db auth providers.
|
|
|
|
DatabaseAuthenticationProviderFactory.get(dbFactory.getUrl())
|
|
|
|
.ifPresent(
|
|
|
|
databaseAuthenticationProvider -> {
|
|
|
|
String token =
|
|
|
|
databaseAuthenticationProvider.authenticate(
|
|
|
|
dbFactory.getUrl(), dbFactory.getUser(), dbFactory.getPassword());
|
|
|
|
dbFactory.setPassword(token);
|
|
|
|
});
|
|
|
|
|
2022-08-25 19:07:44 +05:30
|
|
|
Jdbi jdbi = new JdbiFactory().build(environment, dbFactory, "database");
|
|
|
|
SqlLogger sqlLogger =
|
|
|
|
new SqlLogger() {
|
2023-01-17 11:34:29 -08:00
|
|
|
@Override
|
|
|
|
public void logBeforeExecution(StatementContext context) {
|
|
|
|
LOG.debug("sql {}, parameters {}", context.getRenderedSql(), context.getBinding());
|
|
|
|
}
|
|
|
|
|
2022-08-25 19:07:44 +05:30
|
|
|
@Override
|
|
|
|
public void logAfterExecution(StatementContext context) {
|
|
|
|
LOG.debug(
|
|
|
|
"sql {}, parameters {}, timeTaken {} ms",
|
|
|
|
context.getRenderedSql(),
|
|
|
|
context.getBinding(),
|
|
|
|
context.getElapsedTime(ChronoUnit.MILLIS));
|
|
|
|
}
|
|
|
|
};
|
|
|
|
if (LOG.isDebugEnabled()) {
|
|
|
|
jdbi.setSqlLogger(sqlLogger);
|
|
|
|
}
|
|
|
|
// Set the Database type for choosing correct queries from annotations
|
|
|
|
jdbi.getConfig(SqlObjects.class).setSqlLocator(new ConnectionAwareAnnotationSqlLocator(dbFactory.getDriverClass()));
|
|
|
|
|
|
|
|
return jdbi;
|
|
|
|
}
|
|
|
|
|
2021-08-01 14:27:44 -07:00
|
|
|
@SneakyThrows
|
|
|
|
@Override
|
2022-09-17 11:35:45 -07:00
|
|
|
public void initialize(Bootstrap<OpenMetadataApplicationConfig> bootstrap) {
|
2022-01-11 10:13:49 -08:00
|
|
|
bootstrap.setConfigurationSourceProvider(
|
|
|
|
new SubstitutingSourceProvider(
|
|
|
|
bootstrap.getConfigurationSourceProvider(), new EnvironmentVariableSubstitutor(false)));
|
2021-12-16 07:13:50 -08:00
|
|
|
bootstrap.addBundle(
|
|
|
|
new SwaggerBundle<>() {
|
|
|
|
@Override
|
2022-09-17 11:35:45 -07:00
|
|
|
protected SwaggerBundleConfiguration getSwaggerBundleConfiguration(
|
|
|
|
OpenMetadataApplicationConfig catalogConfig) {
|
2021-12-16 07:13:50 -08:00
|
|
|
return catalogConfig.getSwaggerBundleConfig();
|
|
|
|
}
|
|
|
|
});
|
|
|
|
bootstrap.addBundle(
|
|
|
|
new HealthCheckBundle<>() {
|
|
|
|
@Override
|
2022-09-17 11:35:45 -07:00
|
|
|
protected HealthConfiguration getHealthConfiguration(final OpenMetadataApplicationConfig configuration) {
|
2021-12-16 07:13:50 -08:00
|
|
|
return configuration.getHealthConfiguration();
|
|
|
|
}
|
|
|
|
});
|
2022-11-29 19:37:29 +01:00
|
|
|
bootstrap.addBundle(MicrometerBundleSingleton.getInstance());
|
2023-04-05 07:50:12 -07:00
|
|
|
bootstrap.addBundle(
|
2023-08-18 17:48:17 +05:30
|
|
|
new OMWebBundle<>() {
|
2023-04-05 07:50:12 -07:00
|
|
|
@Override
|
2023-08-18 17:48:17 +05:30
|
|
|
public OMWebConfiguration getWebConfiguration(final OpenMetadataApplicationConfig configuration) {
|
2023-04-05 07:50:12 -07:00
|
|
|
return configuration.getWebConfiguration();
|
|
|
|
}
|
|
|
|
});
|
2021-08-01 14:27:44 -07:00
|
|
|
super.initialize(bootstrap);
|
|
|
|
}
|
|
|
|
|
2023-08-17 14:39:05 -07:00
|
|
|
private void validateMigrations(Jdbi jdbi, OpenMetadataApplicationConfig conf) throws IOException {
|
2022-02-01 08:31:34 +01:00
|
|
|
LOG.info("Validating Flyway migrations");
|
|
|
|
Optional<String> lastMigrated = Migration.lastMigrated(jdbi);
|
2023-08-17 14:39:05 -07:00
|
|
|
String maxMigration = Migration.lastMigrationFile(conf.getMigrationConfiguration());
|
2022-02-01 08:31:34 +01:00
|
|
|
if (lastMigrated.isEmpty()) {
|
2022-04-27 18:58:44 +01:00
|
|
|
throw new IllegalStateException(
|
|
|
|
"Could not validate Flyway migrations in the database. Make sure you have run `./bootstrap/bootstrap_storage.sh migrate-all` at least once.");
|
2022-02-01 08:31:34 +01:00
|
|
|
}
|
|
|
|
if (lastMigrated.get().compareTo(maxMigration) < 0) {
|
2022-04-27 18:58:44 +01:00
|
|
|
throw new IllegalStateException(
|
|
|
|
"There are pending migrations to be run on the database."
|
2022-02-01 08:31:34 +01:00
|
|
|
+ " Please backup your data and run `./bootstrap/bootstrap_storage.sh migrate-all`."
|
|
|
|
+ " You can find more information on upgrading OpenMetadata at"
|
2022-08-29 19:38:54 +05:30
|
|
|
+ " https://docs.open-metadata.org/deployment/upgrade ");
|
2022-02-01 08:31:34 +01:00
|
|
|
}
|
2023-07-03 20:15:13 +05:30
|
|
|
|
2023-08-17 14:39:05 -07:00
|
|
|
LOG.info("Validating native migrations");
|
|
|
|
ConnectionType connectionType = ConnectionType.from(conf.getDataSourceFactory().getDriverClass());
|
|
|
|
MigrationWorkflow migrationWorkflow =
|
|
|
|
new MigrationWorkflow(jdbi, conf.getMigrationConfiguration().getNativePath(), connectionType, false);
|
|
|
|
migrationWorkflow.validateMigrationsForServer();
|
2022-02-01 08:31:34 +01:00
|
|
|
}
|
|
|
|
|
2022-10-21 20:49:41 -07:00
|
|
|
private void validateConfiguration(OpenMetadataApplicationConfig catalogConfig) throws ConfigurationException {
|
|
|
|
if (catalogConfig.getAuthorizerConfiguration().getBotPrincipals() != null) {
|
|
|
|
throw new ConfigurationException(
|
|
|
|
"'botPrincipals' configuration is deprecated. Please remove it from "
|
|
|
|
+ "'openmetadata.yaml and restart the server");
|
|
|
|
}
|
2023-03-30 19:37:45 +02:00
|
|
|
if (catalogConfig.getPipelineServiceClientConfiguration().getAuthConfig() != null) {
|
|
|
|
LOG.warn(
|
|
|
|
"'authProvider' and 'authConfig' from the 'pipelineServiceClientConfiguration' option are deprecated and will be removed in future releases.");
|
|
|
|
}
|
2022-10-21 20:49:41 -07:00
|
|
|
}
|
|
|
|
|
2022-09-17 11:35:45 -07:00
|
|
|
private void registerAuthorizer(OpenMetadataApplicationConfig catalogConfig, Environment environment)
|
2021-12-16 07:13:50 -08:00
|
|
|
throws NoSuchMethodException, ClassNotFoundException, IllegalAccessException, InvocationTargetException,
|
2022-04-17 21:49:10 -07:00
|
|
|
InstantiationException {
|
2021-08-01 14:27:44 -07:00
|
|
|
AuthorizerConfiguration authorizerConf = catalogConfig.getAuthorizerConfiguration();
|
|
|
|
AuthenticationConfiguration authenticationConfiguration = catalogConfig.getAuthenticationConfiguration();
|
2022-06-15 10:14:47 +05:30
|
|
|
// to authenticate request while opening websocket connections
|
2021-08-01 14:27:44 -07:00
|
|
|
if (authorizerConf != null) {
|
2022-05-22 15:28:55 -07:00
|
|
|
authorizer =
|
|
|
|
Class.forName(authorizerConf.getClassName()).asSubclass(Authorizer.class).getConstructor().newInstance();
|
2021-08-01 14:27:44 -07:00
|
|
|
String filterClazzName = authorizerConf.getContainerRequestFilter();
|
|
|
|
ContainerRequestFilter filter;
|
2022-02-28 05:34:39 -08:00
|
|
|
if (!StringUtils.isEmpty(filterClazzName)) {
|
2021-12-16 07:13:50 -08:00
|
|
|
filter =
|
2022-05-22 15:28:55 -07:00
|
|
|
Class.forName(filterClazzName)
|
|
|
|
.asSubclass(ContainerRequestFilter.class)
|
2023-02-02 20:19:32 +05:30
|
|
|
.getConstructor(AuthenticationConfiguration.class, AuthorizerConfiguration.class)
|
|
|
|
.newInstance(authenticationConfiguration, authorizerConf);
|
2022-02-28 05:34:39 -08:00
|
|
|
LOG.info("Registering ContainerRequestFilter: {}", filter.getClass().getCanonicalName());
|
|
|
|
environment.jersey().register(filter);
|
2021-08-01 14:27:44 -07:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
LOG.info("Authorizer config not set, setting noop authorizer");
|
2022-05-22 15:28:55 -07:00
|
|
|
authorizer = new NoopAuthorizer();
|
2023-02-02 20:19:32 +05:30
|
|
|
ContainerRequestFilter filter = new NoopFilter(authenticationConfiguration, null);
|
2021-08-01 14:27:44 -07:00
|
|
|
environment.jersey().register(filter);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-12-16 16:14:40 -08:00
|
|
|
private void registerAuthenticator(OpenMetadataApplicationConfig catalogConfig) {
|
2022-10-27 22:56:20 +05:30
|
|
|
AuthenticationConfiguration authenticationConfiguration = catalogConfig.getAuthenticationConfiguration();
|
|
|
|
switch (authenticationConfiguration.getProvider()) {
|
2023-07-26 11:34:58 -07:00
|
|
|
case BASIC:
|
2022-10-27 22:56:20 +05:30
|
|
|
authenticatorHandler = new BasicAuthenticator();
|
|
|
|
break;
|
2023-07-26 11:34:58 -07:00
|
|
|
case LDAP:
|
2022-10-27 22:56:20 +05:30
|
|
|
authenticatorHandler = new LdapAuthenticator();
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
// For all other types, google, okta etc. auth is handled externally
|
|
|
|
authenticatorHandler = new NoopAuthenticator();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-08 23:19:44 +05:30
|
|
|
private void registerEventFilter(
|
|
|
|
OpenMetadataApplicationConfig catalogConfig, Environment environment, JdbiUnitOfWorkProvider provider) {
|
2021-08-11 14:54:13 -07:00
|
|
|
if (catalogConfig.getEventHandlerConfiguration() != null) {
|
2023-09-08 23:19:44 +05:30
|
|
|
ContainerResponseFilter eventFilter = new EventFilter(catalogConfig, provider);
|
2021-08-11 14:54:13 -07:00
|
|
|
environment.jersey().register(eventFilter);
|
2023-04-05 18:47:51 +05:30
|
|
|
ContainerResponseFilter reindexingJobs = new SearchIndexEvent();
|
|
|
|
environment.jersey().register(reindexingJobs);
|
2021-08-11 14:54:13 -07:00
|
|
|
}
|
2021-08-03 22:40:42 -07:00
|
|
|
}
|
|
|
|
|
2023-09-30 22:13:16 -07:00
|
|
|
private void registerEventPublisher(OpenMetadataApplicationConfig openMetadataApplicationConfig) {
|
2022-01-12 16:35:27 -08:00
|
|
|
// register ElasticSearch Event publisher
|
2022-09-17 11:35:45 -07:00
|
|
|
if (openMetadataApplicationConfig.getElasticSearchConfiguration() != null) {
|
2023-08-09 23:24:10 -07:00
|
|
|
SearchEventPublisher searchEventPublisher =
|
2023-09-30 22:13:16 -07:00
|
|
|
new SearchEventPublisher(openMetadataApplicationConfig.getElasticSearchConfiguration(), collectionDAO);
|
2023-08-09 23:24:10 -07:00
|
|
|
EventPubSub.addEventHandler(searchEventPublisher);
|
2022-01-12 16:35:27 -08:00
|
|
|
}
|
2022-11-28 17:45:18 +01:00
|
|
|
|
|
|
|
if (openMetadataApplicationConfig.getEventMonitorConfiguration() != null) {
|
|
|
|
final EventMonitor eventMonitor =
|
|
|
|
EventMonitorFactory.createEventMonitor(
|
|
|
|
openMetadataApplicationConfig.getEventMonitorConfiguration(),
|
|
|
|
openMetadataApplicationConfig.getClusterName());
|
|
|
|
EventMonitorPublisher eventMonitorPublisher =
|
|
|
|
new EventMonitorPublisher(openMetadataApplicationConfig.getEventMonitorConfiguration(), eventMonitor);
|
|
|
|
EventPubSub.addEventHandler(eventMonitorPublisher);
|
|
|
|
}
|
2022-01-12 16:35:27 -08:00
|
|
|
}
|
|
|
|
|
2023-09-08 23:19:44 +05:30
|
|
|
private void registerResources(
|
2023-09-30 22:13:16 -07:00
|
|
|
OpenMetadataApplicationConfig config, Environment environment, Jdbi jdbi, CollectionDAO daoObject) {
|
2023-06-07 01:02:52 +05:30
|
|
|
List<String> extensionResources =
|
|
|
|
config.getExtensionConfiguration() != null ? config.getExtensionConfiguration().getResourcePackage() : null;
|
|
|
|
CollectionRegistry.initialize(extensionResources);
|
2023-09-08 23:19:44 +05:30
|
|
|
CollectionRegistry.getInstance()
|
2023-09-30 22:13:16 -07:00
|
|
|
.registerResources(jdbi, environment, config, daoObject, authorizer, authenticatorHandler);
|
2021-08-01 14:27:44 -07:00
|
|
|
environment.jersey().register(new JsonPatchProvider());
|
2023-07-13 21:16:05 +05:30
|
|
|
OMErrorPageHandler eph = new OMErrorPageHandler(config.getWebConfiguration());
|
2021-08-01 14:27:44 -07:00
|
|
|
eph.addErrorPage(Response.Status.NOT_FOUND.getStatusCode(), "/");
|
|
|
|
environment.getApplicationContext().setErrorHandler(eph);
|
|
|
|
}
|
|
|
|
|
2022-10-21 20:49:41 -07:00
|
|
|
private void initializeWebsockets(OpenMetadataApplicationConfig catalogConfig, Environment environment) {
|
2022-06-27 16:40:31 +05:30
|
|
|
SocketAddressFilter socketAddressFilter;
|
2022-07-01 19:53:59 +03:00
|
|
|
String pathSpec = "/api/v1/push/feed/*";
|
2022-06-27 16:40:31 +05:30
|
|
|
if (catalogConfig.getAuthorizerConfiguration() != null) {
|
|
|
|
socketAddressFilter =
|
|
|
|
new SocketAddressFilter(
|
2023-02-02 20:19:32 +05:30
|
|
|
catalogConfig.getAuthenticationConfiguration(), catalogConfig.getAuthorizerConfiguration());
|
2022-06-27 16:40:31 +05:30
|
|
|
} else {
|
|
|
|
socketAddressFilter = new SocketAddressFilter();
|
|
|
|
}
|
|
|
|
|
2022-06-15 10:14:47 +05:30
|
|
|
EngineIoServerOptions eioOptions = EngineIoServerOptions.newFromDefault();
|
|
|
|
eioOptions.setAllowedCorsOrigins(null);
|
|
|
|
WebSocketManager.WebSocketManagerBuilder.build(eioOptions);
|
|
|
|
environment.getApplicationContext().setContextPath("/");
|
2022-06-27 16:40:31 +05:30
|
|
|
environment
|
|
|
|
.getApplicationContext()
|
2022-07-01 19:53:59 +03:00
|
|
|
.addFilter(new FilterHolder(socketAddressFilter), pathSpec, EnumSet.of(DispatcherType.REQUEST));
|
|
|
|
environment.getApplicationContext().addServlet(new ServletHolder(new FeedServlet()), pathSpec);
|
2022-06-15 10:14:47 +05:30
|
|
|
// Upgrade connection to websocket from Http
|
|
|
|
try {
|
2023-04-14 11:31:36 +05:30
|
|
|
WebSocketUpgradeFilter.configure(environment.getApplicationContext());
|
|
|
|
NativeWebSocketServletContainerInitializer.configure(
|
|
|
|
environment.getApplicationContext(),
|
2023-07-02 11:32:52 -07:00
|
|
|
(context, container) ->
|
|
|
|
container.addMapping(
|
|
|
|
new ServletPathSpec(pathSpec),
|
|
|
|
(servletUpgradeRequest, servletUpgradeResponse) ->
|
|
|
|
new JettyWebSocketHandler(WebSocketManager.getInstance().getEngineIoServer())));
|
2022-06-15 10:14:47 +05:30
|
|
|
} catch (ServletException ex) {
|
2022-06-27 16:40:31 +05:30
|
|
|
LOG.error("Websocket Upgrade Filter error : " + ex.getMessage());
|
2022-06-15 10:14:47 +05:30
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-01 14:27:44 -07:00
|
|
|
public static void main(String[] args) throws Exception {
|
2022-10-26 18:24:40 +05:30
|
|
|
OpenMetadataApplication openMetadataApplication = new OpenMetadataApplication();
|
|
|
|
openMetadataApplication.run(args);
|
2021-08-01 14:27:44 -07:00
|
|
|
}
|
2021-12-23 20:10:38 -08:00
|
|
|
|
2022-02-23 20:40:25 -08:00
|
|
|
public static class ManagedShutdown implements Managed {
|
2021-12-23 20:10:38 -08:00
|
|
|
|
|
|
|
@Override
|
2022-05-22 15:28:55 -07:00
|
|
|
public void start() {
|
2022-01-09 21:03:50 -08:00
|
|
|
LOG.info("Starting the application");
|
2021-12-23 20:10:38 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2023-05-08 16:54:45 +05:30
|
|
|
public void stop() throws InterruptedException, SchedulerException {
|
2023-08-08 16:42:54 -07:00
|
|
|
LOG.info("Cache with Id Stats {}", EntityRepository.CACHE_WITH_ID.stats());
|
|
|
|
LOG.info("Cache with name Stats {}", EntityRepository.CACHE_WITH_NAME.stats());
|
2021-12-23 20:10:38 -08:00
|
|
|
EventPubSub.shutdown();
|
2023-05-08 16:54:45 +05:30
|
|
|
ReportsHandler.shutDown();
|
2022-01-09 21:03:50 -08:00
|
|
|
LOG.info("Stopping the application");
|
2021-12-23 20:10:38 -08:00
|
|
|
}
|
|
|
|
}
|
2021-08-01 14:27:44 -07:00
|
|
|
}
|