Fix#8577: Bots sensitive passwords fields are stored in secrets store (#8720)

* Services sensitive passwords fields are stored in secrets store

* Update services on application startup in case of changes in the JSON Schema

* Minor changes after manual test

* Bots sensitive passwords fields are stored in secrets store

* Fix Java style
This commit is contained in:
Nahuel 2022-11-14 16:46:05 +01:00 committed by GitHub
parent 4faafe4235
commit 83003a42da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 220 additions and 60 deletions

View File

@ -13,13 +13,13 @@
package org.openmetadata.service.exception;
public class SecretsManagerMigrationException extends RuntimeException {
public class SecretsManagerUpdateException extends RuntimeException {
public SecretsManagerMigrationException(String message, Throwable throwable) {
public SecretsManagerUpdateException(String message, Throwable throwable) {
super(message, throwable);
}
public SecretsManagerMigrationException(String message) {
public SecretsManagerUpdateException(String message) {
super(message);
}
}

View File

@ -37,6 +37,8 @@ import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord;
import org.openmetadata.service.resources.teams.UserResource;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.security.policyevaluator.SubjectCache;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.EntityUtil.Fields;
@ -100,6 +102,13 @@ public class UserRepository extends EntityRepository<User> {
// Don't store roles, teams and href as JSON. Build it on the fly based on relationships
user.withRoles(null).withTeams(null).withHref(null).withInheritedRoles(null);
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
if (secretsManager != null && Boolean.TRUE.equals(user.getIsBot()) && user.getAuthenticationMechanism() != null) {
user.withAuthenticationMechanism(
secretsManager.encryptOrDecryptAuthenticationMechanism(
user.getName(), user.getAuthenticationMechanism(), true));
}
store(user, update);
// Restore the relationships

View File

@ -101,6 +101,8 @@ import org.openmetadata.service.jdbi3.TokenRepository;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.security.AuthorizationException;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.auth.AuthenticatorHandler;
@ -1154,6 +1156,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
}
private User decryptOrNullify(SecurityContext securityContext, User user) {
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
if (Boolean.TRUE.equals(user.getIsBot()) && user.getAuthenticationMechanism() != null) {
try {
authorizer.authorize(
@ -1164,6 +1167,9 @@ public class UserResource extends EntityResource<User, UserRepository> {
user.getAuthenticationMechanism().setConfig(null);
return user;
}
user.withAuthenticationMechanism(
secretsManager.encryptOrDecryptAuthenticationMechanism(
user.getName(), user.getAuthenticationMechanism(), false));
}
return user;
}

View File

@ -23,10 +23,12 @@ import java.util.Locale;
import lombok.Getter;
import org.openmetadata.annotations.PasswordField;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import org.openmetadata.service.exception.InvalidServiceConnectionException;
import org.openmetadata.service.exception.SecretsManagerException;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.util.AuthenticationMechanismBuilder;
import org.openmetadata.service.util.JsonUtils;
public abstract class SecretsManager {
@ -48,18 +50,34 @@ public abstract class SecretsManager {
try {
Class<?> clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType));
Object newConnectionConfig = JsonUtils.convertValue(connectionConfig, clazz);
if (encrypt) {
encryptPasswordFields(newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName));
} else {
decryptPasswordFields(newConnectionConfig);
}
return newConnectionConfig;
return encryptOrDecryptPasswordFields(
newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt);
} catch (Exception e) {
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to encrypt connection instance of %s", connectionType));
}
}
public AuthenticationMechanism encryptOrDecryptAuthenticationMechanism(
String name, AuthenticationMechanism authenticationMechanism, boolean encrypt) {
authenticationMechanism = AuthenticationMechanismBuilder.build(authenticationMechanism);
try {
return (AuthenticationMechanism)
encryptOrDecryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), encrypt);
} catch (Exception e) {
throw InvalidServiceConnectionException.byMessage(name, "Failed to encrypt user bot instance.");
}
}
private Object encryptOrDecryptPasswordFields(Object targetObject, String name, boolean encrypt) {
if (encrypt) {
encryptPasswordFields(targetObject, name);
} else {
decryptPasswordFields(targetObject);
}
return targetObject;
}
private void encryptPasswordFields(Object toEncryptObject, String secretId) {
// for each get method
Arrays.stream(toEncryptObject.getClass().getMethods())

View File

@ -23,7 +23,11 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.ServiceConnectionEntityInterface;
import org.openmetadata.schema.ServiceEntityInterface;
import org.openmetadata.service.exception.SecretsManagerMigrationException;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.SecretsManagerUpdateException;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.ServiceEntityRepository;
import org.openmetadata.service.resources.CollectionRegistry;
@ -40,6 +44,7 @@ import org.openmetadata.service.util.EntityUtil;
public class SecretsManagerUpdateService {
private final SecretsManager secretManager;
private final SecretsManager oldSecretManager;
private final EntityRepository<User> userRepository;
private final Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
connectionTypeRepositoriesMap;
@ -47,12 +52,14 @@ public class SecretsManagerUpdateService {
public SecretsManagerUpdateService(SecretsManager secretsManager, String clusterName) {
this.secretManager = secretsManager;
this.connectionTypeRepositoriesMap = retrieveConnectionTypeRepositoriesMap();
this.userRepository = Entity.getEntityRepository(Entity.USER);
// by default, it is going to be non-managed secrets manager since decrypt is the same for all of them
this.oldSecretManager = SecretsManagerFactory.createSecretsManager(null, clusterName);
}
public void updateEntities() {
updateServices();
updateUsersAuthenticationMechanism();
}
private void updateServices() {
@ -92,7 +99,7 @@ public class SecretsManagerUpdateService {
true));
repository.dao.update(service);
} catch (IOException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
}
}
@ -119,7 +126,7 @@ public class SecretsManagerUpdateService {
!Objects.isNull(service.getConnection()) && !Objects.isNull(service.getConnection().getConfig()))
.collect(Collectors.toList());
} catch (IOException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
}
}
@ -133,7 +140,7 @@ public class SecretsManagerUpdateService {
.map(Optional::get)
.collect(Collectors.toMap(ServiceEntityRepository::getServiceConnectionClass, Function.identity()));
if (connectionTypeRepositoriesMap.isEmpty()) {
throw new SecretsManagerMigrationException("Unexpected error: ServiceRepository not found.");
throw new SecretsManagerUpdateException("Unexpected error: ServiceRepository not found.");
}
return connectionTypeRepositoriesMap;
}
@ -152,8 +159,59 @@ public class SecretsManagerUpdateService {
try {
collectionDetailsClass = Class.forName(collectionDetails.getResourceClass());
} catch (ClassNotFoundException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
}
return collectionDetailsClass;
}
private void updateUsersAuthenticationMechanism() {
LOG.info(
String.format(
"Checking if bot users authentication mechanism updating is needed for secrets manager: [%s]",
secretManager.getSecretsManagerProvider().value()));
List<User> notStoredUsers = retrieveBotUsers();
if (!notStoredUsers.isEmpty()) {
notStoredUsers.forEach(this::updateBotUser);
} else {
LOG.info(
String.format(
"All bot users credentials are already safely stored in [%s] secrets manager",
secretManager.getSecretsManagerProvider().value()));
}
}
private List<User> retrieveBotUsers() {
try {
return userRepository
.listAfter(
null,
new EntityUtil.Fields(List.of("authenticationMechanism")),
new ListFilter(),
userRepository.dao.listCount(new ListFilter()),
null)
.getData().stream()
.filter(this::isBotWithAuthenticationMechanism)
.collect(Collectors.toList());
} catch (IOException e) {
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
}
}
private boolean isBotWithAuthenticationMechanism(User user) {
return Boolean.TRUE.equals(user.getIsBot()) && user.getAuthenticationMechanism() != null;
}
private void updateBotUser(User botUser) {
try {
User user = userRepository.dao.findEntityById(botUser.getId());
AuthenticationMechanism authenticationMechanism =
oldSecretManager.encryptOrDecryptAuthenticationMechanism(
botUser.getName(), user.getAuthenticationMechanism(), false);
userRepository.dao.update(
user.withAuthenticationMechanism(
secretManager.encryptOrDecryptAuthenticationMechanism(botUser.getName(), authenticationMechanism, true)));
} catch (IOException e) {
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
}
}
}

View File

@ -0,0 +1,67 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.util;
import static org.openmetadata.schema.entity.teams.AuthenticationMechanism.AuthType.JWT;
import static org.openmetadata.schema.entity.teams.AuthenticationMechanism.AuthType.SSO;
import org.openmetadata.schema.auth.JWTAuthMechanism;
import org.openmetadata.schema.auth.SSOAuthMechanism;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.security.client.Auth0SSOClientConfig;
import org.openmetadata.schema.security.client.AzureSSOClientConfig;
import org.openmetadata.schema.security.client.CustomOIDCSSOClientConfig;
import org.openmetadata.schema.security.client.GoogleSSOClientConfig;
import org.openmetadata.schema.security.client.OktaSSOClientConfig;
public class AuthenticationMechanismBuilder {
/**
* Build `AuthenticationMechanism` object with concrete class for the config which by definition it is a `Object`.
*
* @param authMechanism the auth mechanism object
* @return auth mechanism object with concrete classes
*/
public static AuthenticationMechanism build(AuthenticationMechanism authMechanism) {
if (authMechanism != null) {
if (JWT.equals(authMechanism.getAuthType())) {
authMechanism.setConfig(JsonUtils.convertValue(authMechanism.getConfig(), JWTAuthMechanism.class));
} else if (SSO.equals(authMechanism.getAuthType())) {
SSOAuthMechanism ssoAuth = JsonUtils.convertValue(authMechanism.getConfig(), SSOAuthMechanism.class);
switch (ssoAuth.getSsoServiceType()) {
case GOOGLE:
ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), GoogleSSOClientConfig.class));
break;
case OKTA:
ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), OktaSSOClientConfig.class));
break;
case AUTH_0:
ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), Auth0SSOClientConfig.class));
break;
case CUSTOM_OIDC:
ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), CustomOIDCSSOClientConfig.class));
break;
case AZURE:
ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), AzureSSOClientConfig.class));
break;
default:
throw new IllegalArgumentException(
String.format("SSO service type [%s] can not be parsed.", ssoAuth.getSsoServiceType()));
}
authMechanism.setConfig(ssoAuth);
}
}
return authMechanism;
}
}

View File

@ -12,16 +12,11 @@
*/
package org.openmetadata.service.secrets;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.reset;
import org.mockito.Mock;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import software.amazon.awssdk.services.ssm.SsmClient;
import software.amazon.awssdk.services.ssm.model.GetParameterRequest;
import software.amazon.awssdk.services.ssm.model.GetParameterResponse;
import software.amazon.awssdk.services.ssm.model.Parameter;
public class AWSSSMSecretsManagerTest extends ExternalSecretsManagerTest {
@ -34,21 +29,8 @@ public class AWSSSMSecretsManagerTest extends ExternalSecretsManagerTest {
reset(ssmClient);
}
@Override
void mockClientGetValue(String value) {
if (value == null) {
lenient()
.when(ssmClient.getParameter(any(GetParameterRequest.class)))
.thenReturn(GetParameterResponse.builder().build());
} else {
lenient()
.when(ssmClient.getParameter(any(GetParameterRequest.class)))
.thenReturn(GetParameterResponse.builder().parameter(Parameter.builder().value(value).build()).build());
}
}
@Override
SecretsManagerProvider expectedSecretManagerProvider() {
return SecretsManagerProvider.AWS_SSM;
return SecretsManagerProvider.MANAGED_AWS_SSM;
}
}

View File

@ -12,15 +12,11 @@
*/
package org.openmetadata.service.secrets;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.reset;
import org.mockito.Mock;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest;
import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse;
public class AWSSecretsManagerTest extends ExternalSecretsManagerTest {
@ -33,21 +29,8 @@ public class AWSSecretsManagerTest extends ExternalSecretsManagerTest {
reset(secretsManagerClient);
}
@Override
void mockClientGetValue(String value) {
if (value == null) {
lenient()
.when(secretsManagerClient.getSecretValue(any(GetSecretValueRequest.class)))
.thenReturn(GetSecretValueResponse.builder().build());
} else {
lenient()
.when(secretsManagerClient.getSecretValue(any(GetSecretValueRequest.class)))
.thenReturn(GetSecretValueResponse.builder().secretString(value).build());
}
}
@Override
SecretsManagerProvider expectedSecretManagerProvider() {
return SecretsManagerProvider.AWS;
return SecretsManagerProvider.MANAGED_AWS;
}
}

View File

@ -21,22 +21,26 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import org.openmetadata.schema.api.services.CreateDatabaseService;
import org.openmetadata.schema.auth.SSOAuthMechanism;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.security.client.OktaSSOClientConfig;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import org.openmetadata.service.fernet.Fernet;
@ExtendWith(MockitoExtension.class)
public abstract class ExternalSecretsManagerTest {
static final boolean DECRYPT = false;
static final String EXPECTED_CONNECTION_JSON =
"{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"password\":\"openmetadata-test\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true,\"supportsQueryComment\":true}";
static final String EXPECTED_SECRET_ID = "/openmetadata/service/database/mysql/test";
static final boolean ENCRYPT = true;
AWSBasedSecretsManager secretsManager;
@BeforeEach
void setUp() {
Fernet fernet = Fernet.getInstance();
fernet.setFernetKey("jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=");
Map<String, String> parameters = new HashMap<>();
parameters.put("region", "eu-west-1");
parameters.put("accessKeyId", "123456");
@ -48,10 +52,24 @@ public abstract class ExternalSecretsManagerTest {
@Test
void testDecryptDatabaseServiceConnectionConfig() {
mockClientGetValue(EXPECTED_CONNECTION_JSON);
testEncryptDecryptServiceConnection(DECRYPT);
}
@Test
void testEncryptDatabaseServiceConnectionConfig() {
testEncryptDecryptServiceConnection(ENCRYPT);
}
@Test
void testDecryptSSOConfig() {
testEncryptDecryptSSOConfig(DECRYPT);
}
@Test
void testEncryptSSSOConfig() {
testEncryptDecryptSSOConfig(ENCRYPT);
}
@Test
void testReturnsExpectedSecretManagerProvider() {
assertEquals(expectedSecretManagerProvider(), secretsManager.getSecretsManagerProvider());
@ -59,20 +77,39 @@ public abstract class ExternalSecretsManagerTest {
abstract void setUpSpecific(SecretsManagerConfiguration config);
abstract void mockClientGetValue(String value);
void testEncryptDecryptServiceConnection(boolean decrypt) {
MysqlConnection mysqlConnection = new MysqlConnection();
mysqlConnection.setPassword("openmetadata-test");
CreateDatabaseService.DatabaseServiceType databaseServiceType = CreateDatabaseService.DatabaseServiceType.Mysql;
String connectionName = "test";
Object actualConfig =
secretsManager.encryptOrDecryptServiceConnectionConfig(
mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt);
MysqlConnection actualConfig =
(MysqlConnection)
secretsManager.encryptOrDecryptServiceConnectionConfig(
mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt);
if (decrypt) {
mysqlConnection.setPassword("secret:/openmetadata/database/test/password");
actualConfig.setPassword(Fernet.getInstance().decrypt(actualConfig.getPassword()));
}
assertEquals(mysqlConnection, actualConfig);
}
void testEncryptDecryptSSOConfig(boolean decrypt) {
OktaSSOClientConfig config = new OktaSSOClientConfig();
config.setPrivateKey("this-is-a-test");
AuthenticationMechanism authenticationMechanism =
new AuthenticationMechanism()
.withAuthType(AuthenticationMechanism.AuthType.SSO)
.withConfig(
new SSOAuthMechanism().withAuthConfig(config).withSsoServiceType(SSOAuthMechanism.SsoServiceType.OKTA));
AuthenticationMechanism actualAuthenticationMechanism =
secretsManager.encryptOrDecryptAuthenticationMechanism("bot", authenticationMechanism, decrypt);
assertEquals(authenticationMechanism, actualAuthenticationMechanism);
}
abstract SecretsManagerProvider expectedSecretManagerProvider();
}