Fix#8577: Bots sensitive passwords fields are stored in secrets store (#8720)

* Services sensitive passwords fields are stored in secrets store

* Update services on application startup in case of changes in the JSON Schema

* Minor changes after manual test

* Bots sensitive passwords fields are stored in secrets store

* Fix Java style

openmetadata-service/src/main/java/org/openmetadata/service/exception/SecretsManagerUpdateException.java

@@ -13,13 +13,13 @@
 
 package org.openmetadata.service.exception;
 
-public class SecretsManagerMigrationException extends RuntimeException {
+public class SecretsManagerUpdateException extends RuntimeException {
 
-  public SecretsManagerMigrationException(String message, Throwable throwable) {
+  public SecretsManagerUpdateException(String message, Throwable throwable) {
     super(message, throwable);
   }
 
-  public SecretsManagerMigrationException(String message) {
+  public SecretsManagerUpdateException(String message) {
     super(message);
   }
 }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java

@@ -37,6 +37,8 @@ import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord;
 import org.openmetadata.service.resources.teams.UserResource;
+import org.openmetadata.service.secrets.SecretsManager;
+import org.openmetadata.service.secrets.SecretsManagerFactory;
 import org.openmetadata.service.security.policyevaluator.SubjectCache;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.EntityUtil.Fields;
@@ -100,6 +102,13 @@ public class UserRepository extends EntityRepository<User> {
     // Don't store roles, teams and href as JSON. Build it on the fly based on relationships
     user.withRoles(null).withTeams(null).withHref(null).withInheritedRoles(null);
 
+    SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
+    if (secretsManager != null && Boolean.TRUE.equals(user.getIsBot()) && user.getAuthenticationMechanism() != null) {
+      user.withAuthenticationMechanism(
+          secretsManager.encryptOrDecryptAuthenticationMechanism(
+              user.getName(), user.getAuthenticationMechanism(), true));
+    }
+
     store(user, update);
 
     // Restore the relationships

openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java

@@ -101,6 +101,8 @@ import org.openmetadata.service.jdbi3.TokenRepository;
 import org.openmetadata.service.jdbi3.UserRepository;
 import org.openmetadata.service.resources.Collection;
 import org.openmetadata.service.resources.EntityResource;
+import org.openmetadata.service.secrets.SecretsManager;
+import org.openmetadata.service.secrets.SecretsManagerFactory;
 import org.openmetadata.service.security.AuthorizationException;
 import org.openmetadata.service.security.Authorizer;
 import org.openmetadata.service.security.auth.AuthenticatorHandler;
@@ -1154,6 +1156,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
   }
 
   private User decryptOrNullify(SecurityContext securityContext, User user) {
+    SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
     if (Boolean.TRUE.equals(user.getIsBot()) && user.getAuthenticationMechanism() != null) {
       try {
         authorizer.authorize(
@@ -1164,6 +1167,9 @@ public class UserResource extends EntityResource<User, UserRepository> {
         user.getAuthenticationMechanism().setConfig(null);
         return user;
       }
+      user.withAuthenticationMechanism(
+          secretsManager.encryptOrDecryptAuthenticationMechanism(
+              user.getName(), user.getAuthenticationMechanism(), false));
     }
     return user;
   }

openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java

@@ -23,10 +23,12 @@ import java.util.Locale;
 import lombok.Getter;
 import org.openmetadata.annotations.PasswordField;
 import org.openmetadata.schema.entity.services.ServiceType;
+import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
 import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
 import org.openmetadata.service.exception.InvalidServiceConnectionException;
 import org.openmetadata.service.exception.SecretsManagerException;
 import org.openmetadata.service.fernet.Fernet;
+import org.openmetadata.service.util.AuthenticationMechanismBuilder;
 import org.openmetadata.service.util.JsonUtils;
 
 public abstract class SecretsManager {
@@ -48,18 +50,34 @@ public abstract class SecretsManager {
     try {
       Class<?> clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType));
       Object newConnectionConfig = JsonUtils.convertValue(connectionConfig, clazz);
-      if (encrypt) {
+      return encryptOrDecryptPasswordFields(
-        encryptPasswordFields(newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName));
+          newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt);
-      } else {
-        decryptPasswordFields(newConnectionConfig);
-      }
-      return newConnectionConfig;
     } catch (Exception e) {
       throw InvalidServiceConnectionException.byMessage(
           connectionType, String.format("Failed to encrypt connection instance of %s", connectionType));
     }
   }
 
+  public AuthenticationMechanism encryptOrDecryptAuthenticationMechanism(
+      String name, AuthenticationMechanism authenticationMechanism, boolean encrypt) {
+    authenticationMechanism = AuthenticationMechanismBuilder.build(authenticationMechanism);
+    try {
+      return (AuthenticationMechanism)
+          encryptOrDecryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), encrypt);
+    } catch (Exception e) {
+      throw InvalidServiceConnectionException.byMessage(name, "Failed to encrypt user bot instance.");
+    }
+  }
+
+  private Object encryptOrDecryptPasswordFields(Object targetObject, String name, boolean encrypt) {
+    if (encrypt) {
+      encryptPasswordFields(targetObject, name);
+    } else {
+      decryptPasswordFields(targetObject);
+    }
+    return targetObject;
+  }
+
   private void encryptPasswordFields(Object toEncryptObject, String secretId) {
     // for each get method
     Arrays.stream(toEncryptObject.getClass().getMethods())

openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java

@@ -23,7 +23,11 @@ import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.schema.ServiceConnectionEntityInterface;
 import org.openmetadata.schema.ServiceEntityInterface;
-import org.openmetadata.service.exception.SecretsManagerMigrationException;
+import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
+import org.openmetadata.schema.entity.teams.User;
+import org.openmetadata.service.Entity;
+import org.openmetadata.service.exception.SecretsManagerUpdateException;
+import org.openmetadata.service.jdbi3.EntityRepository;
 import org.openmetadata.service.jdbi3.ListFilter;
 import org.openmetadata.service.jdbi3.ServiceEntityRepository;
 import org.openmetadata.service.resources.CollectionRegistry;
@@ -40,6 +44,7 @@ import org.openmetadata.service.util.EntityUtil;
 public class SecretsManagerUpdateService {
   private final SecretsManager secretManager;
   private final SecretsManager oldSecretManager;
+  private final EntityRepository<User> userRepository;
 
   private final Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
       connectionTypeRepositoriesMap;
@@ -47,12 +52,14 @@ public class SecretsManagerUpdateService {
   public SecretsManagerUpdateService(SecretsManager secretsManager, String clusterName) {
     this.secretManager = secretsManager;
     this.connectionTypeRepositoriesMap = retrieveConnectionTypeRepositoriesMap();
+    this.userRepository = Entity.getEntityRepository(Entity.USER);
     // by default, it is going to be non-managed secrets manager since decrypt is the same for all of them
     this.oldSecretManager = SecretsManagerFactory.createSecretsManager(null, clusterName);
   }
 
   public void updateEntities() {
     updateServices();
+    updateUsersAuthenticationMechanism();
   }
 
   private void updateServices() {
@@ -92,7 +99,7 @@ public class SecretsManagerUpdateService {
                   true));
       repository.dao.update(service);
     } catch (IOException e) {
-      throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
+      throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
     }
   }
 
@@ -119,7 +126,7 @@ public class SecretsManagerUpdateService {
                   !Objects.isNull(service.getConnection()) && !Objects.isNull(service.getConnection().getConfig()))
           .collect(Collectors.toList());
     } catch (IOException e) {
-      throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
+      throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
     }
   }
 
@@ -133,7 +140,7 @@ public class SecretsManagerUpdateService {
                 .map(Optional::get)
                 .collect(Collectors.toMap(ServiceEntityRepository::getServiceConnectionClass, Function.identity()));
     if (connectionTypeRepositoriesMap.isEmpty()) {
-      throw new SecretsManagerMigrationException("Unexpected error: ServiceRepository not found.");
+      throw new SecretsManagerUpdateException("Unexpected error: ServiceRepository not found.");
     }
     return connectionTypeRepositoriesMap;
   }
@@ -152,8 +159,59 @@ public class SecretsManagerUpdateService {
     try {
       collectionDetailsClass = Class.forName(collectionDetails.getResourceClass());
     } catch (ClassNotFoundException e) {
-      throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
+      throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
     }
     return collectionDetailsClass;
   }
+
+  private void updateUsersAuthenticationMechanism() {
+    LOG.info(
+        String.format(
+            "Checking if bot users authentication mechanism updating is needed for secrets manager: [%s]",
+            secretManager.getSecretsManagerProvider().value()));
+    List<User> notStoredUsers = retrieveBotUsers();
+    if (!notStoredUsers.isEmpty()) {
+      notStoredUsers.forEach(this::updateBotUser);
+    } else {
+      LOG.info(
+          String.format(
+              "All bot users credentials are already safely stored in [%s] secrets manager",
+              secretManager.getSecretsManagerProvider().value()));
+    }
+  }
+
+  private List<User> retrieveBotUsers() {
+    try {
+      return userRepository
+          .listAfter(
+              null,
+              new EntityUtil.Fields(List.of("authenticationMechanism")),
+              new ListFilter(),
+              userRepository.dao.listCount(new ListFilter()),
+              null)
+          .getData().stream()
+          .filter(this::isBotWithAuthenticationMechanism)
+          .collect(Collectors.toList());
+    } catch (IOException e) {
+      throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
+    }
+  }
+
+  private boolean isBotWithAuthenticationMechanism(User user) {
+    return Boolean.TRUE.equals(user.getIsBot()) && user.getAuthenticationMechanism() != null;
+  }
+
+  private void updateBotUser(User botUser) {
+    try {
+      User user = userRepository.dao.findEntityById(botUser.getId());
+      AuthenticationMechanism authenticationMechanism =
+          oldSecretManager.encryptOrDecryptAuthenticationMechanism(
+              botUser.getName(), user.getAuthenticationMechanism(), false);
+      userRepository.dao.update(
+          user.withAuthenticationMechanism(
+              secretManager.encryptOrDecryptAuthenticationMechanism(botUser.getName(), authenticationMechanism, true)));
+    } catch (IOException e) {
+      throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
+    }
+  }
 }

openmetadata-service/src/main/java/org/openmetadata/service/util/AuthenticationMechanismBuilder.java

@@ -0,0 +1,67 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.openmetadata.service.util;
+
+import static org.openmetadata.schema.entity.teams.AuthenticationMechanism.AuthType.JWT;
+import static org.openmetadata.schema.entity.teams.AuthenticationMechanism.AuthType.SSO;
+
+import org.openmetadata.schema.auth.JWTAuthMechanism;
+import org.openmetadata.schema.auth.SSOAuthMechanism;
+import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
+import org.openmetadata.schema.security.client.Auth0SSOClientConfig;
+import org.openmetadata.schema.security.client.AzureSSOClientConfig;
+import org.openmetadata.schema.security.client.CustomOIDCSSOClientConfig;
+import org.openmetadata.schema.security.client.GoogleSSOClientConfig;
+import org.openmetadata.schema.security.client.OktaSSOClientConfig;
+
+public class AuthenticationMechanismBuilder {
+
+  /**
+   * Build `AuthenticationMechanism` object with concrete class for the config which by definition it is a `Object`.
+   *
+   * @param authMechanism the auth mechanism object
+   * @return auth mechanism object with concrete classes
+   */
+  public static AuthenticationMechanism build(AuthenticationMechanism authMechanism) {
+    if (authMechanism != null) {
+      if (JWT.equals(authMechanism.getAuthType())) {
+        authMechanism.setConfig(JsonUtils.convertValue(authMechanism.getConfig(), JWTAuthMechanism.class));
+      } else if (SSO.equals(authMechanism.getAuthType())) {
+        SSOAuthMechanism ssoAuth = JsonUtils.convertValue(authMechanism.getConfig(), SSOAuthMechanism.class);
+        switch (ssoAuth.getSsoServiceType()) {
+          case GOOGLE:
+            ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), GoogleSSOClientConfig.class));
+            break;
+          case OKTA:
+            ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), OktaSSOClientConfig.class));
+            break;
+          case AUTH_0:
+            ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), Auth0SSOClientConfig.class));
+            break;
+          case CUSTOM_OIDC:
+            ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), CustomOIDCSSOClientConfig.class));
+            break;
+          case AZURE:
+            ssoAuth.setAuthConfig(JsonUtils.convertValue(ssoAuth.getAuthConfig(), AzureSSOClientConfig.class));
+            break;
+          default:
+            throw new IllegalArgumentException(
+                String.format("SSO service type [%s] can not be parsed.", ssoAuth.getSsoServiceType()));
+        }
+        authMechanism.setConfig(ssoAuth);
+      }
+    }
+    return authMechanism;
+  }
+}

openmetadata-service/src/test/java/org/openmetadata/service/secrets/AWSSSMSecretsManagerTest.java

@@ -12,16 +12,11 @@
  */
 package org.openmetadata.service.secrets;
 
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.reset;
 
 import org.mockito.Mock;
 import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
 import software.amazon.awssdk.services.ssm.SsmClient;
-import software.amazon.awssdk.services.ssm.model.GetParameterRequest;
-import software.amazon.awssdk.services.ssm.model.GetParameterResponse;
-import software.amazon.awssdk.services.ssm.model.Parameter;
 
 public class AWSSSMSecretsManagerTest extends ExternalSecretsManagerTest {
 
@@ -34,21 +29,8 @@ public class AWSSSMSecretsManagerTest extends ExternalSecretsManagerTest {
     reset(ssmClient);
   }
 
-  @Override
-  void mockClientGetValue(String value) {
-    if (value == null) {
-      lenient()
-          .when(ssmClient.getParameter(any(GetParameterRequest.class)))
-          .thenReturn(GetParameterResponse.builder().build());
-    } else {
-      lenient()
-          .when(ssmClient.getParameter(any(GetParameterRequest.class)))
-          .thenReturn(GetParameterResponse.builder().parameter(Parameter.builder().value(value).build()).build());
-    }
-  }
-
   @Override
   SecretsManagerProvider expectedSecretManagerProvider() {
-    return SecretsManagerProvider.AWS_SSM;
+    return SecretsManagerProvider.MANAGED_AWS_SSM;
   }
 }

openmetadata-service/src/test/java/org/openmetadata/service/secrets/AWSSecretsManagerTest.java

@@ -12,15 +12,11 @@
  */
 package org.openmetadata.service.secrets;
 
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.reset;
 
 import org.mockito.Mock;
 import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
 import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
-import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest;
-import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse;
 
 public class AWSSecretsManagerTest extends ExternalSecretsManagerTest {
 
@@ -33,21 +29,8 @@ public class AWSSecretsManagerTest extends ExternalSecretsManagerTest {
     reset(secretsManagerClient);
   }
 
-  @Override
-  void mockClientGetValue(String value) {
-    if (value == null) {
-      lenient()
-          .when(secretsManagerClient.getSecretValue(any(GetSecretValueRequest.class)))
-          .thenReturn(GetSecretValueResponse.builder().build());
-    } else {
-      lenient()
-          .when(secretsManagerClient.getSecretValue(any(GetSecretValueRequest.class)))
-          .thenReturn(GetSecretValueResponse.builder().secretString(value).build());
-    }
-  }
-
   @Override
   SecretsManagerProvider expectedSecretManagerProvider() {
-    return SecretsManagerProvider.AWS;
+    return SecretsManagerProvider.MANAGED_AWS;
   }
 }

openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java

@@ -21,22 +21,26 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.openmetadata.schema.api.services.CreateDatabaseService;
+import org.openmetadata.schema.auth.SSOAuthMechanism;
 import org.openmetadata.schema.entity.services.ServiceType;
+import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
+import org.openmetadata.schema.security.client.OktaSSOClientConfig;
 import org.openmetadata.schema.services.connections.database.MysqlConnection;
 import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
+import org.openmetadata.service.fernet.Fernet;
 
 @ExtendWith(MockitoExtension.class)
 public abstract class ExternalSecretsManagerTest {
 
   static final boolean DECRYPT = false;
-  static final String EXPECTED_CONNECTION_JSON =
+  static final boolean ENCRYPT = true;
-      "{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"password\":\"openmetadata-test\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true,\"supportsQueryComment\":true}";
-  static final String EXPECTED_SECRET_ID = "/openmetadata/service/database/mysql/test";
 
   AWSBasedSecretsManager secretsManager;
 
   @BeforeEach
   void setUp() {
+    Fernet fernet = Fernet.getInstance();
+    fernet.setFernetKey("jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=");
     Map<String, String> parameters = new HashMap<>();
     parameters.put("region", "eu-west-1");
     parameters.put("accessKeyId", "123456");
@@ -48,10 +52,24 @@ public abstract class ExternalSecretsManagerTest {
 
   @Test
   void testDecryptDatabaseServiceConnectionConfig() {
-    mockClientGetValue(EXPECTED_CONNECTION_JSON);
     testEncryptDecryptServiceConnection(DECRYPT);
   }
 
+  @Test
+  void testEncryptDatabaseServiceConnectionConfig() {
+    testEncryptDecryptServiceConnection(ENCRYPT);
+  }
+
+  @Test
+  void testDecryptSSOConfig() {
+    testEncryptDecryptSSOConfig(DECRYPT);
+  }
+
+  @Test
+  void testEncryptSSSOConfig() {
+    testEncryptDecryptSSOConfig(ENCRYPT);
+  }
+
   @Test
   void testReturnsExpectedSecretManagerProvider() {
     assertEquals(expectedSecretManagerProvider(), secretsManager.getSecretsManagerProvider());
@@ -59,20 +77,39 @@ public abstract class ExternalSecretsManagerTest {
 
   abstract void setUpSpecific(SecretsManagerConfiguration config);
 
-  abstract void mockClientGetValue(String value);
-
   void testEncryptDecryptServiceConnection(boolean decrypt) {
     MysqlConnection mysqlConnection = new MysqlConnection();
     mysqlConnection.setPassword("openmetadata-test");
     CreateDatabaseService.DatabaseServiceType databaseServiceType = CreateDatabaseService.DatabaseServiceType.Mysql;
     String connectionName = "test";
 
-    Object actualConfig =
+    MysqlConnection actualConfig =
-        secretsManager.encryptOrDecryptServiceConnectionConfig(
+        (MysqlConnection)
-            mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt);
+            secretsManager.encryptOrDecryptServiceConnectionConfig(
+                mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt);
+
+    if (decrypt) {
+      mysqlConnection.setPassword("secret:/openmetadata/database/test/password");
+      actualConfig.setPassword(Fernet.getInstance().decrypt(actualConfig.getPassword()));
+    }
 
     assertEquals(mysqlConnection, actualConfig);
   }
 
+  void testEncryptDecryptSSOConfig(boolean decrypt) {
+    OktaSSOClientConfig config = new OktaSSOClientConfig();
+    config.setPrivateKey("this-is-a-test");
+    AuthenticationMechanism authenticationMechanism =
+        new AuthenticationMechanism()
+            .withAuthType(AuthenticationMechanism.AuthType.SSO)
+            .withConfig(
+                new SSOAuthMechanism().withAuthConfig(config).withSsoServiceType(SSOAuthMechanism.SsoServiceType.OKTA));
+
+    AuthenticationMechanism actualAuthenticationMechanism =
+        secretsManager.encryptOrDecryptAuthenticationMechanism("bot", authenticationMechanism, decrypt);
+
+    assertEquals(authenticationMechanism, actualAuthenticationMechanism);
+  }
+
   abstract SecretsManagerProvider expectedSecretManagerProvider();
 }