Change combined EncryptDecrypt methods to separate Encrypt and Decrpyt methods (#11755)

This commit is contained in:
Suresh Srinivas 2023-05-25 12:35:25 -07:00 committed by GitHub
parent 2585c13d33
commit f0f64a7b21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 265 additions and 300 deletions

View File

@ -105,10 +105,9 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager(); SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
if (secretsManager != null) { if (secretsManager != null) {
secretsManager.encryptOrDecryptIngestionPipeline(ingestionPipeline, true); secretsManager.encryptIngestionPipeline(ingestionPipeline);
// We store the OM sensitive values in SM separately // We store the OM sensitive values in SM separately
openmetadataConnection = openmetadataConnection = secretsManager.encryptOpenMetadataConnection(openmetadataConnection, true);
secretsManager.encryptOrDecryptOpenMetadataConnection(openmetadataConnection, true, true);
} }
ingestionPipeline.withService(null).withOpenMetadataServerConnection(null); ingestionPipeline.withService(null).withOpenMetadataServerConnection(null);
@ -298,7 +297,7 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
private static IngestionPipeline buildIngestionPipelineDecrypted(IngestionPipeline original) { private static IngestionPipeline buildIngestionPipelineDecrypted(IngestionPipeline original) {
IngestionPipeline decrypted = JsonUtils.convertValue(JsonUtils.getMap(original), IngestionPipeline.class); IngestionPipeline decrypted = JsonUtils.convertValue(JsonUtils.getMap(original), IngestionPipeline.class);
SecretsManagerFactory.getSecretsManager().encryptOrDecryptIngestionPipeline(decrypted, false); SecretsManagerFactory.getSecretsManager().decryptIngestionPipeline(decrypted);
return decrypted; return decrypted;
} }

View File

@ -72,12 +72,11 @@ public abstract class ServiceEntityRepository<
.getConnection() .getConnection()
.setConfig( .setConfig(
SecretsManagerFactory.getSecretsManager() SecretsManagerFactory.getSecretsManager()
.encryptOrDecryptServiceConnectionConfig( .encryptServiceConnectionConfig(
service.getConnection().getConfig(), service.getConnection().getConfig(),
service.getServiceType().value(), service.getServiceType().value(),
service.getName(), service.getName(),
serviceType, serviceType));
true));
} }
@Override @Override
@ -125,19 +124,11 @@ public abstract class ServiceEntityRepository<
S decryptedUpdatedConn = JsonUtils.readValue(updatedJson, serviceConnectionClass); S decryptedUpdatedConn = JsonUtils.readValue(updatedJson, serviceConnectionClass);
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager(); SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
decryptedOrigConn.setConfig( decryptedOrigConn.setConfig(
secretsManager.encryptOrDecryptServiceConnectionConfig( secretsManager.decryptServiceConnectionConfig(
decryptedOrigConn.getConfig(), decryptedOrigConn.getConfig(), original.getServiceType().value(), serviceType));
original.getServiceType().value(),
original.getName(),
serviceType,
false));
decryptedUpdatedConn.setConfig( decryptedUpdatedConn.setConfig(
secretsManager.encryptOrDecryptServiceConnectionConfig( secretsManager.decryptServiceConnectionConfig(
decryptedUpdatedConn.getConfig(), decryptedUpdatedConn.getConfig(), updated.getServiceType().value(), serviceType));
updated.getServiceType().value(),
updated.getName(),
serviceType,
false));
if (!objectMatch.test(decryptedOrigConn, decryptedUpdatedConn)) { if (!objectMatch.test(decryptedOrigConn, decryptedUpdatedConn)) {
// we don't want save connection config details in our database // we don't want save connection config details in our database
recordChange("connection", "old-encrypted-value", "new-encrypted-value", true); recordChange("connection", "old-encrypted-value", "new-encrypted-value", true);

View File

@ -127,7 +127,7 @@ public class UserRepository extends EntityRepository<User> {
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager(); SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
if (secretsManager != null && Boolean.TRUE.equals(user.getIsBot())) { if (secretsManager != null && Boolean.TRUE.equals(user.getIsBot())) {
secretsManager.encryptOrDecryptAuthenticationMechanism(user.getName(), user.getAuthenticationMechanism(), true); secretsManager.encryptAuthenticationMechanism(user.getName(), user.getAuthenticationMechanism());
} }
store(user, update); store(user, update);

View File

@ -49,7 +49,7 @@ public class WorkflowRepository extends EntityRepository<Workflow> {
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager(); SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
if (secretsManager != null) { if (secretsManager != null) {
entity = secretsManager.encryptOrDecryptWorkflow(entity, true); entity = secretsManager.encryptWorkflow(entity);
} }
// Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships // Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships

View File

@ -511,11 +511,11 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
} }
return workflowConverted; return workflowConverted;
} }
Workflow workflowDecrypted = secretsManager.encryptOrDecryptWorkflow(workflow, false); Workflow workflowDecrypted = secretsManager.decryptWorkflow(workflow);
OpenMetadataConnection openMetadataServerConnection = OpenMetadataConnection openMetadataServerConnection =
new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build(); new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build();
workflowDecrypted.setOpenMetadataServerConnection( workflowDecrypted.setOpenMetadataServerConnection(
secretsManager.encryptOrDecryptOpenMetadataConnection(openMetadataServerConnection, true, false)); secretsManager.encryptOpenMetadataConnection(openMetadataServerConnection, false));
if (authorizer.shouldMaskPasswords(securityContext)) { if (authorizer.shouldMaskPasswords(securityContext)) {
workflowDecrypted = EntityMaskerFactory.getEntityMasker().maskWorkflow(workflowDecrypted); workflowDecrypted = EntityMaskerFactory.getEntityMasker().maskWorkflow(workflowDecrypted);
} }

View File

@ -62,8 +62,8 @@ public abstract class ServiceEntityResource<
private Object retrieveServiceConnectionConfig(T service, boolean maskPassword) { private Object retrieveServiceConnectionConfig(T service, boolean maskPassword) {
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager(); SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
Object config = Object config =
secretsManager.encryptOrDecryptServiceConnectionConfig( secretsManager.decryptServiceConnectionConfig(
service.getConnection().getConfig(), extractServiceType(service), service.getName(), serviceType, false); service.getConnection().getConfig(), extractServiceType(service), serviceType);
if (maskPassword) { if (maskPassword) {
config = config =
EntityMaskerFactory.getEntityMasker() EntityMaskerFactory.getEntityMasker()

View File

@ -853,11 +853,11 @@ public class IngestionPipelineResource extends EntityResource<IngestionPipeline,
} catch (AuthorizationException | IOException e) { } catch (AuthorizationException | IOException e) {
ingestionPipeline.getSourceConfig().setConfig(null); ingestionPipeline.getSourceConfig().setConfig(null);
} }
secretsManager.encryptOrDecryptIngestionPipeline(ingestionPipeline, false); secretsManager.decryptIngestionPipeline(ingestionPipeline);
OpenMetadataConnection openMetadataServerConnection = OpenMetadataConnection openMetadataServerConnection =
new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build(); new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build();
ingestionPipeline.setOpenMetadataServerConnection( ingestionPipeline.setOpenMetadataServerConnection(
secretsManager.encryptOrDecryptOpenMetadataConnection(openMetadataServerConnection, true, false)); secretsManager.encryptOpenMetadataConnection(openMetadataServerConnection, false));
if (authorizer.shouldMaskPasswords(securityContext) && !forceNotMask) { if (authorizer.shouldMaskPasswords(securityContext) && !forceNotMask) {
EntityMaskerFactory.getEntityMasker().maskIngestionPipeline(ingestionPipeline); EntityMaskerFactory.getEntityMasker().maskIngestionPipeline(ingestionPipeline);
} }

View File

@ -1402,7 +1402,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
} catch (AuthorizationException | IOException e) { } catch (AuthorizationException | IOException e) {
user.getAuthenticationMechanism().setConfig(null); user.getAuthenticationMechanism().setConfig(null);
} }
secretsManager.encryptOrDecryptAuthenticationMechanism(user.getName(), user.getAuthenticationMechanism(), false); secretsManager.decryptAuthenticationMechanism(user.getName(), user.getAuthenticationMechanism());
if (authorizer.shouldMaskPasswords(securityContext)) { if (authorizer.shouldMaskPasswords(securityContext)) {
EntityMaskerFactory.getEntityMasker() EntityMaskerFactory.getEntityMasker()
.maskAuthenticationMechanism(user.getName(), user.getAuthenticationMechanism()); .maskAuthenticationMechanism(user.getName(), user.getAuthenticationMechanism());

View File

@ -54,15 +54,14 @@ public abstract class SecretsManager {
this.fernet = Fernet.getInstance(); this.fernet = Fernet.getInstance();
} }
public Object encryptOrDecryptServiceConnectionConfig( public Object encryptServiceConnectionConfig(
Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt) { Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType) {
try { try {
Class<?> clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType); Class<?> clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType);
Object newConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig); Object newConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig);
return encryptOrDecryptPasswordFields( return encryptPasswordFields(newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), true);
newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt, true);
} catch (Exception e) { } catch (Exception e) {
String message = SecretsUtil.buildExceptionMessageConnection(e.getMessage(), connectionType, encrypt); String message = SecretsUtil.buildExceptionMessageConnection(e.getMessage(), connectionType, true);
if (message != null) { if (message != null) {
throw new InvalidServiceConnectionException(message); throw new InvalidServiceConnectionException(message);
} }
@ -71,12 +70,27 @@ public abstract class SecretsManager {
} }
} }
public void encryptOrDecryptAuthenticationMechanism( public Object decryptServiceConnectionConfig(
String name, AuthenticationMechanism authenticationMechanism, boolean encrypt) { Object connectionConfig, String connectionType, ServiceType serviceType) {
try {
Class<?> clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType);
Object newConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig);
return decryptPasswordFields(newConnectionConfig);
} catch (Exception e) {
String message = SecretsUtil.buildExceptionMessageConnection(e.getMessage(), connectionType, false);
if (message != null) {
throw new InvalidServiceConnectionException(message);
}
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to encrypt connection instance of %s", connectionType));
}
}
public void encryptAuthenticationMechanism(String name, AuthenticationMechanism authenticationMechanism) {
if (authenticationMechanism != null) { if (authenticationMechanism != null) {
AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism); AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism);
try { try {
encryptOrDecryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), encrypt, true); encryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), true);
} catch (Exception e) { } catch (Exception e) {
throw new CustomExceptionMessage( throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, String.format("Failed to encrypt user bot instance [%s]", name)); Response.Status.BAD_REQUEST, String.format("Failed to encrypt user bot instance [%s]", name));
@ -84,15 +98,26 @@ public abstract class SecretsManager {
} }
} }
public void encryptOrDecryptIngestionPipeline(IngestionPipeline ingestionPipeline, boolean encrypt) { public void decryptAuthenticationMechanism(String name, AuthenticationMechanism authenticationMechanism) {
if (authenticationMechanism != null) {
AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism);
try {
decryptPasswordFields(authenticationMechanism);
} catch (Exception e) {
throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, String.format("Failed to encrypt user bot instance [%s]", name));
}
}
}
public void encryptIngestionPipeline(IngestionPipeline ingestionPipeline) {
OpenMetadataConnection openMetadataConnection = OpenMetadataConnection openMetadataConnection =
encryptOrDecryptOpenMetadataConnection(ingestionPipeline.getOpenMetadataServerConnection(), encrypt, true); encryptOpenMetadataConnection(ingestionPipeline.getOpenMetadataServerConnection(), true);
ingestionPipeline.setOpenMetadataServerConnection(null); ingestionPipeline.setOpenMetadataServerConnection(null);
// we don't store OM conn sensitive data // we don't store OM conn sensitive data
IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline); IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline);
try { try {
encryptOrDecryptPasswordFields( encryptPasswordFields(ingestionPipeline, buildSecretId(true, "pipeline", ingestionPipeline.getName()), true);
ingestionPipeline, buildSecretId(true, "pipeline", ingestionPipeline.getName()), encrypt, true);
} catch (Exception e) { } catch (Exception e) {
throw new CustomExceptionMessage( throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, Response.Status.BAD_REQUEST,
@ -101,15 +126,30 @@ public abstract class SecretsManager {
ingestionPipeline.setOpenMetadataServerConnection(openMetadataConnection); ingestionPipeline.setOpenMetadataServerConnection(openMetadataConnection);
} }
public Workflow encryptOrDecryptWorkflow(Workflow workflow, boolean encrypt) { public void decryptIngestionPipeline(IngestionPipeline ingestionPipeline) {
OpenMetadataConnection openMetadataConnection = OpenMetadataConnection openMetadataConnection =
encryptOrDecryptOpenMetadataConnection(workflow.getOpenMetadataServerConnection(), encrypt, true); decryptOpenMetadataConnection(ingestionPipeline.getOpenMetadataServerConnection(), true);
ingestionPipeline.setOpenMetadataServerConnection(null);
// we don't store OM conn sensitive data
IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline);
try {
decryptPasswordFields(ingestionPipeline);
} catch (Exception e) {
throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST,
String.format("Failed to encrypt ingestion pipeline instance [%s]", ingestionPipeline.getName()));
}
ingestionPipeline.setOpenMetadataServerConnection(openMetadataConnection);
}
public Workflow encryptWorkflow(Workflow workflow) {
OpenMetadataConnection openMetadataConnection =
encryptOpenMetadataConnection(workflow.getOpenMetadataServerConnection(), true);
Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow); Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow);
// we don't store OM conn sensitive data // we don't store OM conn sensitive data
workflowConverted.setOpenMetadataServerConnection(null); workflowConverted.setOpenMetadataServerConnection(null);
try { try {
encryptOrDecryptPasswordFields( encryptPasswordFields(workflowConverted, buildSecretId(true, "workflow", workflow.getName()), true);
workflowConverted, buildSecretId(true, "workflow", workflow.getName()), encrypt, true);
} catch (Exception e) { } catch (Exception e) {
throw new CustomExceptionMessage( throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, String.format("Failed to encrypt workflow instance [%s]", workflow.getName())); Response.Status.BAD_REQUEST, String.format("Failed to encrypt workflow instance [%s]", workflow.getName()));
@ -118,15 +158,30 @@ public abstract class SecretsManager {
return workflowConverted; return workflowConverted;
} }
public OpenMetadataConnection encryptOrDecryptOpenMetadataConnection( public Workflow decryptWorkflow(Workflow workflow) {
OpenMetadataConnection openMetadataConnection, boolean encrypt, boolean store) { OpenMetadataConnection openMetadataConnection =
decryptOpenMetadataConnection(workflow.getOpenMetadataServerConnection(), true);
Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow);
// we don't store OM conn sensitive data
workflowConverted.setOpenMetadataServerConnection(null);
try {
decryptPasswordFields(workflowConverted);
} catch (Exception e) {
throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, String.format("Failed to encrypt workflow instance [%s]", workflow.getName()));
}
workflowConverted.setOpenMetadataServerConnection(openMetadataConnection);
return workflowConverted;
}
public OpenMetadataConnection encryptOpenMetadataConnection(
OpenMetadataConnection openMetadataConnection, boolean store) {
if (openMetadataConnection != null) { if (openMetadataConnection != null) {
OpenMetadataConnection openMetadataConnectionConverted = OpenMetadataConnection openMetadataConnectionConverted =
(OpenMetadataConnection) (OpenMetadataConnection)
ClassConverterFactory.getConverter(OpenMetadataConnection.class).convert(openMetadataConnection); ClassConverterFactory.getConverter(OpenMetadataConnection.class).convert(openMetadataConnection);
try { try {
encryptOrDecryptPasswordFields( encryptPasswordFields(openMetadataConnectionConverted, buildSecretId(true, "serverconnection"), store);
openMetadataConnectionConverted, buildSecretId(true, "serverconnection"), encrypt, store);
} catch (Exception e) { } catch (Exception e) {
throw new CustomExceptionMessage( throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, "Failed to encrypt OpenMetadataConnection instance."); Response.Status.BAD_REQUEST, "Failed to encrypt OpenMetadataConnection instance.");
@ -136,16 +191,24 @@ public abstract class SecretsManager {
return null; return null;
} }
private Object encryptOrDecryptPasswordFields(Object targetObject, String name, boolean encrypt, boolean store) { public OpenMetadataConnection decryptOpenMetadataConnection(
if (encrypt) { OpenMetadataConnection openMetadataConnection, boolean store) {
encryptPasswordFields(targetObject, name, store); if (openMetadataConnection != null) {
} else { OpenMetadataConnection openMetadataConnectionConverted =
decryptPasswordFields(targetObject); (OpenMetadataConnection)
ClassConverterFactory.getConverter(OpenMetadataConnection.class).convert(openMetadataConnection);
try {
decryptPasswordFields(openMetadataConnectionConverted);
} catch (Exception e) {
throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, "Failed to encrypt OpenMetadataConnection instance.");
} }
return targetObject; return openMetadataConnectionConverted;
}
return null;
} }
private void encryptPasswordFields(Object toEncryptObject, String secretId, boolean store) { private Object encryptPasswordFields(Object toEncryptObject, String secretId, boolean store) {
if (!DO_NOT_ENCRYPT_CLASSES.contains(toEncryptObject.getClass())) { if (!DO_NOT_ENCRYPT_CLASSES.contains(toEncryptObject.getClass())) {
// for each get method // for each get method
Arrays.stream(toEncryptObject.getClass().getMethods()) Arrays.stream(toEncryptObject.getClass().getMethods())
@ -174,9 +237,10 @@ public abstract class SecretsManager {
} }
}); });
} }
return toEncryptObject;
} }
private void decryptPasswordFields(Object toDecryptObject) { private Object decryptPasswordFields(Object toDecryptObject) {
// for each get method // for each get method
Arrays.stream(toDecryptObject.getClass().getMethods()) Arrays.stream(toDecryptObject.getClass().getMethods())
.filter(ReflectionUtil::isGetMethodOfObject) .filter(ReflectionUtil::isGetMethodOfObject)
@ -198,6 +262,7 @@ public abstract class SecretsManager {
toDecryptObject, Fernet.isTokenized(fieldValue) ? fernet.decrypt(fieldValue) : fieldValue, toSet); toDecryptObject, Fernet.isTokenized(fieldValue) ? fernet.decrypt(fieldValue) : fieldValue, toSet);
} }
}); });
return toDecryptObject;
} }
protected abstract String storeValue(String fieldName, String value, String secretId, boolean store); protected abstract String storeValue(String fieldName, String value, String secretId, boolean store);

View File

@ -117,21 +117,16 @@ public class SecretsManagerUpdateService {
service service
.getConnection() .getConnection()
.setConfig( .setConfig(
oldSecretManager.encryptOrDecryptServiceConnectionConfig( oldSecretManager.decryptServiceConnectionConfig(
service.getConnection().getConfig(), service.getConnection().getConfig(), service.getServiceType().value(), repository.getServiceType()));
service.getServiceType().value(),
service.getName(),
repository.getServiceType(),
false));
service service
.getConnection() .getConnection()
.setConfig( .setConfig(
secretManager.encryptOrDecryptServiceConnectionConfig( secretManager.encryptServiceConnectionConfig(
service.getConnection().getConfig(), service.getConnection().getConfig(),
service.getServiceType().value(), service.getServiceType().value(),
service.getName(), service.getName(),
repository.getServiceType(), repository.getServiceType()));
true));
repository.dao.update(service); repository.dao.update(service);
} catch (IOException e) { } catch (IOException e) {
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause()); throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
@ -218,9 +213,8 @@ public class SecretsManagerUpdateService {
private void updateBotUser(User botUser) { private void updateBotUser(User botUser) {
try { try {
User user = userRepository.dao.findEntityById(botUser.getId()); User user = userRepository.dao.findEntityById(botUser.getId());
oldSecretManager.encryptOrDecryptAuthenticationMechanism( oldSecretManager.decryptAuthenticationMechanism(botUser.getName(), user.getAuthenticationMechanism());
botUser.getName(), user.getAuthenticationMechanism(), false); secretManager.encryptAuthenticationMechanism(botUser.getName(), user.getAuthenticationMechanism());
secretManager.encryptOrDecryptAuthenticationMechanism(botUser.getName(), user.getAuthenticationMechanism(), true);
userRepository.dao.update(user); userRepository.dao.update(user);
} catch (IOException e) { } catch (IOException e) {
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause()); throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
@ -261,8 +255,8 @@ public class SecretsManagerUpdateService {
try { try {
IngestionPipeline ingestion = ingestionPipelineRepository.dao.findEntityById(ingestionPipeline.getId()); IngestionPipeline ingestion = ingestionPipelineRepository.dao.findEntityById(ingestionPipeline.getId());
// we have to decrypt using the old secrets manager and encrypt again with the new one // we have to decrypt using the old secrets manager and encrypt again with the new one
oldSecretManager.encryptOrDecryptIngestionPipeline(ingestionPipeline, false); oldSecretManager.decryptIngestionPipeline(ingestionPipeline);
secretManager.encryptOrDecryptIngestionPipeline(ingestionPipeline, true); secretManager.encryptIngestionPipeline(ingestionPipeline);
ingestionPipelineRepository.dao.update(ingestion); ingestionPipelineRepository.dao.update(ingestion);
} catch (IOException e) { } catch (IOException e) {
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause()); throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());
@ -273,8 +267,8 @@ public class SecretsManagerUpdateService {
try { try {
Workflow workflowObject = workflowRepository.dao.findEntityById(workflow.getId()); Workflow workflowObject = workflowRepository.dao.findEntityById(workflow.getId());
// we have to decrypt using the old secrets manager and encrypt again with the new one // we have to decrypt using the old secrets manager and encrypt again with the new one
workflowObject = oldSecretManager.encryptOrDecryptWorkflow(workflowObject, false); workflowObject = oldSecretManager.decryptWorkflow(workflowObject);
workflowObject = secretManager.encryptOrDecryptWorkflow(workflowObject, true); workflowObject = secretManager.encryptWorkflow(workflowObject);
ingestionPipelineRepository.dao.update(workflowObject); ingestionPipelineRepository.dao.update(workflowObject);
} catch (IOException e) { } catch (IOException e) {
throw new SecretsManagerUpdateException(e.getMessage(), e.getCause()); throw new SecretsManagerUpdateException(e.getMessage(), e.getCause());

View File

@ -403,10 +403,6 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
return createRequest(getEntityName(test)).withDescription("").withDisplayName(null).withOwner(null); return createRequest(getEntityName(test)).withDescription("").withDisplayName(null).withOwner(null);
} }
public final K createPutRequest(TestInfo test) {
return createPutRequest(getEntityName(test)).withDescription("").withDisplayName(null).withOwner(null);
}
public final K createRequest(TestInfo test, int index) { public final K createRequest(TestInfo test, int index) {
return createRequest(getEntityName(test, index)).withDescription("").withDisplayName(null).withOwner(null); return createRequest(getEntityName(test, index)).withDescription("").withDisplayName(null).withOwner(null);
} }
@ -421,27 +417,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
.withOwner(reduceEntityReference(owner)); .withOwner(reduceEntityReference(owner));
} }
public final K createPutRequest(String name, String description, String displayName, EntityReference owner) {
if (!supportsEmptyDescription && description == null) {
throw new IllegalArgumentException("Entity " + entityType + " does not support empty description");
}
return createPutRequest(name)
.withDescription(description)
.withDisplayName(displayName)
.withOwner(reduceEntityReference(owner));
}
public abstract K createRequest(String name); public abstract K createRequest(String name);
public K createPutRequest(String name) {
return createRequest(name);
}
// Add all possible relationships to check if the entity is missing any of them after deletion
public T beforeDeletion(TestInfo test, T entity) throws HttpResponseException {
return entity;
}
// Get container entity used in createRequest that has CONTAINS relationship to the entity created with this // Get container entity used in createRequest that has CONTAINS relationship to the entity created with this
// request has . For table, it is database. For database, it is databaseService. See Relationship.CONTAINS for // request has . For table, it is database. For database, it is databaseService. See Relationship.CONTAINS for
// details. // details.
@ -784,7 +761,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
} }
// Create an entity using POST // Create an entity using POST
K create = createRequest(test); K create = createRequest(test);
T entity = beforeDeletion(test, createEntity(create, ADMIN_AUTH_HEADERS)); T entity = createEntity(create, ADMIN_AUTH_HEADERS);
T entityBeforeDeletion = getEntity(entity.getId(), allFields, ADMIN_AUTH_HEADERS); T entityBeforeDeletion = getEntity(entity.getId(), allFields, ADMIN_AUTH_HEADERS);
// Soft delete the entity // Soft delete the entity
@ -1047,7 +1024,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
// Remove ownership (from USER_OWNER1) using PUT request. Owner is expected to remain the same // Remove ownership (from USER_OWNER1) using PUT request. Owner is expected to remain the same
// and not removed. // and not removed.
request = createPutRequest(entity.getName(), "description", "displayName", null); request = createRequest(entity.getName(), "description", "displayName", null);
updateEntity(request, OK, ADMIN_AUTH_HEADERS); updateEntity(request, OK, ADMIN_AUTH_HEADERS);
checkOwnerOwns(USER1_REF, entity.getId(), true); checkOwnerOwns(USER1_REF, entity.getId(), true);
} }
@ -1152,7 +1129,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
T entity = createEntity(request, ADMIN_AUTH_HEADERS); T entity = createEntity(request, ADMIN_AUTH_HEADERS);
// Update null description with a new description // Update null description with a new description
request = createPutRequest(entity.getName(), "updatedDescription", "displayName", null); request = createRequest(entity.getName(), "updatedDescription", "displayName", null);
ChangeDescription change = getChangeDescription(entity.getVersion()); ChangeDescription change = getChangeDescription(entity.getVersion());
fieldAdded(change, "description", "updatedDescription"); fieldAdded(change, "description", "updatedDescription");
updateAndCheckEntity(request, OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); updateAndCheckEntity(request, OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);

View File

@ -123,7 +123,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
.withPassword(password)); .withPassword(password));
CreateDashboardService update = CreateDashboardService update =
createPutRequest(test).withDescription("description1").withConnection(dashboardConnection1); createRequest(test).withDescription("description1").withConnection(dashboardConnection1);
ChangeDescription change = getChangeDescription(service.getVersion()); ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1"); fieldAdded(change, "description", "description1");
@ -147,7 +147,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
.withUsername("user") .withUsername("user")
.withPassword(password); .withPassword(password);
DashboardConnection dashboardConnection2 = new DashboardConnection().withConfig(metabaseConnection); DashboardConnection dashboardConnection2 = new DashboardConnection().withConfig(metabaseConnection);
update = createPutRequest(test).withDescription("description1").withConnection(dashboardConnection2); update = createRequest(test).withDescription("description1").withConnection(dashboardConnection2);
fieldUpdated(change, "connection", dashboardConnection1, dashboardConnection2); fieldUpdated(change, "connection", dashboardConnection1, dashboardConnection2);
updateAndCheckEntity(update, OK, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change); updateAndCheckEntity(update, OK, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);

View File

@ -129,7 +129,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
DatabaseService service = createAndCheckEntity(createRequest(test).withDescription(null), ADMIN_AUTH_HEADERS); DatabaseService service = createAndCheckEntity(createRequest(test).withDescription(null), ADMIN_AUTH_HEADERS);
// Update database description and ingestion service that are null // Update database description and ingestion service that are null
CreateDatabaseService update = createPutRequest(test).withDescription("description1"); CreateDatabaseService update = createRequest(test).withDescription("description1");
ChangeDescription change = getChangeDescription(service.getVersion()); ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1"); fieldAdded(change, "description", "description1");
@ -176,7 +176,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
"InvalidServiceConnectionException for service [Snowflake] due to [Failed to encrypt connection instance of Snowflake]"); "InvalidServiceConnectionException for service [Snowflake] due to [Failed to encrypt connection instance of Snowflake]");
DatabaseService service = createAndCheckEntity(createRequest(test).withDescription(null), ADMIN_AUTH_HEADERS); DatabaseService service = createAndCheckEntity(createRequest(test).withDescription(null), ADMIN_AUTH_HEADERS);
// Update database description and ingestion service that are null // Update database description and ingestion service that are null
CreateDatabaseService update = createPutRequest(test).withDescription("description1"); CreateDatabaseService update = createRequest(test).withDescription("description1");
ChangeDescription change = getChangeDescription(service.getVersion()); ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1"); fieldAdded(change, "description", "description1");
@ -287,10 +287,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
public void validateCreatedEntity( public void validateCreatedEntity(
DatabaseService service, CreateDatabaseService createRequest, Map<String, String> authHeaders) { DatabaseService service, CreateDatabaseService createRequest, Map<String, String> authHeaders) {
assertEquals(createRequest.getName(), service.getName()); assertEquals(createRequest.getName(), service.getName());
boolean maskPasswords = true; boolean maskPasswords = !INGESTION_BOT_AUTH_HEADERS.equals(authHeaders);
if (INGESTION_BOT_AUTH_HEADERS.equals(authHeaders)) {
maskPasswords = false;
}
validateDatabaseConnection( validateDatabaseConnection(
createRequest.getConnection(), service.getConnection(), service.getServiceType(), maskPasswords); createRequest.getConnection(), service.getConnection(), service.getServiceType(), maskPasswords);
} }

View File

@ -126,7 +126,7 @@ public class MetadataServiceResourceTest extends EntityResourceTest<MetadataServ
.withPassword(secretPassword)); .withPassword(secretPassword));
// Update metadata description // Update metadata description
CreateMetadataService update = CreateMetadataService update =
createPutRequest(test).withDescription("description1").withConnection(metadataConnection); createRequest(test).withDescription("description1").withConnection(metadataConnection);
ChangeDescription change = getChangeDescription(service.getVersion()); ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1"); fieldAdded(change, "description", "description1");
service = updateAndCheckEntity(update, OK, ADMIN_AUTH_HEADERS, TestUtils.UpdateType.MINOR_UPDATE, change); service = updateAndCheckEntity(update, OK, ADMIN_AUTH_HEADERS, TestUtils.UpdateType.MINOR_UPDATE, change);

View File

@ -13,6 +13,8 @@
package org.openmetadata.service.secrets; package org.openmetadata.service.secrets;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.openmetadata.schema.api.services.CreateDatabaseService.DatabaseServiceType.Mysql;
import java.util.Map; import java.util.Map;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
@ -20,7 +22,6 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import org.openmetadata.schema.api.services.CreateDatabaseService;
import org.openmetadata.schema.api.services.DatabaseConnection; import org.openmetadata.schema.api.services.DatabaseConnection;
import org.openmetadata.schema.auth.SSOAuthMechanism; import org.openmetadata.schema.auth.SSOAuthMechanism;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
@ -48,10 +49,6 @@ import org.openmetadata.service.util.JsonUtils;
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
public abstract class ExternalSecretsManagerTest { public abstract class ExternalSecretsManagerTest {
static final boolean DECRYPT = false;
static final boolean ENCRYPT = true;
AWSBasedSecretsManager secretsManager; AWSBasedSecretsManager secretsManager;
@BeforeEach @BeforeEach
@ -68,57 +65,117 @@ public abstract class ExternalSecretsManagerTest {
} }
@Test @Test
void testDecryptDatabaseServiceConnectionConfig() { void testEncryptDecryptDatabaseServiceConnectionConfig() {
testEncryptDecryptServiceConnection(DECRYPT); String password = "openmetadata-test";
MysqlConnection expectedConnection = new MysqlConnection().withPassword(password);
Map<String, String> mysqlConnection = Map.of("password", password);
// Ensure encrypted service connection config encrypts the password
MysqlConnection actualConnection =
(MysqlConnection)
secretsManager.encryptServiceConnectionConfig(mysqlConnection, Mysql.value(), "test", ServiceType.DATABASE);
assertNotEquals(password, actualConnection.getPassword());
// Decrypt the encrypted password and validate
actualConnection =
(MysqlConnection)
secretsManager.decryptServiceConnectionConfig(mysqlConnection, Mysql.value(), ServiceType.DATABASE);
assertEquals(password, actualConnection.getPassword());
assertEquals(expectedConnection, actualConnection);
} }
@Test @Test
void testEncryptDatabaseServiceConnectionConfig() { void testEncryptDecryptSSSOConfig() {
testEncryptDecryptServiceConnection(ENCRYPT); String privateKey = "secret:/openmetadata/bot/bot/config/authconfig/privatekey";
OktaSSOClientConfig config = new OktaSSOClientConfig().withPrivateKey(privateKey);
AuthenticationMechanism expectedAuthMechanism =
new AuthenticationMechanism()
.withAuthType(AuthenticationMechanism.AuthType.SSO)
.withConfig(
new SSOAuthMechanism().withAuthConfig(config).withSsoServiceType(SSOAuthMechanism.SsoServiceType.OKTA));
AuthenticationMechanism actualAuthMechanism =
JsonUtils.convertValue(expectedAuthMechanism, AuthenticationMechanism.class);
// Encrypt private key and ensure it is indeed encrypted
secretsManager.encryptAuthenticationMechanism("bot", actualAuthMechanism);
assertNotEquals(privateKey, getPrivateKey(actualAuthMechanism));
System.out.println("XXX privateKey encrypted is " + getPrivateKey(actualAuthMechanism));
// Decrypt private key and ensure it is decrypted
secretsManager.decryptAuthenticationMechanism("bot", actualAuthMechanism);
System.out.println("XXX privateKey decrypted is " + getPrivateKey(actualAuthMechanism));
assertEquals(privateKey, getPrivateKey(actualAuthMechanism));
} }
@Test @Test
void testDecryptSSOConfig() { void testEncryptDecryptIngestionPipelineDBTConfig() {
testEncryptDecryptSSOConfig(DECRYPT); String secretKey =
"secret:/openmetadata/pipeline/my-pipeline/sourceconfig/config/dbtconfigsource"
+ "/dbtsecurityconfig/awssecretaccesskey";
AWSCredentials credentials = new AWSCredentials().withAwsSecretAccessKey(secretKey).withAwsRegion("eu-west-1");
DbtS3Config config = new DbtS3Config().withDbtSecurityConfig(credentials);
DbtPipeline dbtPipeline = new DbtPipeline().withDbtConfigSource(config);
SourceConfig sourceConfig = new SourceConfig().withConfig(dbtPipeline);
IngestionPipeline expectedIngestionPipeline =
new IngestionPipeline()
.withName("my-pipeline")
.withPipelineType(PipelineType.DBT)
.withService(new DatabaseService().getEntityReference().withType(Entity.DATABASE_SERVICE))
.withSourceConfig(sourceConfig);
IngestionPipeline actualIngestionPipeline =
JsonUtils.convertValue(expectedIngestionPipeline, IngestionPipeline.class);
// Encrypt the pipeline and make sure it is secret key encrypted
secretsManager.encryptIngestionPipeline(actualIngestionPipeline);
System.out.println("XXX encrypted aws secret access key is " + getAwsSecretAccessKey(actualIngestionPipeline));
assertNotEquals(secretKey, getAwsSecretAccessKey(actualIngestionPipeline));
// Decrypt the pipeline and make sure the secret key is decrypted
secretsManager.decryptIngestionPipeline(actualIngestionPipeline);
System.out.println("XXX decrypted aws secret access key is " + getAwsSecretAccessKey(actualIngestionPipeline));
assertEquals(secretKey, getAwsSecretAccessKey(actualIngestionPipeline));
assertEquals(expectedIngestionPipeline, actualIngestionPipeline);
} }
@Test @Test
void testEncryptSSSOConfig() { void testEncryptDecryptWorkflow() {
testEncryptDecryptSSOConfig(ENCRYPT); String password = "secret:/openmetadata/workflow/my-workflow/request/connection/config/password";
} String secretKey = "secret:/openmetadata/serverconnection/securityconfig/secretkey";
OpenMetadataConnection connection =
new OpenMetadataConnection().withSecurityConfig(new GoogleSSOClientConfig().withSecretKey(secretKey));
DatabaseConnection dbConnection = new DatabaseConnection().withConfig(new MysqlConnection().withPassword(password));
TestServiceConnectionRequest testRequest =
new TestServiceConnectionRequest()
.withConnection(dbConnection)
.withServiceType(ServiceType.DATABASE)
.withConnectionType("Mysql");
Workflow expectedWorkflow =
new Workflow().withName("my-workflow").withOpenMetadataServerConnection(connection).withRequest(testRequest);
Workflow actualWorkflow = JsonUtils.convertValue(expectedWorkflow, Workflow.class);
@Test // Encrypt the workflow and ensure password and secrete key are encrypted
void testDecryptIngestionPipelineDBTConfig() { actualWorkflow = secretsManager.encryptWorkflow(actualWorkflow);
testEncryptDecryptDBTConfig(DECRYPT); assertNotEquals(password, getPassword(actualWorkflow));
} assertNotEquals(secretKey, getSecretKey(actualWorkflow));
@Test // Decrypt the workflow and ensure password and secrete key are decrypted
void testEncryptIngestionPipelineDBTConfig() { actualWorkflow = secretsManager.decryptWorkflow(actualWorkflow);
testEncryptDecryptDBTConfig(ENCRYPT); assertEquals(password, getPassword(actualWorkflow));
} assertEquals(secretKey, getSecretKey(actualWorkflow));
assertEquals(expectedWorkflow, actualWorkflow);
@Test
void testDecryptWorkflow() {
testEncryptWorkflowObject(DECRYPT);
}
@Test
void testEncryptWorkflow() {
testEncryptWorkflowObject(ENCRYPT);
} }
@Test @Test
void testExceptionConnection() { void testExceptionConnection() {
CreateDatabaseService.DatabaseServiceType databaseServiceType = CreateDatabaseService.DatabaseServiceType.Mysql;
String connectionName = "test";
Map<String, String> mysqlConnection = Map.of("password", "openmetadata-test", "username1", "openmetadata-test"); Map<String, String> mysqlConnection = Map.of("password", "openmetadata-test", "username1", "openmetadata-test");
InvalidServiceConnectionException thrown = InvalidServiceConnectionException thrown =
Assertions.assertThrows( Assertions.assertThrows(
InvalidServiceConnectionException.class, InvalidServiceConnectionException.class,
() -> () ->
secretsManager.encryptOrDecryptServiceConnectionConfig( secretsManager.encryptServiceConnectionConfig(
mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, true)); mysqlConnection, Mysql.value(), "test", ServiceType.DATABASE));
Assertions.assertEquals( Assertions.assertEquals(
"Failed to encrypt 'Mysql' connection stored in DB due to an unrecognized field: 'username1'", "Failed to encrypt 'Mysql' connection stored in DB due to an unrecognized field: 'username1'",
@ -126,9 +183,7 @@ public abstract class ExternalSecretsManagerTest {
thrown = thrown =
Assertions.assertThrows( Assertions.assertThrows(
InvalidServiceConnectionException.class, InvalidServiceConnectionException.class,
() -> () -> secretsManager.decryptServiceConnectionConfig(mysqlConnection, Mysql.value(), ServiceType.DATABASE));
secretsManager.encryptOrDecryptServiceConnectionConfig(
mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, false));
Assertions.assertEquals( Assertions.assertEquals(
"Failed to decrypt 'Mysql' connection stored in DB due to an unrecognized field: 'username1'", "Failed to decrypt 'Mysql' connection stored in DB due to an unrecognized field: 'username1'",
@ -142,131 +197,25 @@ public abstract class ExternalSecretsManagerTest {
abstract void setUpSpecific(SecretsManagerConfiguration config); abstract void setUpSpecific(SecretsManagerConfiguration config);
void testEncryptDecryptServiceConnection(boolean decrypt) {
MysqlConnection expectedMysqlConnection = new MysqlConnection();
expectedMysqlConnection.setPassword("openmetadata-test");
CreateDatabaseService.DatabaseServiceType databaseServiceType = CreateDatabaseService.DatabaseServiceType.Mysql;
String connectionName = "test";
Map<String, String> mysqlConnection = Map.of("password", "openmetadata-test");
MysqlConnection actualMysqlConnection =
(MysqlConnection)
secretsManager.encryptOrDecryptServiceConnectionConfig(
mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt);
if (decrypt) {
expectedMysqlConnection.setPassword("secret:/openmetadata/database/test/password");
actualMysqlConnection.setPassword(Fernet.getInstance().decrypt(actualMysqlConnection.getPassword()));
}
assertEquals(expectedMysqlConnection, actualMysqlConnection);
}
void testEncryptDecryptSSOConfig(boolean decrypt) {
OktaSSOClientConfig config = new OktaSSOClientConfig();
config.setPrivateKey(decrypt ? "secret:/openmetadata/bot/bot/config/authconfig/privatekey" : "this-is-a-test");
AuthenticationMechanism expectedAuthenticationMechanism =
new AuthenticationMechanism()
.withAuthType(AuthenticationMechanism.AuthType.SSO)
.withConfig(
new SSOAuthMechanism().withAuthConfig(config).withSsoServiceType(SSOAuthMechanism.SsoServiceType.OKTA));
AuthenticationMechanism actualAuthenticationMechanism =
JsonUtils.convertValue(expectedAuthenticationMechanism, AuthenticationMechanism.class);
secretsManager.encryptOrDecryptAuthenticationMechanism("bot", actualAuthenticationMechanism, decrypt);
if (decrypt) {
String privateKey =
((OktaSSOClientConfig) ((SSOAuthMechanism) actualAuthenticationMechanism.getConfig()).getAuthConfig())
.getPrivateKey();
((OktaSSOClientConfig) ((SSOAuthMechanism) actualAuthenticationMechanism.getConfig()).getAuthConfig())
.setPrivateKey(Fernet.getInstance().decrypt(privateKey));
}
assertEquals(expectedAuthenticationMechanism, actualAuthenticationMechanism);
}
void testEncryptDecryptDBTConfig(boolean decrypt) {
IngestionPipeline expectedIngestionPipeline =
new IngestionPipeline()
.withName("my-pipeline")
.withPipelineType(PipelineType.DBT)
.withService(new DatabaseService().getEntityReference().withType(Entity.DATABASE_SERVICE))
.withSourceConfig(
new SourceConfig()
.withConfig(
new DbtPipeline()
.withDbtConfigSource(
new DbtS3Config()
.withDbtSecurityConfig(
new AWSCredentials()
.withAwsSecretAccessKey("secret-password")
.withAwsRegion("eu-west-1")))));
IngestionPipeline actualIngestionPipeline =
JsonUtils.convertValue(expectedIngestionPipeline, IngestionPipeline.class);
secretsManager.encryptOrDecryptIngestionPipeline(actualIngestionPipeline, decrypt);
if (decrypt) {
DbtPipeline expectedDbtPipeline = ((DbtPipeline) expectedIngestionPipeline.getSourceConfig().getConfig());
DbtPipeline actualDbtPipeline = ((DbtPipeline) actualIngestionPipeline.getSourceConfig().getConfig());
((DbtS3Config) expectedDbtPipeline.getDbtConfigSource())
.getDbtSecurityConfig()
.setAwsSecretAccessKey(
"secret:/openmetadata/pipeline/my-pipeline/sourceconfig/config/dbtconfigsource/dbtsecurityconfig/awssecretaccesskey");
((DbtS3Config) actualDbtPipeline.getDbtConfigSource())
.getDbtSecurityConfig()
.setAwsSecretAccessKey(
Fernet.getInstance()
.decrypt(
((DbtS3Config) actualDbtPipeline.getDbtConfigSource())
.getDbtSecurityConfig()
.getAwsSecretAccessKey()));
}
assertEquals(expectedIngestionPipeline, actualIngestionPipeline);
}
void testEncryptWorkflowObject(boolean encrypt) {
Workflow expectedWorkflow =
new Workflow()
.withName("my-workflow")
.withOpenMetadataServerConnection(
new OpenMetadataConnection()
.withSecurityConfig(new GoogleSSOClientConfig().withSecretKey("google-secret")))
.withRequest(
new TestServiceConnectionRequest()
.withConnection(
new DatabaseConnection().withConfig(new MysqlConnection().withPassword("openmetadata-test")))
.withServiceType(ServiceType.DATABASE)
.withConnectionType("Mysql"));
Workflow workflow = JsonUtils.convertValue(expectedWorkflow, Workflow.class);
Workflow actualWorkflow = secretsManager.encryptOrDecryptWorkflow(workflow, encrypt);
if (encrypt) {
((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) expectedWorkflow.getRequest()).getConnection())
.getConfig())
.setPassword("secret:/openmetadata/workflow/my-workflow/request/connection/config/password");
MysqlConnection mysqlConnection =
(MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) actualWorkflow.getRequest()).getConnection())
.getConfig();
mysqlConnection.setPassword(Fernet.getInstance().decrypt(mysqlConnection.getPassword()));
((GoogleSSOClientConfig) (expectedWorkflow.getOpenMetadataServerConnection()).getSecurityConfig())
.setSecretKey("secret:/openmetadata/serverconnection/securityconfig/secretkey");
GoogleSSOClientConfig googleSSOClientConfig =
((GoogleSSOClientConfig) (actualWorkflow.getOpenMetadataServerConnection()).getSecurityConfig());
googleSSOClientConfig.setSecretKey(Fernet.getInstance().decrypt(googleSSOClientConfig.getSecretKey()));
}
assertEquals(expectedWorkflow, actualWorkflow);
}
protected abstract SecretsManagerProvider expectedSecretManagerProvider(); protected abstract SecretsManagerProvider expectedSecretManagerProvider();
private String getPrivateKey(AuthenticationMechanism authMechanism) {
return ((OktaSSOClientConfig) ((SSOAuthMechanism) authMechanism.getConfig()).getAuthConfig()).getPrivateKey();
}
private String getAwsSecretAccessKey(IngestionPipeline ingestionPipeline) {
DbtPipeline expectedDbtPipeline = ((DbtPipeline) ingestionPipeline.getSourceConfig().getConfig());
return ((DbtS3Config) expectedDbtPipeline.getDbtConfigSource()).getDbtSecurityConfig().getAwsSecretAccessKey();
}
private String getPassword(Workflow workflow) {
return ((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) workflow.getRequest()).getConnection()).getConfig())
.getPassword();
}
private String getSecretKey(Workflow expectedWorkflow) {
return ((GoogleSSOClientConfig) (expectedWorkflow.getOpenMetadataServerConnection()).getSecurityConfig())
.getSecretKey();
}
} }

View File

@ -16,6 +16,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.lenient;
import static org.openmetadata.schema.api.services.CreateDatabaseService.DatabaseServiceType.Mysql;
import static org.openmetadata.schema.api.services.CreateMlModelService.MlModelServiceType.Sklearn;
import static org.openmetadata.schema.entity.services.ServiceType.ML_MODEL;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
@ -23,8 +26,6 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import org.openmetadata.schema.api.services.CreateDatabaseService;
import org.openmetadata.schema.api.services.CreateMlModelService;
import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.security.secrets.SecretsManagerProvider; import org.openmetadata.schema.security.secrets.SecretsManagerProvider;
import org.openmetadata.schema.services.connections.database.MysqlConnection; import org.openmetadata.schema.services.connections.database.MysqlConnection;
@ -34,8 +35,6 @@ import org.openmetadata.service.fernet.Fernet;
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
public class NoopSecretsManagerTest { public class NoopSecretsManagerTest {
private static final boolean ENCRYPT = true;
private static final boolean DECRYPT = false;
private static final String ENCRYPTED_VALUE = "fernet:abcdef"; private static final String ENCRYPTED_VALUE = "fernet:abcdef";
private static final String DECRYPTED_VALUE = "123456"; private static final String DECRYPTED_VALUE = "123456";
private static NoopSecretsManager secretsManager; private static NoopSecretsManager secretsManager;
@ -58,22 +57,26 @@ public class NoopSecretsManagerTest {
@Test @Test
void testEncryptDatabaseServiceConnectionConfig() { void testEncryptDatabaseServiceConnectionConfig() {
testEncryptDecryptServiceConnection(ENCRYPT); testEncryptServiceConnection();
} }
@Test @Test
void testDecryptDatabaseServiceConnectionConfig() { void testDecryptDatabaseServiceConnectionConfig() {
testEncryptDecryptServiceConnection(DECRYPT); testDecryptServiceConnection();
} }
@Test @Test
void testEncryptServiceConnectionWithoutPassword() { void testEncryptServiceConnectionWithoutPassword() {
testEncryptDecryptServiceConnectionWithoutPassword(ENCRYPT); SklearnConnection connection = new SklearnConnection();
Object actualConfig = secretsManager.encryptServiceConnectionConfig(connection, Sklearn.value(), "test", ML_MODEL);
assertNotSame(connection, actualConfig);
} }
@Test @Test
void testEncryptDecryptServiceConnectionWithoutPassword() { void testDecryptServiceConnectionWithoutPassword() {
testEncryptDecryptServiceConnectionWithoutPassword(DECRYPT); SklearnConnection connection = new SklearnConnection();
Object actualConfig = secretsManager.decryptServiceConnectionConfig(connection, Sklearn.value(), ML_MODEL);
assertNotSame(connection, actualConfig);
} }
@Test @Test
@ -81,29 +84,19 @@ public class NoopSecretsManagerTest {
assertEquals(SecretsManagerProvider.NOOP, secretsManager.getSecretsManagerProvider()); assertEquals(SecretsManagerProvider.NOOP, secretsManager.getSecretsManagerProvider());
} }
private void testEncryptDecryptServiceConnectionWithoutPassword(boolean decrypt) { private void testEncryptServiceConnection() {
SklearnConnection sklearnConnection = new SklearnConnection(); MysqlConnection connection = new MysqlConnection().withPassword(ENCRYPTED_VALUE);
CreateMlModelService.MlModelServiceType databaseServiceType = CreateMlModelService.MlModelServiceType.Sklearn;
String connectionName = "test";
Object actualConfig = Object actualConfig =
secretsManager.encryptOrDecryptServiceConnectionConfig( secretsManager.encryptServiceConnectionConfig(connection, Mysql.value(), "test", ServiceType.DATABASE);
sklearnConnection, databaseServiceType.value(), connectionName, ServiceType.ML_MODEL, decrypt); assertEquals(ENCRYPTED_VALUE, ((MysqlConnection) actualConfig).getPassword());
assertNotSame(connection, actualConfig);
assertNotSame(sklearnConnection, actualConfig);
} }
private void testEncryptDecryptServiceConnection(boolean encrypt) { private void testDecryptServiceConnection() {
MysqlConnection mysqlConnection = new MysqlConnection(); MysqlConnection mysqlConnection = new MysqlConnection().withPassword(DECRYPTED_VALUE);
mysqlConnection.setPassword(encrypt ? ENCRYPTED_VALUE : DECRYPTED_VALUE);
CreateDatabaseService.DatabaseServiceType databaseServiceType = CreateDatabaseService.DatabaseServiceType.Mysql;
String connectionName = "test";
Object actualConfig = Object actualConfig =
secretsManager.encryptOrDecryptServiceConnectionConfig( secretsManager.decryptServiceConnectionConfig(mysqlConnection, Mysql.value(), ServiceType.DATABASE);
mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, encrypt); assertEquals(DECRYPTED_VALUE, ((MysqlConnection) actualConfig).getPassword());
assertEquals(encrypt ? ENCRYPTED_VALUE : DECRYPTED_VALUE, ((MysqlConnection) actualConfig).getPassword());
assertNotSame(mysqlConnection, actualConfig); assertNotSame(mysqlConnection, actualConfig);
} }
} }