yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-09 05:55:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix#8577: Services sensitive passwords fields are stored in secrets store (#8671)

Browse Source 
* Services sensitive passwords fields are stored in secrets store

* Update services on application startup in case of changes in the JSON Schema

* Minor changes after manual test
This commit is contained in:
Nahuel 2022-11-14 13:48:50 +01:00 committed by GitHub
parent 92781d7949
commit e5abdc8c5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 567 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java
View File

@ -19,6 +19,6 @@ public class OpenMetadataAnnotator extends CompositeAnnotator {
public OpenMetadataAnnotator() {
// we can add multiple annotators
super(new ExposedAnnotator(), new MaskedAnnotator());
super(new ExposedAnnotator(), new MaskedAnnotator(), new PasswordAnnotator());
}
}

80
common/src/main/java/org/openmetadata/annotations/PasswordAnnotator.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.annotations;
import com.fasterxml.jackson.databind.JsonNode;
import com.sun.codemodel.JAnnotationUse;
import com.sun.codemodel.JClass;
import com.sun.codemodel.JDefinedClass;
import com.sun.codemodel.JFieldVar;
import com.sun.codemodel.JMethod;
import java.lang.reflect.Field;
import org.jsonschema2pojo.AbstractAnnotator;
/** Add {@link PasswordField} annotation to generated Java classes */
public class PasswordAnnotator extends AbstractAnnotator {
/** Add {@link PasswordField} annotation property fields */
@Override
public void propertyField(JFieldVar field, JDefinedClass clazz, String propertyName, JsonNode propertyNode) {
super.propertyField(field, clazz, propertyName, propertyNode);
if (propertyNode.get("format") != null && "password".equals(propertyNode.get("format").asText())) {
field.annotate(PasswordField.class);
}
}
/** Add {@link PasswordField} annotation to getter methods */
@Override
public void propertyGetter(JMethod getter, JDefinedClass clazz, String propertyName) {
super.propertyGetter(getter, clazz, propertyName);
addMaskedFieldAnnotationIfApplies(getter, propertyName);
}
/** Add {@link PasswordField} annotation to setter methods */
@Override
public void propertySetter(JMethod setter, JDefinedClass clazz, String propertyName) {
super.propertySetter(setter, clazz, propertyName);
addMaskedFieldAnnotationIfApplies(setter, propertyName);
}
/**
* Use reflection methods to access the {@link JDefinedClass} of the {@link JMethod} object. If the {@link JMethod} is
* pointing to a field annotated with {@link PasswordField} then annotates the {@link JMethod} object with {@link
* PasswordField}
*/
private void addMaskedFieldAnnotationIfApplies(JMethod jMethod, String propertyName) {
try {
Field outerClassField = JMethod.class.getDeclaredField("outer");
outerClassField.setAccessible(true);
JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod);
if (outerClass.fields().containsKey(propertyName)
&& outerClass.fields().get(propertyName).annotations().stream()
.anyMatch(annotation -> PasswordField.class.getName().equals(getAnnotationClassName(annotation)))) {
jMethod.annotate(PasswordField.class);
}
} catch (NoSuchFieldException | IllegalAccessException e) {
throw new RuntimeException(e);
}
}
private String getAnnotationClassName(JAnnotationUse annotation) {
try {
Field clazzField = JAnnotationUse.class.getDeclaredField("clazz");
clazzField.setAccessible(true);
return ((JClass) clazzField.get(annotation)).fullName();
} catch (NoSuchFieldException | IllegalAccessException e) {
throw new RuntimeException(e);
}
}
}

27
common/src/main/java/org/openmetadata/annotations/PasswordField.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.annotations;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Specifies that the field or method is exposed, i.e., if the serialization will take into account those fields
* annotated with
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.METHOD, ElementType.ANNOTATION_TYPE})
public @interface PasswordField {}

6
ingestion/src/metadata/clients/aws_client.py
View File

@ -40,7 +40,11 @@ class AWSClient:
AWSCredentials,
)
self.config = AWSCredentials.parse_obj(config) if config else config
self.config = (
config
if isinstance(config, AWSCredentials)
else (AWSCredentials.parse_obj(config) if config else config)
)
def _get_session(self) -> Session:
if (

10
ingestion/src/metadata/utils/secrets/secrets_manager_factory.py
View File

@ -64,9 +64,15 @@ class SecretsManagerFactory(metaclass=Singleton):
or secrets_manager_provider == SecretsManagerProvider.noop
):
return NoopSecretsManager()
if secrets_manager_provider == SecretsManagerProvider.aws:
if secrets_manager_provider in (
SecretsManagerProvider.aws,
SecretsManagerProvider.managed_aws_ssm,
):
return AWSSecretsManager(credentials)
if secrets_manager_provider == SecretsManagerProvider.aws_ssm:
if secrets_manager_provider in (
SecretsManagerProvider.aws_ssm,
SecretsManagerProvider.managed_aws_ssm,
):
return AWSSSMSecretsManager(credentials)
raise NotImplementedError(f"[{secrets_manager_provider}] is not implemented.")

4
openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java
View File

@ -77,6 +77,7 @@ import org.openmetadata.service.resources.CollectionRegistry;
import org.openmetadata.service.resources.tags.TagLabelCache;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.secrets.SecretsManagerUpdateService;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.NoopAuthorizer;
import org.openmetadata.service.security.NoopFilter;
@ -157,6 +158,9 @@ public class OpenMetadataApplication extends Application<OpenMetadataApplication
// Register Event publishers
registerEventPublisher(catalogConfig, jdbi);
// update entities secrets if required
new SecretsManagerUpdateService(secretsManager, catalogConfig.getClusterName()).updateEntities();
// start authorizer after event publishers
// authorizer creates admin/bot users, ES publisher should start before to index users created by authorizer
authorizer.init(catalogConfig, jdbi);

2
openmetadata-service/src/main/java/org/openmetadata/service/secrets/AWSBasedSecretsManager.java
View File

@ -28,7 +28,7 @@ public abstract class AWSBasedSecretsManager extends ExternalSecretsManager {
protected AWSBasedSecretsManager(
SecretsManagerProvider awsProvider, SecretsManagerConfiguration config, String clusterPrefix) {
super(awsProvider, clusterPrefix);
super(awsProvider, clusterPrefix, 100);
// initialize the secret client depending on the SecretsManagerConfiguration passed
if (config != null
&& config.getParameters() != null

4
openmetadata-service/src/main/java/org/openmetadata/service/secrets/AWSSSMSecretsManager.java
View File

@ -12,7 +12,7 @@
*/
package org.openmetadata.service.secrets;
import static org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider.AWS_SSM;
import static org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider.MANAGED_AWS_SSM;
import com.google.common.annotations.VisibleForTesting;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@ -29,7 +29,7 @@ public class AWSSSMSecretsManager extends AWSBasedSecretsManager {
private SsmClient ssmClient;
private AWSSSMSecretsManager(SecretsManagerConfiguration config, String clusterPrefix) {
super(AWS_SSM, config, clusterPrefix);
super(MANAGED_AWS_SSM, config, clusterPrefix);
}
@Override

4
openmetadata-service/src/main/java/org/openmetadata/service/secrets/AWSSecretsManager.java
View File

@ -13,7 +13,7 @@
package org.openmetadata.service.secrets;
import static org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider.AWS;
import static org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider.MANAGED_AWS;
import com.google.common.annotations.VisibleForTesting;
import java.util.Objects;
@ -31,7 +31,7 @@ public class AWSSecretsManager extends AWSBasedSecretsManager {
private SecretsManagerClient secretsClient;
private AWSSecretsManager(SecretsManagerConfiguration config, String clusterPrefix) {
super(AWS, config, clusterPrefix);
super(MANAGED_AWS, config, clusterPrefix);
}
@Override

38
openmetadata-service/src/main/java/org/openmetadata/service/secrets/ExternalSecretsManager.java
View File

@ -13,33 +13,48 @@
package org.openmetadata.service.secrets;
import org.openmetadata.schema.entity.services.ServiceType;
import java.util.Locale;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
public abstract class ExternalSecretsManager extends SecretsManager {
public static final String NULL_SECRET_STRING = "null";
public static final String SECRET_FIELD_PREFIX = "secret:";
protected ExternalSecretsManager(SecretsManagerProvider secretsManagerProvider, String clusterPrefix) {
private final long WAIT_TIME_BETWEEN_STORE_CALLS;
protected ExternalSecretsManager(
SecretsManagerProvider secretsManagerProvider, String clusterPrefix, long waitTimeBetweenCalls) {
super(secretsManagerProvider, clusterPrefix);
WAIT_TIME_BETWEEN_STORE_CALLS = waitTimeBetweenCalls;
}
@Override
public Object encryptOrDecryptServiceConnectionConfig(
Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt) {
return connectionConfig;
protected String storeValue(String fieldName, String value, String secretId) {
String fieldSecretId = buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT));
// check if value does not start with 'config:' only String can have password annotation
if (!value.startsWith(SECRET_FIELD_PREFIX)) {
upsertSecret(fieldSecretId, value);
return SECRET_FIELD_PREFIX + fieldSecretId;
} else {
return value;
}
}
public void upsertSecret(String secretName, String secretValue) {
if (existSecret(secretName)) {
updateSecret(secretName, secretValue != null ? secretValue : NULL_SECRET_STRING);
sleep();
} else {
storeSecret(secretName, secretValue != null ? secretValue : NULL_SECRET_STRING);
sleep();
}
}
public boolean existSecret(String secretName) {
try {
return getSecret(secretName) != null;
boolean exists = getSecret(secretName) != null;
sleep();
return exists;
} catch (Exception e) {
return false;
}
@ -50,4 +65,15 @@ public abstract class ExternalSecretsManager extends SecretsManager {
abstract void updateSecret(String secretName, String secretValue);
abstract String getSecret(String secretName);
private void sleep() {
// delay reaching secrets manager quotas
if (WAIT_TIME_BETWEEN_STORE_CALLS > 0) {
try {
Thread.sleep(WAIT_TIME_BETWEEN_STORE_CALLS);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}
}

2
openmetadata-service/src/main/java/org/openmetadata/service/secrets/InMemorySecretsManager.java
View File

@ -27,7 +27,7 @@ public class InMemorySecretsManager extends ExternalSecretsManager {
@Getter private final Map<String, String> secretsMap = new HashMap<>();
protected InMemorySecretsManager(SecretsManagerProvider secretsManagerProvider, String clusterPrefix) {
super(secretsManagerProvider, clusterPrefix);
super(secretsManagerProvider, clusterPrefix, 0);
}
public static InMemorySecretsManager getInstance(String clusterPrefix) {

48
openmetadata-service/src/main/java/org/openmetadata/service/secrets/NoopSecretsManager.java
View File

@ -13,56 +13,14 @@
package org.openmetadata.service.secrets;
import com.google.common.annotations.VisibleForTesting;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import org.openmetadata.service.exception.InvalidServiceConnectionException;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.util.JsonUtils;
public class NoopSecretsManager extends SecretsManager {
private static NoopSecretsManager INSTANCE;
private Fernet fernet;
private NoopSecretsManager(String clusterPrefix, SecretsManagerProvider secretsManagerProvider) {
super(secretsManagerProvider, clusterPrefix);
this.fernet = Fernet.getInstance();
}
@Override
public Object encryptOrDecryptServiceConnectionConfig(
Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt) {
try {
Class<?> clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType));
Object newConnectionConfig = JsonUtils.convertValue(connectionConfig, clazz);
encryptOrDecryptField(newConnectionConfig, "Password", clazz, encrypt);
return newConnectionConfig;
} catch (Exception e) {
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to construct connection instance of %s", connectionType));
}
}
private void encryptOrDecryptField(Object connConfig, String field, Class<?> clazz, boolean encrypt)
throws InvocationTargetException, IllegalAccessException {
try {
Method getPasswordMethod = clazz.getMethod("get" + field);
Method setPasswordMethod = clazz.getMethod("set" + field, String.class);
String password = (String) getPasswordMethod.invoke(connConfig);
if (password != null && !password.equals("")) {
if (!Fernet.isTokenized(password) && encrypt) {
password = fernet.encrypt(password);
} else if (Fernet.isTokenized(password) && !encrypt) {
password = fernet.decrypt(password);
}
setPasswordMethod.invoke(connConfig, password);
}
} catch (NoSuchMethodException ignore) {
}
}
public static NoopSecretsManager getInstance(String clusterPrefix, SecretsManagerProvider secretsManagerProvider) {
@ -70,8 +28,8 @@ public class NoopSecretsManager extends SecretsManager {
return INSTANCE;
}
@VisibleForTesting
void setFernet(Fernet fernet) {
this.fernet = fernet;
@Override
protected String storeValue(String fieldName, String value, String secretId) {
return value;
}
}

148
openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java
View File

@ -15,13 +15,18 @@ package org.openmetadata.service.secrets;
import static java.util.Objects.isNull;
import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Locale;
import lombok.Getter;
import org.openmetadata.annotations.PasswordField;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import org.openmetadata.service.exception.InvalidServiceConnectionException;
import org.openmetadata.service.exception.SecretsManagerException;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.util.JsonUtils;
public abstract class SecretsManager {
@ -30,13 +35,115 @@ public abstract class SecretsManager {
@Getter private final SecretsManagerProvider secretsManagerProvider;
private Fernet fernet;
protected SecretsManager(SecretsManagerProvider secretsManagerProvider, String clusterPrefix) {
this.secretsManagerProvider = secretsManagerProvider;
this.clusterPrefix = clusterPrefix;
this.fernet = Fernet.getInstance();
}
public abstract Object encryptOrDecryptServiceConnectionConfig(
Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt);
public Object encryptOrDecryptServiceConnectionConfig(
Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt) {
try {
Class<?> clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType));
Object newConnectionConfig = JsonUtils.convertValue(connectionConfig, clazz);
if (encrypt) {
encryptPasswordFields(newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName));
} else {
decryptPasswordFields(newConnectionConfig);
}
return newConnectionConfig;
} catch (Exception e) {
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to encrypt connection instance of %s", connectionType));
}
}
private void encryptPasswordFields(Object toEncryptObject, String secretId) {
// for each get method
Arrays.stream(toEncryptObject.getClass().getMethods())
.filter(this::isGetMethodOfObject)
.forEach(
method -> {
Object obj = getObjectFromMethod(method, toEncryptObject);
String fieldName = method.getName().replaceFirst("get", "");
// if the object matches the package of openmetadata
if (obj != null && obj.getClass().getPackageName().startsWith("org.openmetadata")) {
// encryptPasswordFields
encryptPasswordFields(obj, buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT)));
// check if it has annotation
} else if (obj != null && method.getAnnotation(PasswordField.class) != null) {
// store value if proceed
String newFieldValue = storeValue(fieldName, (String) obj, secretId);
// get setMethod
Method toSet = getToSetMethod(toEncryptObject, obj, fieldName);
// set new value
setValueInMethod(
toEncryptObject,
Fernet.isTokenized(newFieldValue) ? newFieldValue : fernet.encrypt(newFieldValue),
toSet);
}
});
}
private void decryptPasswordFields(Object toDecryptObject) {
// for each get method
Arrays.stream(toDecryptObject.getClass().getMethods())
.filter(this::isGetMethodOfObject)
.forEach(
method -> {
Object obj = getObjectFromMethod(method, toDecryptObject);
String fieldName = method.getName().replaceFirst("get", "");
// if the object matches the package of openmetadata
if (obj != null && obj.getClass().getPackageName().startsWith("org.openmetadata")) {
// encryptPasswordFields
decryptPasswordFields(obj);
// check if it has annotation
} else if (obj != null && method.getAnnotation(PasswordField.class) != null) {
String fieldValue = (String) obj;
// get setMethod
Method toSet = getToSetMethod(toDecryptObject, obj, fieldName);
// set new value
setValueInMethod(
toDecryptObject, Fernet.isTokenized(fieldValue) ? fernet.decrypt(fieldValue) : fieldValue, toSet);
}
});
}
protected abstract String storeValue(String fieldName, String value, String secretId);
private void setValueInMethod(Object toEncryptObject, String fieldValue, Method toSet) {
try {
toSet.invoke(toEncryptObject, fieldValue);
} catch (IllegalAccessException | InvocationTargetException e) {
throw new SecretsManagerException(e.getMessage());
}
}
private Method getToSetMethod(Object toEncryptObject, Object obj, String fieldName) {
try {
return toEncryptObject.getClass().getMethod("set" + fieldName, obj.getClass());
} catch (NoSuchMethodException e) {
throw new SecretsManagerException(e.getMessage());
}
}
private Object getObjectFromMethod(Method method, Object toEncryptObject) {
Object obj;
try {
obj = method.invoke(toEncryptObject);
} catch (IllegalAccessException | InvocationTargetException e) {
throw new SecretsManagerException(e.getMessage());
}
return obj;
}
private boolean isGetMethodOfObject(Method method) {
return method.getName().startsWith("get")
&& !method.getReturnType().equals(Void.TYPE)
&& !method.getReturnType().isPrimitive();
}
protected String getSecretSeparator() {
return "/";
@ -46,17 +153,25 @@ public abstract class SecretsManager {
return true;
}
protected String buildSecretId(String... secretIdValues) {
protected String buildSecretId(boolean addClusterPrefix, String... secretIdValues) {
StringBuilder format = new StringBuilder();
format.append(startsWithSeparator() ? getSecretSeparator() : "");
format.append(clusterPrefix);
for (String secretIdValue : List.of(secretIdValues)) {
if (isNull(secretIdValue)) {
throw new SecretsManagerException("Cannot build a secret id with null values.");
}
format.append(getSecretSeparator());
if (addClusterPrefix) {
format.append(startsWithSeparator() ? getSecretSeparator() : "");
format.append(clusterPrefix);
} else {
format.append("%s");
}
// skip first one in case of addClusterPrefix is false to avoid adding extra separator at the beginning
Arrays.stream(secretIdValues)
.skip(addClusterPrefix ? 0 : 1)
.forEach(
secretIdValue -> {
if (isNull(secretIdValue)) {
throw new SecretsManagerException("Cannot build a secret id with null values.");
}
format.append(getSecretSeparator());
format.append("%s");
});
return String.format(format.toString(), (Object[]) secretIdValues).toLowerCase();
}
@ -72,13 +187,8 @@ public abstract class SecretsManager {
return serviceType.value().toLowerCase(Locale.ROOT);
}
public void validateServiceConnection(Object connectionConfig, String connectionType, ServiceType serviceType) {
try {
Class<?> clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType));
JsonUtils.readValue(JsonUtils.pojoToJson(connectionConfig), clazz);
} catch (Exception exception) {
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to construct connection instance of %s", connectionType));
}
@VisibleForTesting
void setFernet(Fernet fernet) {
this.fernet = fernet;
}
}

4
openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerFactory.java
View File

@ -35,7 +35,6 @@ public class SecretsManagerFactory {
case NOOP:
case AWS_SSM:
case AWS:
case IN_MEMORY:
secretsManager = NoopSecretsManager.getInstance(clusterName, secretsManagerProvider);
break;
case MANAGED_AWS:
@ -44,6 +43,9 @@ public class SecretsManagerFactory {
case MANAGED_AWS_SSM:
secretsManager = AWSSSMSecretsManager.getInstance(config, clusterName);
break;
case IN_MEMORY:
secretsManager = InMemorySecretsManager.getInstance(clusterName);
break;
default:
throw new IllegalArgumentException("Not implemented secret manager store: " + secretsManagerProvider);
}

159
openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java Normal file
View File

@ -0,0 +1,159 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.secrets;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.ServiceConnectionEntityInterface;
import org.openmetadata.schema.ServiceEntityInterface;
import org.openmetadata.service.exception.SecretsManagerMigrationException;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.ServiceEntityRepository;
import org.openmetadata.service.resources.CollectionRegistry;
import org.openmetadata.service.resources.CollectionRegistry.CollectionDetails;
import org.openmetadata.service.resources.services.ServiceEntityResource;
import org.openmetadata.service.util.EntityUtil;
/**
* Update service using the configured secret manager.
*
* <p>- It will update all the services entities with connection parameters
*/
@Slf4j
public class SecretsManagerUpdateService {
private final SecretsManager secretManager;
private final SecretsManager oldSecretManager;
private final Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
connectionTypeRepositoriesMap;
public SecretsManagerUpdateService(SecretsManager secretsManager, String clusterName) {
this.secretManager = secretsManager;
this.connectionTypeRepositoriesMap = retrieveConnectionTypeRepositoriesMap();
// by default, it is going to be non-managed secrets manager since decrypt is the same for all of them
this.oldSecretManager = SecretsManagerFactory.createSecretsManager(null, clusterName);
}
public void updateEntities() {
updateServices();
}
private void updateServices() {
LOG.info(
String.format(
"Checking if services updating is needed for secrets manager: [%s]",
secretManager.getSecretsManagerProvider().value()));
List<ServiceEntityInterface> notStoredServices = retrieveServices();
if (!notStoredServices.isEmpty()) {
notStoredServices.forEach(this::updateService);
}
}
private void updateService(ServiceEntityInterface serviceEntityInterface) {
ServiceEntityRepository<?, ?> repository =
connectionTypeRepositoriesMap.get(serviceEntityInterface.getConnection().getClass());
try {
ServiceEntityInterface service = repository.dao.findEntityById(serviceEntityInterface.getId());
// we have to decrypt using the old secrets manager and encrypt again with the new one
service
.getConnection()
.setConfig(
oldSecretManager.encryptOrDecryptServiceConnectionConfig(
service.getConnection().getConfig(),
service.getServiceType().value(),
service.getName(),
repository.getServiceType(),
false));
service
.getConnection()
.setConfig(
secretManager.encryptOrDecryptServiceConnectionConfig(
service.getConnection().getConfig(),
service.getServiceType().value(),
service.getName(),
repository.getServiceType(),
true));
repository.dao.update(service);
} catch (IOException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
}
private List<ServiceEntityInterface> retrieveServices() {
return connectionTypeRepositoriesMap.values().stream()
.map(this::retrieveServices)
.flatMap(List<ServiceEntityInterface>::stream)
.collect(Collectors.toList());
}
private List<ServiceEntityInterface> retrieveServices(ServiceEntityRepository<?, ?> serviceEntityRepository) {
try {
return serviceEntityRepository
.listAfter(
null,
EntityUtil.Fields.EMPTY_FIELDS,
new ListFilter(),
serviceEntityRepository.dao.listCount(new ListFilter()),
null)
.getData().stream()
.map(ServiceEntityInterface.class::cast)
.filter(
service ->
!Objects.isNull(service.getConnection()) && !Objects.isNull(service.getConnection().getConfig()))
.collect(Collectors.toList());
} catch (IOException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
}
private Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
retrieveConnectionTypeRepositoriesMap() {
Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
connectionTypeRepositoriesMap =
CollectionRegistry.getInstance().getCollectionMap().values().stream()
.map(this::retrieveServiceRepository)
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toMap(ServiceEntityRepository::getServiceConnectionClass, Function.identity()));
if (connectionTypeRepositoriesMap.isEmpty()) {
throw new SecretsManagerMigrationException("Unexpected error: ServiceRepository not found.");
}
return connectionTypeRepositoriesMap;
}
private Optional<ServiceEntityRepository<?, ?>> retrieveServiceRepository(CollectionDetails collectionDetails) {
Class<?> collectionDetailsClass = extractCollectionDetailsClass(collectionDetails);
if (ServiceEntityResource.class.isAssignableFrom(collectionDetailsClass)) {
return Optional.of(
((ServiceEntityResource<?, ?, ?>) collectionDetails.getResource()).getServiceEntityRepository());
}
return Optional.empty();
}
private Class<?> extractCollectionDetailsClass(CollectionDetails collectionDetails) {
Class<?> collectionDetailsClass;
try {
collectionDetailsClass = Class.forName(collectionDetails.getResourceClass());
} catch (ClassNotFoundException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
return collectionDetailsClass;
}
}

38
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -374,6 +374,10 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
return createRequest(getEntityName(test)).withDescription("").withDisplayName(null).withOwner(null);
}
public final K createPutRequest(TestInfo test) {
return createPutRequest(getEntityName(test)).withDescription("").withDisplayName(null).withOwner(null);
}
public final K createRequest(TestInfo test, int index) {
return createRequest(getEntityName(test, index)).withDescription("").withDisplayName(null).withOwner(null);
}
@ -388,8 +392,22 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
.withOwner(reduceEntityReference(owner));
}
public final K createPutRequest(String name, String description, String displayName, EntityReference owner) {
if (!supportsEmptyDescription && description == null) {
throw new IllegalArgumentException("Entity " + entityType + " does not support empty description");
}
return createPutRequest(name)
.withDescription(description)
.withDisplayName(displayName)
.withOwner(reduceEntityReference(owner));
}
public abstract K createRequest(String name);
public K createPutRequest(String name) {
return createRequest(name);
}
// Add all possible relationships to check if the entity is missing any of them after deletion
public T beforeDeletion(TestInfo test, T entity) throws HttpResponseException {
return entity;
@ -909,7 +927,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
T entity = createAndCheckEntity(request, ADMIN_AUTH_HEADERS);
// Update the entity as USER_OWNER1
request = createRequest(getEntityName(test), "newDescription", null, USER1_REF);
request = createPutRequest(getEntityName(test), "newDescription", null, USER1_REF);
ChangeDescription change = getChangeDescription(entity.getVersion());
fieldUpdated(change, "description", "", "newDescription");
updateAndCheckEntity(request, OK, authHeaders(USER1.getEmail()), MINOR_UPDATE, change);
@ -925,14 +943,14 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
T entity = createAndCheckEntity(request, ADMIN_AUTH_HEADERS);
// Set TEAM_OWNER1 as owner using PUT request
request = createRequest(getEntityName(test), "description", "displayName", TEAM11_REF);
request = createPutRequest(getEntityName(test), "description", "displayName", TEAM11_REF);
ChangeDescription change = getChangeDescription(entity.getVersion());
fieldAdded(change, FIELD_OWNER, TEAM11_REF);
entity = updateAndCheckEntity(request, OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
checkOwnerOwns(TEAM11_REF, entity.getId(), true);
// Change owner from TEAM_OWNER1 to USER_OWNER1 using PUT request
request = createRequest(getEntityName(test), "description", "displayName", USER1_REF);
request = createPutRequest(getEntityName(test), "description", "displayName", USER1_REF);
change = getChangeDescription(entity.getVersion());
fieldUpdated(change, FIELD_OWNER, TEAM11_REF, USER1_REF);
entity = updateAndCheckEntity(request, OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
@ -940,14 +958,14 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
checkOwnerOwns(TEAM11_REF, entity.getId(), false);
// Set the owner to the existing owner. No ownership change must be recorded.
request = createRequest(getEntityName(test), "description", "displayName", USER1_REF);
request = createPutRequest(getEntityName(test), "description", "displayName", USER1_REF);
change = getChangeDescription(entity.getVersion());
entity = updateAndCheckEntity(request, OK, ADMIN_AUTH_HEADERS, NO_CHANGE, change);
checkOwnerOwns(USER1_REF, entity.getId(), true);
// Remove ownership (from USER_OWNER1) using PUT request. Owner is expected to remain the same
// and not removed.
request = createRequest(getEntityName(test), "description", "displayName", null);
request = createPutRequest(getEntityName(test), "description", "displayName", null);
updateEntity(request, OK, ADMIN_AUTH_HEADERS);
checkOwnerOwns(USER1_REF, entity.getId(), true);
}
@ -1046,7 +1064,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
T entity = createEntity(request, ADMIN_AUTH_HEADERS);
// Update null description with a new description
request = createRequest(getEntityName(test), "updatedDescription", "displayName", null);
request = createPutRequest(getEntityName(test), "updatedDescription", "displayName", null);
ChangeDescription change = getChangeDescription(entity.getVersion());
fieldAdded(change, "description", "updatedDescription");
updateAndCheckEntity(request, OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
@ -1059,7 +1077,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
T entity = createEntity(request, ADMIN_AUTH_HEADERS);
// Update empty description with a new description
request = createRequest(getEntityName(test), "updatedDescription", "displayName", null);
request = createPutRequest(getEntityName(test), "updatedDescription", "displayName", null);
ChangeDescription change = getChangeDescription(entity.getVersion());
fieldUpdated(change, "description", "", "updatedDescription");
updateAndCheckEntity(request, OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
@ -1073,7 +1091,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
// BOT user can update empty description and empty displayName
ChangeDescription change = getChangeDescription(entity.getVersion());
request = createRequest(getEntityName(test), "description", "displayName", null);
request = createPutRequest(getEntityName(test), "description", "displayName", null);
if (supportsEmptyDescription) {
fieldAdded(change, "description", "description");
}
@ -1081,14 +1099,14 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
entity = updateAndCheckEntity(request, OK, INGESTION_BOT_AUTH_HEADERS, MINOR_UPDATE, change);
// Updating non-empty description and non-empty displayName is allowed for users other than bots
request = createRequest(getEntityName(test), "updatedDescription", "updatedDisplayName", null);
request = createPutRequest(getEntityName(test), "updatedDescription", "updatedDisplayName", null);
change = getChangeDescription(entity.getVersion());
fieldUpdated(change, "description", "description", "updatedDescription");
fieldUpdated(change, "displayName", "displayName", "updatedDisplayName");
updateAndCheckEntity(request, OK, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
// Updating non-empty description and non-empty displayName is ignored for bot users
request = createRequest(getEntityName(test), "updatedDescription2", "updatedDisplayName2", null);
request = createPutRequest(getEntityName(test), "updatedDescription2", "updatedDisplayName2", null);
updateAndCheckEntity(request, OK, INGESTION_BOT_AUTH_HEADERS, NO_CHANGE, null);
}

47
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DashboardServiceResourceTest.java
View File

@ -30,6 +30,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Locale;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
@ -45,6 +46,7 @@ import org.openmetadata.schema.services.connections.dashboard.SupersetConnection
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.DashboardConnection;
import org.openmetadata.service.Entity;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.charts.ChartResourceTest;
import org.openmetadata.service.resources.services.dashboard.DashboardServiceResource.DashboardServiceList;
@ -97,9 +99,14 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
@Test
void put_updateService_as_admin_2xx(TestInfo test) throws IOException, URISyntaxException {
String secretPassword = "secret:/openmetadata/dashboard/" + getEntityName(test) + "/password";
DashboardConnection dashboardConnection =
new DashboardConnection()
.withConfig(new SupersetConnection().withHostPort(new URI("http://localhost:8080")).withUsername("user"));
.withConfig(
new SupersetConnection()
.withHostPort(new URI("http://localhost:8080"))
.withUsername("user")
.withPassword(secretPassword));
DashboardService service =
createAndCheckEntity(
createRequest(test).withDescription(null).withConnection(dashboardConnection), ADMIN_AUTH_HEADERS);
@ -107,10 +114,14 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
// Update dashboard description and ingestion service that are null
DashboardConnection dashboardConnection1 =
new DashboardConnection()
.withConfig(new SupersetConnection().withHostPort(new URI("http://localhost:9000")).withUsername("user1"));
.withConfig(
new SupersetConnection()
.withHostPort(new URI("http://localhost:9000"))
.withUsername("user1")
.withPassword(secretPassword));
CreateDashboardService update =
createRequest(test).withDescription("description1").withConnection(dashboardConnection1);
createPutRequest(test).withDescription("description1").withConnection(dashboardConnection1);
ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1");
@ -129,9 +140,12 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
JsonUtils.readValue(JsonUtils.pojoToJson(updatedService.getConnection().getConfig()), SupersetConnection.class)
.getUsername());
SupersetConnection supersetConnection =
new SupersetConnection().withHostPort(new URI("http://localhost:8080")).withUsername("user");
new SupersetConnection()
.withHostPort(new URI("http://localhost:8080"))
.withUsername("user")
.withPassword(secretPassword);
DashboardConnection dashboardConnection2 = new DashboardConnection().withConfig(supersetConnection);
update = createRequest(test).withDescription("description1").withConnection(dashboardConnection2);
update = createPutRequest(test).withDescription("description1").withConnection(dashboardConnection2);
fieldUpdated(change, "connection", dashboardConnection1, dashboardConnection2);
updateAndCheckEntity(update, OK, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
@ -159,6 +173,26 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
return null;
}
@Override
public CreateDashboardService createPutRequest(String name) {
String secretPassword = "secret:/openmetadata/dashboard/" + name + "/password";
try {
return new CreateDashboardService()
.withName(name)
.withServiceType(CreateDashboardService.DashboardServiceType.Superset)
.withConnection(
new DashboardConnection()
.withConfig(
new SupersetConnection()
.withHostPort(new URI("http://localhost:8080"))
.withUsername("admin")
.withPassword(Fernet.getInstance().encrypt(secretPassword.toLowerCase(Locale.ROOT)))));
} catch (URISyntaxException e) {
e.printStackTrace();
}
return null;
}
@Override
public void validateCreatedEntity(
DashboardService service, CreateDashboardService createRequest, Map<String, String> authHeaders) {
@ -220,7 +254,8 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
assertEquals(expectedSupersetConnection.getProvider(), actualSupersetConnection.getProvider());
if (ADMIN_AUTH_HEADERS.equals(authHeaders) || INGESTION_BOT_AUTH_HEADERS.equals(authHeaders)) {
assertEquals(expectedSupersetConnection.getUsername(), actualSupersetConnection.getUsername());
assertEquals(expectedSupersetConnection.getPassword(), actualSupersetConnection.getPassword());
assertTrue(actualSupersetConnection.getPassword().startsWith("secret:/openmetadata/dashboard/"));
assertTrue(actualSupersetConnection.getPassword().endsWith("/password"));
} else {
assertNull(actualSupersetConnection.getUsername());
assertNull(actualSupersetConnection.getPassword());

23
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DatabaseServiceResourceTest.java
View File

@ -21,11 +21,13 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.SNOWFLAKE_DATABASE_CONNECTION;
import static org.openmetadata.service.util.TestUtils.TEST_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.assertResponseContains;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
@ -50,6 +52,7 @@ import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Schedule;
import org.openmetadata.service.Entity;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.services.database.DatabaseServiceResource.DatabaseServiceList;
import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineResourceTest;
@ -124,7 +127,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
DatabaseService service = createAndCheckEntity(createRequest(test).withDescription(null), ADMIN_AUTH_HEADERS);
// Update database description and ingestion service that are null
CreateDatabaseService update = createRequest(test).withDescription("description1");
CreateDatabaseService update = createPutRequest(test).withDescription("description1");
ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1");
@ -166,10 +169,10 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
assertResponseContains(
() -> createEntity(createRequest(test).withDescription(null).withConnection(dbConn), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"InvalidServiceConnectionException for service [Snowflake] due to [Failed to construct connection instance of Snowflake]");
"InvalidServiceConnectionException for service [Snowflake] due to [Failed to encrypt connection instance of Snowflake]");
DatabaseService service = createAndCheckEntity(createRequest(test).withDescription(null), ADMIN_AUTH_HEADERS);
// Update database description and ingestion service that are null
CreateDatabaseService update = createRequest(test).withDescription("description1");
CreateDatabaseService update = createPutRequest(test).withDescription("description1");
ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1");
@ -180,7 +183,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
assertResponseContains(
() -> updateEntity(update, OK, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"InvalidServiceConnectionException for service [Snowflake] due to [Failed to construct connection instance of Snowflake]");
"InvalidServiceConnectionException for service [Snowflake] due to [Failed to encrypt connection instance of Snowflake]");
}
@Test
@ -252,6 +255,18 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
.withConnection(TestUtils.SNOWFLAKE_DATABASE_CONNECTION);
}
@Override
public CreateDatabaseService createPutRequest(String name) {
String secretPassword = "secret:/openmetadata/database/" + name + "/password";
return new CreateDatabaseService()
.withName(name)
.withServiceType(DatabaseServiceType.Snowflake)
.withConnection(
TestUtils.SNOWFLAKE_DATABASE_CONNECTION.withConfig(
((SnowflakeConnection) SNOWFLAKE_DATABASE_CONNECTION.getConfig())
.withPassword(Fernet.getInstance().encrypt(secretPassword.toLowerCase(Locale.ROOT)))));
}
@Override
public void validateCreatedEntity(
DatabaseService service, CreateDatabaseService createRequest, Map<String, String> authHeaders) {

21
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/MetadataServiceResourceTest.java
View File

@ -13,6 +13,7 @@ import static org.openmetadata.service.util.TestUtils.assertResponse;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Locale;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
@ -25,6 +26,7 @@ import org.openmetadata.schema.services.connections.metadata.AmundsenConnection;
import org.openmetadata.schema.services.connections.metadata.AtlasConnection;
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.service.Entity;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.services.metadata.MetadataServiceResource;
import org.openmetadata.service.util.JsonUtils;
@ -98,6 +100,7 @@ public class MetadataServiceResourceTest extends EntityResourceTest<MetadataServ
@Test
void put_updateService_as_admin_2xx(TestInfo test) throws IOException, URISyntaxException {
String secretPassword = "secret:/openmetadata/metadata/" + getEntityName(test) + "/password";
MetadataService service =
createAndCheckEntity(
createRequest(test)
@ -108,7 +111,7 @@ public class MetadataServiceResourceTest extends EntityResourceTest<MetadataServ
new AmundsenConnection()
.withHostPort(new URI("localhost:9092"))
.withUsername("admin")
.withPassword("admin"))),
.withPassword(secretPassword))),
ADMIN_AUTH_HEADERS);
MetadataConnection metadataConnection =
@ -117,10 +120,10 @@ public class MetadataServiceResourceTest extends EntityResourceTest<MetadataServ
new AmundsenConnection()
.withHostPort(new URI("localhost:9092"))
.withUsername("admin")
.withPassword("admin"));
.withPassword(secretPassword));
// Update metadata description
CreateMetadataService update =
createRequest(test).withDescription("description1").withConnection(metadataConnection);
createPutRequest(test).withDescription("description1").withConnection(metadataConnection);
ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1");
service = updateAndCheckEntity(update, OK, ADMIN_AUTH_HEADERS, TestUtils.UpdateType.MINOR_UPDATE, change);
@ -161,6 +164,18 @@ public class MetadataServiceResourceTest extends EntityResourceTest<MetadataServ
.withConnection(AMUNDSEN_CONNECTION);
}
@Override
public CreateMetadataService createPutRequest(String name) {
String secretPassword = "secret:/openmetadata/metadata/" + name + "/password";
return new CreateMetadataService()
.withName(name)
.withServiceType(CreateMetadataService.MetadataServiceType.Amundsen)
.withConnection(
AMUNDSEN_CONNECTION.withConfig(
((AmundsenConnection) AMUNDSEN_CONNECTION.getConfig())
.withPassword(Fernet.getInstance().encrypt(secretPassword.toLowerCase(Locale.ROOT)))));
}
@Override
public void validateCreatedEntity(
MetadataService service, CreateMetadataService createRequest, Map<String, String> authHeaders) {

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/PipelineServiceResourceTest.java
View File

@ -157,7 +157,7 @@ public class PipelineServiceResourceTest extends EntityResourceTest<PipelineServ
createEntity(
createRequest(test).withDescription(null).withConnection(pipelineConnection), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"InvalidServiceConnectionException for service [Airflow] due to [Failed to construct connection instance of Airflow]");
"InvalidServiceConnectionException for service [Airflow] due to [Failed to encrypt connection instance of Airflow]");
}
@Test

32
openmetadata-service/src/test/java/org/openmetadata/service/secrets/AWSSSMSecretsManagerTest.java
View File

@ -12,26 +12,16 @@
*/
package org.openmetadata.service.secrets;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.util.Objects;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import org.openmetadata.service.util.JsonUtils;
import software.amazon.awssdk.services.ssm.SsmClient;
import software.amazon.awssdk.services.ssm.model.GetParameterRequest;
import software.amazon.awssdk.services.ssm.model.GetParameterResponse;
import software.amazon.awssdk.services.ssm.model.Parameter;
import software.amazon.awssdk.services.ssm.model.PutParameterRequest;
public class AWSSSMSecretsManagerTest extends ExternalSecretsManagerTest {
@ -57,28 +47,6 @@ public class AWSSSMSecretsManagerTest extends ExternalSecretsManagerTest {
}
}
@Override
void verifySecretIdGetCalls(String expectedSecretId, int times) {
ArgumentCaptor<GetParameterRequest> getSecretCaptor = ArgumentCaptor.forClass(GetParameterRequest.class);
verify(ssmClient, times(times)).getParameter(getSecretCaptor.capture());
for (int i = 0; i < times; i++) {
assertEquals(expectedSecretId, getSecretCaptor.getAllValues().get(i).name());
}
}
@Override
void verifyClientCalls(Object expectedAuthProviderConfig, String expectedSecretId) throws JsonProcessingException {
ArgumentCaptor<PutParameterRequest> createSecretCaptor = ArgumentCaptor.forClass(PutParameterRequest.class);
if (Objects.isNull(expectedAuthProviderConfig)) {
verifyNoInteractions(ssmClient);
} else {
verify(ssmClient).putParameter(createSecretCaptor.capture());
assertEquals(expectedSecretId, createSecretCaptor.getValue().name());
assertNotNull(createSecretCaptor.getValue().value());
assertEquals(JsonUtils.pojoToJson(expectedAuthProviderConfig), createSecretCaptor.getValue().value());
}
}
@Override
SecretsManagerProvider expectedSecretManagerProvider() {
return SecretsManagerProvider.AWS_SSM;

32
openmetadata-service/src/test/java/org/openmetadata/service/secrets/AWSSecretsManagerTest.java
View File

@ -12,23 +12,13 @@
*/
package org.openmetadata.service.secrets;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.util.Objects;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider;
import org.openmetadata.service.util.JsonUtils;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest;
import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest;
import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse;
@ -56,28 +46,6 @@ public class AWSSecretsManagerTest extends ExternalSecretsManagerTest {
}
}
@Override
void verifySecretIdGetCalls(String expectedSecretId, int times) {
ArgumentCaptor<GetSecretValueRequest> getSecretCaptor = ArgumentCaptor.forClass(GetSecretValueRequest.class);
verify(secretsManagerClient, times(times)).getSecretValue(getSecretCaptor.capture());
for (int i = 0; i < times; i++) {
assertEquals(expectedSecretId, getSecretCaptor.getAllValues().get(i).secretId());
}
}
@Override
void verifyClientCalls(Object expectedAuthProviderConfig, String expectedSecretId) throws JsonProcessingException {
ArgumentCaptor<CreateSecretRequest> createSecretCaptor = ArgumentCaptor.forClass(CreateSecretRequest.class);
if (Objects.isNull(expectedAuthProviderConfig)) {
verifyNoInteractions(secretsManagerClient);
} else {
verify(secretsManagerClient).createSecret(createSecretCaptor.capture());
assertEquals(expectedSecretId, createSecretCaptor.getValue().name());
assertNotNull(createSecretCaptor.getValue().secretString());
assertEquals(JsonUtils.pojoToJson(expectedAuthProviderConfig), createSecretCaptor.getValue().secretString());
}
}
@Override
SecretsManagerProvider expectedSecretManagerProvider() {
return SecretsManagerProvider.AWS;

11
openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java
View File

@ -13,9 +13,7 @@
package org.openmetadata.service.secrets;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
@ -30,9 +28,6 @@ import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvi
@ExtendWith(MockitoExtension.class)
public abstract class ExternalSecretsManagerTest {
static final boolean ENCRYPT = true;
static final String AUTH_PROVIDER_SECRET_ID_PREFIX = "auth-provider";
static final String TEST_CONNECTION_SECRET_ID_PREFIX = "test-connection-temp";
static final boolean DECRYPT = false;
static final String EXPECTED_CONNECTION_JSON =
"{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"password\":\"openmetadata-test\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true,\"supportsQueryComment\":true}";
@ -66,11 +61,6 @@ public abstract class ExternalSecretsManagerTest {
abstract void mockClientGetValue(String value);
abstract void verifySecretIdGetCalls(String expectedSecretId, int times);
abstract void verifyClientCalls(Object expectedAuthProviderConfig, String expectedSecretId)
throws JsonProcessingException;
void testEncryptDecryptServiceConnection(boolean decrypt) {
MysqlConnection mysqlConnection = new MysqlConnection();
mysqlConnection.setPassword("openmetadata-test");
@ -82,7 +72,6 @@ public abstract class ExternalSecretsManagerTest {
mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt);
assertEquals(mysqlConnection, actualConfig);
assertSame(mysqlConnection, actualConfig);
}
abstract SecretsManagerProvider expectedSecretManagerProvider();

6
openmetadata-service/src/test/java/org/openmetadata/service/secrets/NoopSecretsManagerTest.java
View File

@ -62,7 +62,7 @@ public class NoopSecretsManagerTest {
@Test
void testDecryptDatabaseServiceConnectionConfig() {
testEncryptDecryptServiceConnection(ENCRYPTED_VALUE, DECRYPTED_VALUE, DECRYPT);
testEncryptDecryptServiceConnection(DECRYPTED_VALUE, ENCRYPTED_VALUE, DECRYPT);
}
@Test
@ -94,7 +94,7 @@ public class NoopSecretsManagerTest {
private void testEncryptDecryptServiceConnection(String encryptedValue, String decryptedValue, boolean decrypt) {
MysqlConnection mysqlConnection = new MysqlConnection();
mysqlConnection.setPassword(encryptedValue);
mysqlConnection.setPassword(decrypt ? encryptedValue : decryptedValue);
CreateDatabaseService.DatabaseServiceType databaseServiceType = CreateDatabaseService.DatabaseServiceType.Mysql;
String connectionName = "test";
@ -102,7 +102,7 @@ public class NoopSecretsManagerTest {
secretsManager.encryptOrDecryptServiceConnectionConfig(
mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt);
assertEquals(decryptedValue, ((MysqlConnection) actualConfig).getPassword());
assertEquals(decrypt ? decryptedValue : encryptedValue, ((MysqlConnection) actualConfig).getPassword());
assertNotSame(mysqlConnection, actualConfig);
}
}