diff --git a/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java b/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java index 5c3ec51147f..15f0bcd948b 100644 --- a/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java +++ b/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java @@ -19,6 +19,6 @@ public class OpenMetadataAnnotator extends CompositeAnnotator { public OpenMetadataAnnotator() { // we can add multiple annotators - super(new ExposedAnnotator(), new MaskedAnnotator()); + super(new ExposedAnnotator(), new MaskedAnnotator(), new PasswordAnnotator()); } } diff --git a/common/src/main/java/org/openmetadata/annotations/PasswordAnnotator.java b/common/src/main/java/org/openmetadata/annotations/PasswordAnnotator.java new file mode 100644 index 00000000000..91cd406e411 --- /dev/null +++ b/common/src/main/java/org/openmetadata/annotations/PasswordAnnotator.java @@ -0,0 +1,80 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.annotations; + +import com.fasterxml.jackson.databind.JsonNode; +import com.sun.codemodel.JAnnotationUse; +import com.sun.codemodel.JClass; +import com.sun.codemodel.JDefinedClass; +import com.sun.codemodel.JFieldVar; +import com.sun.codemodel.JMethod; +import java.lang.reflect.Field; +import org.jsonschema2pojo.AbstractAnnotator; + +/** Add {@link PasswordField} annotation to generated Java classes */ +public class PasswordAnnotator extends AbstractAnnotator { + + /** Add {@link PasswordField} annotation property fields */ + @Override + public void propertyField(JFieldVar field, JDefinedClass clazz, String propertyName, JsonNode propertyNode) { + super.propertyField(field, clazz, propertyName, propertyNode); + if (propertyNode.get("format") != null && "password".equals(propertyNode.get("format").asText())) { + field.annotate(PasswordField.class); + } + } + + /** Add {@link PasswordField} annotation to getter methods */ + @Override + public void propertyGetter(JMethod getter, JDefinedClass clazz, String propertyName) { + super.propertyGetter(getter, clazz, propertyName); + addMaskedFieldAnnotationIfApplies(getter, propertyName); + } + + /** Add {@link PasswordField} annotation to setter methods */ + @Override + public void propertySetter(JMethod setter, JDefinedClass clazz, String propertyName) { + super.propertySetter(setter, clazz, propertyName); + addMaskedFieldAnnotationIfApplies(setter, propertyName); + } + + /** + * Use reflection methods to access the {@link JDefinedClass} of the {@link JMethod} object. If the {@link JMethod} is + * pointing to a field annotated with {@link PasswordField} then annotates the {@link JMethod} object with {@link + * PasswordField} + */ + private void addMaskedFieldAnnotationIfApplies(JMethod jMethod, String propertyName) { + try { + Field outerClassField = JMethod.class.getDeclaredField("outer"); + outerClassField.setAccessible(true); + JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod); + if (outerClass.fields().containsKey(propertyName) + && outerClass.fields().get(propertyName).annotations().stream() + .anyMatch(annotation -> PasswordField.class.getName().equals(getAnnotationClassName(annotation)))) { + jMethod.annotate(PasswordField.class); + } + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private String getAnnotationClassName(JAnnotationUse annotation) { + try { + Field clazzField = JAnnotationUse.class.getDeclaredField("clazz"); + clazzField.setAccessible(true); + return ((JClass) clazzField.get(annotation)).fullName(); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } +} diff --git a/common/src/main/java/org/openmetadata/annotations/PasswordField.java b/common/src/main/java/org/openmetadata/annotations/PasswordField.java new file mode 100644 index 00000000000..34c3b7ee573 --- /dev/null +++ b/common/src/main/java/org/openmetadata/annotations/PasswordField.java @@ -0,0 +1,27 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Specifies that the field or method is exposed, i.e., if the serialization will take into account those fields + * annotated with + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.ANNOTATION_TYPE}) +public @interface PasswordField {} diff --git a/ingestion/src/metadata/clients/aws_client.py b/ingestion/src/metadata/clients/aws_client.py index 0e675ec56f5..fa9c3a17a35 100644 --- a/ingestion/src/metadata/clients/aws_client.py +++ b/ingestion/src/metadata/clients/aws_client.py @@ -40,7 +40,11 @@ class AWSClient: AWSCredentials, ) - self.config = AWSCredentials.parse_obj(config) if config else config + self.config = ( + config + if isinstance(config, AWSCredentials) + else (AWSCredentials.parse_obj(config) if config else config) + ) def _get_session(self) -> Session: if ( diff --git a/ingestion/src/metadata/utils/secrets/secrets_manager_factory.py b/ingestion/src/metadata/utils/secrets/secrets_manager_factory.py index 2c9bcfcbee0..12cb0ddecaa 100644 --- a/ingestion/src/metadata/utils/secrets/secrets_manager_factory.py +++ b/ingestion/src/metadata/utils/secrets/secrets_manager_factory.py @@ -64,9 +64,15 @@ class SecretsManagerFactory(metaclass=Singleton): or secrets_manager_provider == SecretsManagerProvider.noop ): return NoopSecretsManager() - if secrets_manager_provider == SecretsManagerProvider.aws: + if secrets_manager_provider in ( + SecretsManagerProvider.aws, + SecretsManagerProvider.managed_aws_ssm, + ): return AWSSecretsManager(credentials) - if secrets_manager_provider == SecretsManagerProvider.aws_ssm: + if secrets_manager_provider in ( + SecretsManagerProvider.aws_ssm, + SecretsManagerProvider.managed_aws_ssm, + ): return AWSSSMSecretsManager(credentials) raise NotImplementedError(f"[{secrets_manager_provider}] is not implemented.") diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 6cd11f2c744..13a6045ba55 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -77,6 +77,7 @@ import org.openmetadata.service.resources.CollectionRegistry; import org.openmetadata.service.resources.tags.TagLabelCache; import org.openmetadata.service.secrets.SecretsManager; import org.openmetadata.service.secrets.SecretsManagerFactory; +import org.openmetadata.service.secrets.SecretsManagerUpdateService; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.NoopAuthorizer; import org.openmetadata.service.security.NoopFilter; @@ -157,6 +158,9 @@ public class OpenMetadataApplication extends Application 0) { + try { + Thread.sleep(WAIT_TIME_BETWEEN_STORE_CALLS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/InMemorySecretsManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/InMemorySecretsManager.java index 4f08ef2d9ac..7dcde7f77a1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/InMemorySecretsManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/InMemorySecretsManager.java @@ -27,7 +27,7 @@ public class InMemorySecretsManager extends ExternalSecretsManager { @Getter private final Map secretsMap = new HashMap<>(); protected InMemorySecretsManager(SecretsManagerProvider secretsManagerProvider, String clusterPrefix) { - super(secretsManagerProvider, clusterPrefix); + super(secretsManagerProvider, clusterPrefix, 0); } public static InMemorySecretsManager getInstance(String clusterPrefix) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/NoopSecretsManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/NoopSecretsManager.java index 8f99eef40f4..562ed6114ee 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/NoopSecretsManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/NoopSecretsManager.java @@ -13,56 +13,14 @@ package org.openmetadata.service.secrets; -import com.google.common.annotations.VisibleForTesting; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider; -import org.openmetadata.service.exception.InvalidServiceConnectionException; -import org.openmetadata.service.fernet.Fernet; -import org.openmetadata.service.util.JsonUtils; public class NoopSecretsManager extends SecretsManager { private static NoopSecretsManager INSTANCE; - private Fernet fernet; - private NoopSecretsManager(String clusterPrefix, SecretsManagerProvider secretsManagerProvider) { super(secretsManagerProvider, clusterPrefix); - this.fernet = Fernet.getInstance(); - } - - @Override - public Object encryptOrDecryptServiceConnectionConfig( - Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt) { - try { - Class clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType)); - Object newConnectionConfig = JsonUtils.convertValue(connectionConfig, clazz); - encryptOrDecryptField(newConnectionConfig, "Password", clazz, encrypt); - return newConnectionConfig; - } catch (Exception e) { - throw InvalidServiceConnectionException.byMessage( - connectionType, String.format("Failed to construct connection instance of %s", connectionType)); - } - } - - private void encryptOrDecryptField(Object connConfig, String field, Class clazz, boolean encrypt) - throws InvocationTargetException, IllegalAccessException { - try { - Method getPasswordMethod = clazz.getMethod("get" + field); - Method setPasswordMethod = clazz.getMethod("set" + field, String.class); - String password = (String) getPasswordMethod.invoke(connConfig); - if (password != null && !password.equals("")) { - if (!Fernet.isTokenized(password) && encrypt) { - password = fernet.encrypt(password); - } else if (Fernet.isTokenized(password) && !encrypt) { - password = fernet.decrypt(password); - } - setPasswordMethod.invoke(connConfig, password); - } - } catch (NoSuchMethodException ignore) { - } } public static NoopSecretsManager getInstance(String clusterPrefix, SecretsManagerProvider secretsManagerProvider) { @@ -70,8 +28,8 @@ public class NoopSecretsManager extends SecretsManager { return INSTANCE; } - @VisibleForTesting - void setFernet(Fernet fernet) { - this.fernet = fernet; + @Override + protected String storeValue(String fieldName, String value, String secretId) { + return value; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java index 2ad73c2a040..379f3d8edd4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java @@ -15,13 +15,18 @@ package org.openmetadata.service.secrets; import static java.util.Objects.isNull; -import java.util.List; +import com.google.common.annotations.VisibleForTesting; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Locale; import lombok.Getter; +import org.openmetadata.annotations.PasswordField; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider; import org.openmetadata.service.exception.InvalidServiceConnectionException; import org.openmetadata.service.exception.SecretsManagerException; +import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.util.JsonUtils; public abstract class SecretsManager { @@ -30,13 +35,115 @@ public abstract class SecretsManager { @Getter private final SecretsManagerProvider secretsManagerProvider; + private Fernet fernet; + protected SecretsManager(SecretsManagerProvider secretsManagerProvider, String clusterPrefix) { this.secretsManagerProvider = secretsManagerProvider; this.clusterPrefix = clusterPrefix; + this.fernet = Fernet.getInstance(); } - public abstract Object encryptOrDecryptServiceConnectionConfig( - Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt); + public Object encryptOrDecryptServiceConnectionConfig( + Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt) { + try { + Class clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType)); + Object newConnectionConfig = JsonUtils.convertValue(connectionConfig, clazz); + if (encrypt) { + encryptPasswordFields(newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName)); + } else { + decryptPasswordFields(newConnectionConfig); + } + return newConnectionConfig; + } catch (Exception e) { + throw InvalidServiceConnectionException.byMessage( + connectionType, String.format("Failed to encrypt connection instance of %s", connectionType)); + } + } + + private void encryptPasswordFields(Object toEncryptObject, String secretId) { + // for each get method + Arrays.stream(toEncryptObject.getClass().getMethods()) + .filter(this::isGetMethodOfObject) + .forEach( + method -> { + Object obj = getObjectFromMethod(method, toEncryptObject); + String fieldName = method.getName().replaceFirst("get", ""); + // if the object matches the package of openmetadata + if (obj != null && obj.getClass().getPackageName().startsWith("org.openmetadata")) { + // encryptPasswordFields + encryptPasswordFields(obj, buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT))); + // check if it has annotation + } else if (obj != null && method.getAnnotation(PasswordField.class) != null) { + // store value if proceed + String newFieldValue = storeValue(fieldName, (String) obj, secretId); + // get setMethod + Method toSet = getToSetMethod(toEncryptObject, obj, fieldName); + // set new value + setValueInMethod( + toEncryptObject, + Fernet.isTokenized(newFieldValue) ? newFieldValue : fernet.encrypt(newFieldValue), + toSet); + } + }); + } + + private void decryptPasswordFields(Object toDecryptObject) { + // for each get method + Arrays.stream(toDecryptObject.getClass().getMethods()) + .filter(this::isGetMethodOfObject) + .forEach( + method -> { + Object obj = getObjectFromMethod(method, toDecryptObject); + String fieldName = method.getName().replaceFirst("get", ""); + // if the object matches the package of openmetadata + if (obj != null && obj.getClass().getPackageName().startsWith("org.openmetadata")) { + // encryptPasswordFields + decryptPasswordFields(obj); + // check if it has annotation + } else if (obj != null && method.getAnnotation(PasswordField.class) != null) { + String fieldValue = (String) obj; + // get setMethod + Method toSet = getToSetMethod(toDecryptObject, obj, fieldName); + // set new value + setValueInMethod( + toDecryptObject, Fernet.isTokenized(fieldValue) ? fernet.decrypt(fieldValue) : fieldValue, toSet); + } + }); + } + + protected abstract String storeValue(String fieldName, String value, String secretId); + + private void setValueInMethod(Object toEncryptObject, String fieldValue, Method toSet) { + try { + toSet.invoke(toEncryptObject, fieldValue); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new SecretsManagerException(e.getMessage()); + } + } + + private Method getToSetMethod(Object toEncryptObject, Object obj, String fieldName) { + try { + return toEncryptObject.getClass().getMethod("set" + fieldName, obj.getClass()); + } catch (NoSuchMethodException e) { + throw new SecretsManagerException(e.getMessage()); + } + } + + private Object getObjectFromMethod(Method method, Object toEncryptObject) { + Object obj; + try { + obj = method.invoke(toEncryptObject); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new SecretsManagerException(e.getMessage()); + } + return obj; + } + + private boolean isGetMethodOfObject(Method method) { + return method.getName().startsWith("get") + && !method.getReturnType().equals(Void.TYPE) + && !method.getReturnType().isPrimitive(); + } protected String getSecretSeparator() { return "/"; @@ -46,17 +153,25 @@ public abstract class SecretsManager { return true; } - protected String buildSecretId(String... secretIdValues) { + protected String buildSecretId(boolean addClusterPrefix, String... secretIdValues) { StringBuilder format = new StringBuilder(); - format.append(startsWithSeparator() ? getSecretSeparator() : ""); - format.append(clusterPrefix); - for (String secretIdValue : List.of(secretIdValues)) { - if (isNull(secretIdValue)) { - throw new SecretsManagerException("Cannot build a secret id with null values."); - } - format.append(getSecretSeparator()); + if (addClusterPrefix) { + format.append(startsWithSeparator() ? getSecretSeparator() : ""); + format.append(clusterPrefix); + } else { format.append("%s"); } + // skip first one in case of addClusterPrefix is false to avoid adding extra separator at the beginning + Arrays.stream(secretIdValues) + .skip(addClusterPrefix ? 0 : 1) + .forEach( + secretIdValue -> { + if (isNull(secretIdValue)) { + throw new SecretsManagerException("Cannot build a secret id with null values."); + } + format.append(getSecretSeparator()); + format.append("%s"); + }); return String.format(format.toString(), (Object[]) secretIdValues).toLowerCase(); } @@ -72,13 +187,8 @@ public abstract class SecretsManager { return serviceType.value().toLowerCase(Locale.ROOT); } - public void validateServiceConnection(Object connectionConfig, String connectionType, ServiceType serviceType) { - try { - Class clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType)); - JsonUtils.readValue(JsonUtils.pojoToJson(connectionConfig), clazz); - } catch (Exception exception) { - throw InvalidServiceConnectionException.byMessage( - connectionType, String.format("Failed to construct connection instance of %s", connectionType)); - } + @VisibleForTesting + void setFernet(Fernet fernet) { + this.fernet = fernet; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerFactory.java index 1d528b341ae..75c410a96e4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerFactory.java @@ -35,7 +35,6 @@ public class SecretsManagerFactory { case NOOP: case AWS_SSM: case AWS: - case IN_MEMORY: secretsManager = NoopSecretsManager.getInstance(clusterName, secretsManagerProvider); break; case MANAGED_AWS: @@ -44,6 +43,9 @@ public class SecretsManagerFactory { case MANAGED_AWS_SSM: secretsManager = AWSSSMSecretsManager.getInstance(config, clusterName); break; + case IN_MEMORY: + secretsManager = InMemorySecretsManager.getInstance(clusterName); + break; default: throw new IllegalArgumentException("Not implemented secret manager store: " + secretsManagerProvider); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java new file mode 100644 index 00000000000..fcbfd4a039c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java @@ -0,0 +1,159 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.ServiceConnectionEntityInterface; +import org.openmetadata.schema.ServiceEntityInterface; +import org.openmetadata.service.exception.SecretsManagerMigrationException; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.ServiceEntityRepository; +import org.openmetadata.service.resources.CollectionRegistry; +import org.openmetadata.service.resources.CollectionRegistry.CollectionDetails; +import org.openmetadata.service.resources.services.ServiceEntityResource; +import org.openmetadata.service.util.EntityUtil; + +/** + * Update service using the configured secret manager. + * + *

- It will update all the services entities with connection parameters + */ +@Slf4j +public class SecretsManagerUpdateService { + private final SecretsManager secretManager; + private final SecretsManager oldSecretManager; + + private final Map, ServiceEntityRepository> + connectionTypeRepositoriesMap; + + public SecretsManagerUpdateService(SecretsManager secretsManager, String clusterName) { + this.secretManager = secretsManager; + this.connectionTypeRepositoriesMap = retrieveConnectionTypeRepositoriesMap(); + // by default, it is going to be non-managed secrets manager since decrypt is the same for all of them + this.oldSecretManager = SecretsManagerFactory.createSecretsManager(null, clusterName); + } + + public void updateEntities() { + updateServices(); + } + + private void updateServices() { + LOG.info( + String.format( + "Checking if services updating is needed for secrets manager: [%s]", + secretManager.getSecretsManagerProvider().value())); + List notStoredServices = retrieveServices(); + if (!notStoredServices.isEmpty()) { + notStoredServices.forEach(this::updateService); + } + } + + private void updateService(ServiceEntityInterface serviceEntityInterface) { + ServiceEntityRepository repository = + connectionTypeRepositoriesMap.get(serviceEntityInterface.getConnection().getClass()); + try { + ServiceEntityInterface service = repository.dao.findEntityById(serviceEntityInterface.getId()); + // we have to decrypt using the old secrets manager and encrypt again with the new one + service + .getConnection() + .setConfig( + oldSecretManager.encryptOrDecryptServiceConnectionConfig( + service.getConnection().getConfig(), + service.getServiceType().value(), + service.getName(), + repository.getServiceType(), + false)); + service + .getConnection() + .setConfig( + secretManager.encryptOrDecryptServiceConnectionConfig( + service.getConnection().getConfig(), + service.getServiceType().value(), + service.getName(), + repository.getServiceType(), + true)); + repository.dao.update(service); + } catch (IOException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + } + + private List retrieveServices() { + return connectionTypeRepositoriesMap.values().stream() + .map(this::retrieveServices) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + private List retrieveServices(ServiceEntityRepository serviceEntityRepository) { + try { + return serviceEntityRepository + .listAfter( + null, + EntityUtil.Fields.EMPTY_FIELDS, + new ListFilter(), + serviceEntityRepository.dao.listCount(new ListFilter()), + null) + .getData().stream() + .map(ServiceEntityInterface.class::cast) + .filter( + service -> + !Objects.isNull(service.getConnection()) && !Objects.isNull(service.getConnection().getConfig())) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + } + + private Map, ServiceEntityRepository> + retrieveConnectionTypeRepositoriesMap() { + Map, ServiceEntityRepository> + connectionTypeRepositoriesMap = + CollectionRegistry.getInstance().getCollectionMap().values().stream() + .map(this::retrieveServiceRepository) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toMap(ServiceEntityRepository::getServiceConnectionClass, Function.identity())); + if (connectionTypeRepositoriesMap.isEmpty()) { + throw new SecretsManagerMigrationException("Unexpected error: ServiceRepository not found."); + } + return connectionTypeRepositoriesMap; + } + + private Optional> retrieveServiceRepository(CollectionDetails collectionDetails) { + Class collectionDetailsClass = extractCollectionDetailsClass(collectionDetails); + if (ServiceEntityResource.class.isAssignableFrom(collectionDetailsClass)) { + return Optional.of( + ((ServiceEntityResource) collectionDetails.getResource()).getServiceEntityRepository()); + } + return Optional.empty(); + } + + private Class extractCollectionDetailsClass(CollectionDetails collectionDetails) { + Class collectionDetailsClass; + try { + collectionDetailsClass = Class.forName(collectionDetails.getResourceClass()); + } catch (ClassNotFoundException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + return collectionDetailsClass; + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index 80d3fdbee6e..5ce591ef589 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -374,6 +374,10 @@ public abstract class EntityResourceTest authHeaders) { @@ -220,7 +254,8 @@ public class DashboardServiceResourceTest extends EntityResourceTest createEntity(createRequest(test).withDescription(null).withConnection(dbConn), ADMIN_AUTH_HEADERS), BAD_REQUEST, - "InvalidServiceConnectionException for service [Snowflake] due to [Failed to construct connection instance of Snowflake]"); + "InvalidServiceConnectionException for service [Snowflake] due to [Failed to encrypt connection instance of Snowflake]"); DatabaseService service = createAndCheckEntity(createRequest(test).withDescription(null), ADMIN_AUTH_HEADERS); // Update database description and ingestion service that are null - CreateDatabaseService update = createRequest(test).withDescription("description1"); + CreateDatabaseService update = createPutRequest(test).withDescription("description1"); ChangeDescription change = getChangeDescription(service.getVersion()); fieldAdded(change, "description", "description1"); @@ -180,7 +183,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest updateEntity(update, OK, ADMIN_AUTH_HEADERS), BAD_REQUEST, - "InvalidServiceConnectionException for service [Snowflake] due to [Failed to construct connection instance of Snowflake]"); + "InvalidServiceConnectionException for service [Snowflake] due to [Failed to encrypt connection instance of Snowflake]"); } @Test @@ -252,6 +255,18 @@ public class DatabaseServiceResourceTest extends EntityResourceTest authHeaders) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/MetadataServiceResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/MetadataServiceResourceTest.java index 678801372dc..7a1b8c2e293 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/MetadataServiceResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/MetadataServiceResourceTest.java @@ -13,6 +13,7 @@ import static org.openmetadata.service.util.TestUtils.assertResponse; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.Locale; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; @@ -25,6 +26,7 @@ import org.openmetadata.schema.services.connections.metadata.AmundsenConnection; import org.openmetadata.schema.services.connections.metadata.AtlasConnection; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.service.Entity; +import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.services.metadata.MetadataServiceResource; import org.openmetadata.service.util.JsonUtils; @@ -98,6 +100,7 @@ public class MetadataServiceResourceTest extends EntityResourceTest authHeaders) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/PipelineServiceResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/PipelineServiceResourceTest.java index 84f1e869cab..67530e947c0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/PipelineServiceResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/PipelineServiceResourceTest.java @@ -157,7 +157,7 @@ public class PipelineServiceResourceTest extends EntityResourceTest getSecretCaptor = ArgumentCaptor.forClass(GetParameterRequest.class); - verify(ssmClient, times(times)).getParameter(getSecretCaptor.capture()); - for (int i = 0; i < times; i++) { - assertEquals(expectedSecretId, getSecretCaptor.getAllValues().get(i).name()); - } - } - - @Override - void verifyClientCalls(Object expectedAuthProviderConfig, String expectedSecretId) throws JsonProcessingException { - ArgumentCaptor createSecretCaptor = ArgumentCaptor.forClass(PutParameterRequest.class); - if (Objects.isNull(expectedAuthProviderConfig)) { - verifyNoInteractions(ssmClient); - } else { - verify(ssmClient).putParameter(createSecretCaptor.capture()); - assertEquals(expectedSecretId, createSecretCaptor.getValue().name()); - assertNotNull(createSecretCaptor.getValue().value()); - assertEquals(JsonUtils.pojoToJson(expectedAuthProviderConfig), createSecretCaptor.getValue().value()); - } - } - @Override SecretsManagerProvider expectedSecretManagerProvider() { return SecretsManagerProvider.AWS_SSM; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/AWSSecretsManagerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/AWSSecretsManagerTest.java index fec2995acae..6f6fd8ed7d8 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/AWSSecretsManagerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/AWSSecretsManagerTest.java @@ -12,23 +12,13 @@ */ package org.openmetadata.service.secrets; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import com.fasterxml.jackson.core.JsonProcessingException; -import java.util.Objects; -import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvider; -import org.openmetadata.service.util.JsonUtils; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; -import software.amazon.awssdk.services.secretsmanager.model.CreateSecretRequest; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; @@ -56,28 +46,6 @@ public class AWSSecretsManagerTest extends ExternalSecretsManagerTest { } } - @Override - void verifySecretIdGetCalls(String expectedSecretId, int times) { - ArgumentCaptor getSecretCaptor = ArgumentCaptor.forClass(GetSecretValueRequest.class); - verify(secretsManagerClient, times(times)).getSecretValue(getSecretCaptor.capture()); - for (int i = 0; i < times; i++) { - assertEquals(expectedSecretId, getSecretCaptor.getAllValues().get(i).secretId()); - } - } - - @Override - void verifyClientCalls(Object expectedAuthProviderConfig, String expectedSecretId) throws JsonProcessingException { - ArgumentCaptor createSecretCaptor = ArgumentCaptor.forClass(CreateSecretRequest.class); - if (Objects.isNull(expectedAuthProviderConfig)) { - verifyNoInteractions(secretsManagerClient); - } else { - verify(secretsManagerClient).createSecret(createSecretCaptor.capture()); - assertEquals(expectedSecretId, createSecretCaptor.getValue().name()); - assertNotNull(createSecretCaptor.getValue().secretString()); - assertEquals(JsonUtils.pojoToJson(expectedAuthProviderConfig), createSecretCaptor.getValue().secretString()); - } - } - @Override SecretsManagerProvider expectedSecretManagerProvider() { return SecretsManagerProvider.AWS; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java index 9f933a9329e..c255e097d5d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java @@ -13,9 +13,7 @@ package org.openmetadata.service.secrets; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; -import com.fasterxml.jackson.core.JsonProcessingException; import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.BeforeEach; @@ -30,9 +28,6 @@ import org.openmetadata.schema.services.connections.metadata.SecretsManagerProvi @ExtendWith(MockitoExtension.class) public abstract class ExternalSecretsManagerTest { - static final boolean ENCRYPT = true; - static final String AUTH_PROVIDER_SECRET_ID_PREFIX = "auth-provider"; - static final String TEST_CONNECTION_SECRET_ID_PREFIX = "test-connection-temp"; static final boolean DECRYPT = false; static final String EXPECTED_CONNECTION_JSON = "{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"password\":\"openmetadata-test\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true,\"supportsQueryComment\":true}"; @@ -66,11 +61,6 @@ public abstract class ExternalSecretsManagerTest { abstract void mockClientGetValue(String value); - abstract void verifySecretIdGetCalls(String expectedSecretId, int times); - - abstract void verifyClientCalls(Object expectedAuthProviderConfig, String expectedSecretId) - throws JsonProcessingException; - void testEncryptDecryptServiceConnection(boolean decrypt) { MysqlConnection mysqlConnection = new MysqlConnection(); mysqlConnection.setPassword("openmetadata-test"); @@ -82,7 +72,6 @@ public abstract class ExternalSecretsManagerTest { mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt); assertEquals(mysqlConnection, actualConfig); - assertSame(mysqlConnection, actualConfig); } abstract SecretsManagerProvider expectedSecretManagerProvider(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/NoopSecretsManagerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/NoopSecretsManagerTest.java index 0cb8da5380b..36d05ad71c9 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/NoopSecretsManagerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/NoopSecretsManagerTest.java @@ -62,7 +62,7 @@ public class NoopSecretsManagerTest { @Test void testDecryptDatabaseServiceConnectionConfig() { - testEncryptDecryptServiceConnection(ENCRYPTED_VALUE, DECRYPTED_VALUE, DECRYPT); + testEncryptDecryptServiceConnection(DECRYPTED_VALUE, ENCRYPTED_VALUE, DECRYPT); } @Test @@ -94,7 +94,7 @@ public class NoopSecretsManagerTest { private void testEncryptDecryptServiceConnection(String encryptedValue, String decryptedValue, boolean decrypt) { MysqlConnection mysqlConnection = new MysqlConnection(); - mysqlConnection.setPassword(encryptedValue); + mysqlConnection.setPassword(decrypt ? encryptedValue : decryptedValue); CreateDatabaseService.DatabaseServiceType databaseServiceType = CreateDatabaseService.DatabaseServiceType.Mysql; String connectionName = "test"; @@ -102,7 +102,7 @@ public class NoopSecretsManagerTest { secretsManager.encryptOrDecryptServiceConnectionConfig( mysqlConnection, databaseServiceType.value(), connectionName, ServiceType.DATABASE, decrypt); - assertEquals(decryptedValue, ((MysqlConnection) actualConfig).getPassword()); + assertEquals(decrypt ? decryptedValue : encryptedValue, ((MysqlConnection) actualConfig).getPassword()); assertNotSame(mysqlConnection, actualConfig); } }