diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql index 7f52309653e..c598788722a 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql @@ -73,3 +73,7 @@ CREATE TABLE IF NOT EXISTS automations_workflow ( PRIMARY KEY (id), UNIQUE (name) ); + +-- Do not store OM server connection, we'll set it dynamically on the resource +UPDATE ingestion_pipeline_entity +SET json = JSON_REMOVE(json, '$.openMetadataServerConnection'); diff --git a/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql index 96768946efd..558614e819e 100644 --- a/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql @@ -73,3 +73,7 @@ CREATE TABLE IF NOT EXISTS automations_workflow ( PRIMARY KEY (id), UNIQUE (name) ); + +-- Do not store OM server connection, we'll set it dynamically on the resource +UPDATE ingestion_pipeline_entity +SET json = json::jsonb #- '{openMetadataServerConnection}'; diff --git a/ingestion/tests/integration/ometa/test_ometa_workflow_api.py b/ingestion/tests/integration/ometa/test_ometa_workflow_api.py index e5c850ec640..525a2334880 100644 --- a/ingestion/tests/integration/ometa/test_ometa_workflow_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_workflow_api.py @@ -28,6 +28,7 @@ from metadata.generated.schema.entity.automations.workflow import ( ) from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( MysqlConnection, + MySQLType, ) from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, @@ -72,6 +73,7 @@ class OMetaWorkflowTest(TestCase): fullyQualifiedName="test", request=TestServiceConnectionRequest( serviceType=ServiceType.Database, + connectionType=MySQLType.Mysql.value, connection=DatabaseConnection( config=MysqlConnection( username="username", @@ -91,6 +93,7 @@ class OMetaWorkflowTest(TestCase): workflowType=WorkflowType.TEST_CONNECTION, request=TestServiceConnectionRequest( serviceType=ServiceType.Database, + connectionType=MySQLType.Mysql.value, connection=DatabaseConnection( config=MysqlConnection( username="username", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java index 7cf58109ee7..a6abc006e59 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java @@ -85,19 +85,25 @@ public class IngestionPipelineRepository extends EntityRepository { @@ -15,8 +17,6 @@ public class WorkflowRepository extends EntityRepository { private static final String UPDATE_FIELDS = "owner"; private static final String PATCH_FIELDS = "owner"; - private static PipelineServiceClient pipelineServiceClient; - public WorkflowRepository(CollectionDAO dao) { super( WorkflowResource.COLLECTION_PATH, @@ -28,10 +28,6 @@ public class WorkflowRepository extends EntityRepository { UPDATE_FIELDS); } - public void setPipelineServiceClient(PipelineServiceClient client) { - pipelineServiceClient = client; - } - @Override public Workflow setFields(Workflow entity, EntityUtil.Fields fields) throws IOException { return entity.withOwner(fields.contains(Entity.FIELD_OWNER) ? getOwner(entity) : null); @@ -48,12 +44,19 @@ public class WorkflowRepository extends EntityRepository { @Override public void storeEntity(Workflow entity, boolean update) throws IOException { EntityReference owner = entity.getOwner(); + OpenMetadataConnection openmetadataConnection = entity.getOpenMetadataServerConnection(); + SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager(); + + if (secretsManager != null) { + entity = secretsManager.encryptOrDecryptWorkflow(entity, true); + } + // Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships - entity.withOwner(null).withHref(null); + entity.withOwner(null).withHref(null).withOpenMetadataServerConnection(null); store(entity, update); // Restore the relationships - entity.withOwner(owner); + entity.withOwner(owner).withOpenMetadataServerConnection(openmetadataConnection); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/operations/WorkflowResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/operations/WorkflowResource.java index 43495eefd72..a1f9519749b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/operations/WorkflowResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/operations/WorkflowResource.java @@ -1,5 +1,6 @@ package org.openmetadata.service.resources.operations; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.service.Entity.FIELD_OWNER; import com.google.inject.Inject; @@ -14,6 +15,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import java.io.IOException; import java.util.UUID; +import java.util.stream.Collectors; import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -35,14 +37,17 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.ServiceConnectionEntityInterface; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.automations.CreateWorkflow; +import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.entity.automations.WorkflowStatus; import org.openmetadata.schema.entity.automations.WorkflowType; import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.sdk.PipelineServiceClient; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; @@ -52,7 +57,13 @@ import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.WorkflowRepository; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.secrets.SecretsManager; +import org.openmetadata.service.secrets.SecretsManagerFactory; +import org.openmetadata.service.secrets.converter.ClassConverterFactory; +import org.openmetadata.service.secrets.masker.EntityMaskerFactory; +import org.openmetadata.service.security.AuthorizationException; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.OpenMetadataConnectionBuilder; import org.openmetadata.service.util.RestUtil; @@ -90,7 +101,6 @@ public class WorkflowResource extends EntityResource { @@ -162,7 +172,13 @@ public class WorkflowResource extends EntityResource workflows = + super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); + workflows.setData( + listOrEmpty(workflows.getData()).stream() + .map(service -> decryptOrNullify(securityContext, service)) + .collect(Collectors.toList())); + return workflows; } @GET @@ -215,7 +231,7 @@ public class WorkflowResource extends EntityResource clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType); Object newConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig); return encryptOrDecryptPasswordFields( - newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt); + newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt, true); } catch (Exception e) { throw InvalidServiceConnectionException.byMessage( connectionType, String.format("Failed to encrypt connection instance of %s", connectionType)); @@ -68,36 +72,76 @@ public abstract class SecretsManager { if (authenticationMechanism != null) { AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism); try { - encryptOrDecryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), encrypt); + encryptOrDecryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), encrypt, true); } catch (Exception e) { - throw InvalidServiceConnectionException.byMessage( - name, String.format("Failed to encrypt user bot instance [%s]", name)); + throw new CustomExceptionMessage( + Response.Status.BAD_REQUEST, String.format("Failed to encrypt user bot instance [%s]", name)); } } } public void encryptOrDecryptIngestionPipeline(IngestionPipeline ingestionPipeline, boolean encrypt) { + OpenMetadataConnection openMetadataConnection = + encryptOrDecryptOpenMetadataConnection(ingestionPipeline.getOpenMetadataServerConnection(), encrypt, true); + ingestionPipeline.setOpenMetadataServerConnection(null); + // we don't store OM conn sensitive data IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline); try { encryptOrDecryptPasswordFields( - ingestionPipeline, buildSecretId(true, "pipeline", ingestionPipeline.getName()), encrypt); + ingestionPipeline, buildSecretId(true, "pipeline", ingestionPipeline.getName()), encrypt, true); } catch (Exception e) { - throw InvalidServiceConnectionException.byMessage( - ingestionPipeline.getName(), + throw new CustomExceptionMessage( + Response.Status.BAD_REQUEST, String.format("Failed to encrypt ingestion pipeline instance [%s]", ingestionPipeline.getName())); } + ingestionPipeline.setOpenMetadataServerConnection(openMetadataConnection); } - private Object encryptOrDecryptPasswordFields(Object targetObject, String name, boolean encrypt) { + public Workflow encryptOrDecryptWorkflow(Workflow workflow, boolean encrypt) { + OpenMetadataConnection openMetadataConnection = + encryptOrDecryptOpenMetadataConnection(workflow.getOpenMetadataServerConnection(), encrypt, true); + Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow); + // we don't store OM conn sensitive data + workflowConverted.setOpenMetadataServerConnection(null); + try { + encryptOrDecryptPasswordFields( + workflowConverted, buildSecretId(true, "workflow", workflow.getName()), encrypt, true); + } catch (Exception e) { + throw new CustomExceptionMessage( + Response.Status.BAD_REQUEST, String.format("Failed to encrypt workflow instance [%s]", workflow.getName())); + } + workflowConverted.setOpenMetadataServerConnection(openMetadataConnection); + return workflowConverted; + } + + public OpenMetadataConnection encryptOrDecryptOpenMetadataConnection( + OpenMetadataConnection openMetadataConnection, boolean encrypt, boolean store) { + if (openMetadataConnection != null) { + OpenMetadataConnection openMetadataConnectionConverted = + (OpenMetadataConnection) + ClassConverterFactory.getConverter(OpenMetadataConnection.class).convert(openMetadataConnection); + try { + encryptOrDecryptPasswordFields( + openMetadataConnectionConverted, buildSecretId(true, "serverconnection"), encrypt, store); + } catch (Exception e) { + throw new CustomExceptionMessage( + Response.Status.BAD_REQUEST, "Failed to encrypt OpenMetadataConnection instance."); + } + return openMetadataConnectionConverted; + } + return null; + } + + private Object encryptOrDecryptPasswordFields(Object targetObject, String name, boolean encrypt, boolean store) { if (encrypt) { - encryptPasswordFields(targetObject, name); + encryptPasswordFields(targetObject, name, store); } else { decryptPasswordFields(targetObject); } return targetObject; } - private void encryptPasswordFields(Object toEncryptObject, String secretId) { + private void encryptPasswordFields(Object toEncryptObject, String secretId, boolean store) { if (!DO_NOT_ENCRYPT_CLASSES.contains(toEncryptObject.getClass())) { // for each get method Arrays.stream(toEncryptObject.getClass().getMethods()) @@ -109,17 +153,19 @@ public abstract class SecretsManager { // if the object matches the package of openmetadata if (obj != null && obj.getClass().getPackageName().startsWith("org.openmetadata")) { // encryptPasswordFields - encryptPasswordFields(obj, buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT))); + encryptPasswordFields(obj, buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT)), store); // check if it has annotation } else if (obj != null && method.getAnnotation(PasswordField.class) != null) { // store value if proceed - String newFieldValue = storeValue(fieldName, fernet.decryptIfApplies((String) obj), secretId); + String newFieldValue = storeValue(fieldName, fernet.decryptIfApplies((String) obj), secretId, store); // get setMethod Method toSet = ReflectionUtil.getToSetMethod(toEncryptObject, obj, fieldName); // set new value ReflectionUtil.setValueInMethod( toEncryptObject, - Fernet.isTokenized(newFieldValue) ? newFieldValue : fernet.encrypt(newFieldValue), + Fernet.isTokenized(newFieldValue) + ? newFieldValue + : store ? fernet.encrypt(newFieldValue) : newFieldValue, toSet); } }); @@ -150,7 +196,7 @@ public abstract class SecretsManager { }); } - protected abstract String storeValue(String fieldName, String value, String secretId); + protected abstract String storeValue(String fieldName, String value, String secretId, boolean store); protected String getSecretSeparator() { return "/"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index 90517574af4..431b57c5995 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -18,6 +18,8 @@ import java.util.HashMap; import java.util.Map; import lombok.Getter; import org.openmetadata.schema.auth.SSOAuthMechanism; +import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; +import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.metadataIngestion.DbtPipeline; import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtGCSConfig; import org.openmetadata.schema.security.credentials.GCSCredentials; @@ -50,6 +52,8 @@ public class ClassConverterFactory { put(GCSConfig.class, new GCSConfigClassConverter()); put(BigQueryConnection.class, new BigQueryConnectionClassConverter()); put(DbtGCSConfig.class, new DbtGCSConfigClassConverter()); + put(TestServiceConnectionRequest.class, new TestServiceConnectionRequestClassConverter()); + put(Workflow.class, new WorkflowClassConverter()); } }); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestServiceConnectionRequestClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestServiceConnectionRequestClassConverter.java new file mode 100644 index 00000000000..6431815cc13 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestServiceConnectionRequestClassConverter.java @@ -0,0 +1,70 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.ServiceConnectionEntityInterface; +import org.openmetadata.schema.api.services.DatabaseConnection; +import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; +import org.openmetadata.schema.entity.services.MetadataConnection; +import org.openmetadata.schema.type.DashboardConnection; +import org.openmetadata.schema.type.MessagingConnection; +import org.openmetadata.schema.type.MlModelConnection; +import org.openmetadata.schema.type.PipelineConnection; +import org.openmetadata.service.exception.InvalidServiceConnectionException; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.ReflectionUtil; + +/** Converter class to get an `TestServiceConnectionRequest` object. */ +public class TestServiceConnectionRequestClassConverter extends ClassConverter { + + private static final List> CONNECTION_CLASSES = + List.of( + DatabaseConnection.class, + DashboardConnection.class, + MessagingConnection.class, + PipelineConnection.class, + MlModelConnection.class, + MetadataConnection.class); + + public TestServiceConnectionRequestClassConverter() { + super(TestServiceConnectionRequest.class); + } + + @Override + public Object convert(Object object) { + TestServiceConnectionRequest testServiceConnectionRequest = + (TestServiceConnectionRequest) JsonUtils.convertValue(object, this.clazz); + + try { + Class clazz = + ReflectionUtil.createConnectionConfigClass( + testServiceConnectionRequest.getConnectionType(), testServiceConnectionRequest.getServiceType()); + + tryToConvertOrFail(testServiceConnectionRequest.getConnection(), CONNECTION_CLASSES) + .ifPresent(testServiceConnectionRequest::setConnection); + + Object newConnectionConfig = + ClassConverterFactory.getConverter(clazz) + .convert(((ServiceConnectionEntityInterface) testServiceConnectionRequest.getConnection()).getConfig()); + ((ServiceConnectionEntityInterface) testServiceConnectionRequest.getConnection()).setConfig(newConnectionConfig); + } catch (Exception e) { + throw InvalidServiceConnectionException.byMessage( + testServiceConnectionRequest.getConnectionType(), + String.format("Failed to convert class instance of %s", testServiceConnectionRequest.getConnectionType())); + } + + return testServiceConnectionRequest; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java new file mode 100644 index 00000000000..61eb23802bb --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; +import org.openmetadata.schema.entity.automations.Workflow; +import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `Workflow` object. */ +public class WorkflowClassConverter extends ClassConverter { + + public WorkflowClassConverter() { + super(Workflow.class); + } + + @Override + public Object convert(Object object) { + Workflow workflow = (Workflow) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail(workflow.getRequest(), List.of(TestServiceConnectionRequest.class)) + .ifPresent(workflow::setRequest); + + if (workflow.getOpenMetadataServerConnection() != null) { + workflow.setOpenMetadataServerConnection( + (OpenMetadataConnection) + ClassConverterFactory.getConverter(OpenMetadataConnection.class) + .convert(workflow.getOpenMetadataServerConnection())); + } + + return workflow; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/EntityMasker.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/EntityMasker.java index 65100183b02..7e26a1d96aa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/EntityMasker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/EntityMasker.java @@ -15,6 +15,7 @@ package org.openmetadata.service.secrets.masker; import java.util.Set; import org.openmetadata.schema.auth.BasicAuthMechanism; +import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.teams.AuthenticationMechanism; @@ -32,6 +33,8 @@ public abstract class EntityMasker { public abstract void maskIngestionPipeline(IngestionPipeline ingestionPipeline); + public abstract Workflow maskWorkflow(Workflow workflow); + public abstract Object unmaskServiceConnectionConfig( Object connectionConfig, Object originalConnectionConfig, String connectionType, ServiceType serviceType); @@ -42,4 +45,6 @@ public abstract class EntityMasker { String name, AuthenticationMechanism authenticationMechanism, AuthenticationMechanism originalAuthenticationMechanism); + + public abstract Workflow unmaskWorkflow(Workflow workflow, Workflow originalWorkflow); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/NoopEntityMasker.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/NoopEntityMasker.java index c7e27475c16..924df0cb076 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/NoopEntityMasker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/NoopEntityMasker.java @@ -13,6 +13,7 @@ package org.openmetadata.service.secrets.masker; +import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.teams.AuthenticationMechanism; @@ -45,6 +46,11 @@ public class NoopEntityMasker extends EntityMasker { // do nothing } + @Override + public Workflow maskWorkflow(Workflow workflow) { + return workflow; + } + @Override public Object unmaskServiceConnectionConfig( Object connectionConfig, Object originalConnectionConfig, String connectionType, ServiceType serviceType) { @@ -64,4 +70,9 @@ public class NoopEntityMasker extends EntityMasker { AuthenticationMechanism originalAuthenticationMechanism) { // do nothing } + + @Override + public Workflow unmaskWorkflow(Workflow workflow, Workflow originalWorkflow) { + return workflow; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/PasswordEntityMasker.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/PasswordEntityMasker.java index 9678d4ece14..8343b1c905f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/PasswordEntityMasker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/PasswordEntityMasker.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; import org.openmetadata.annotations.PasswordField; +import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.teams.AuthenticationMechanism; @@ -46,14 +47,17 @@ public class PasswordEntityMasker extends EntityMasker { } public Object maskServiceConnectionConfig(Object connectionConfig, String connectionType, ServiceType serviceType) { - try { - Class clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType); - Object newConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig); - maskPasswordFields(newConnectionConfig); - return newConnectionConfig; - } catch (Exception e) { - throw new EntityMaskException(String.format("Failed to mask connection instance of %s", connectionType)); + if (connectionConfig != null) { + try { + Class clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType); + Object convertedConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig); + maskPasswordFields(convertedConnectionConfig); + return convertedConnectionConfig; + } catch (Exception e) { + throw new EntityMaskException(String.format("Failed to mask connection instance of %s", connectionType)); + } } + return null; } public void maskAuthenticationMechanism(String name, AuthenticationMechanism authenticationMechanism) { @@ -68,15 +72,31 @@ public class PasswordEntityMasker extends EntityMasker { } public void maskIngestionPipeline(IngestionPipeline ingestionPipeline) { - IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline); - try { - maskPasswordFields(ingestionPipeline); - } catch (Exception e) { - throw new EntityMaskException( - String.format("Failed to mask ingestion pipeline instance [%s]", ingestionPipeline.getName())); + if (ingestionPipeline != null) { + IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline); + try { + maskPasswordFields(ingestionPipeline); + } catch (Exception e) { + throw new EntityMaskException( + String.format("Failed to mask ingestion pipeline instance [%s]", ingestionPipeline.getName())); + } } } + @Override + public Workflow maskWorkflow(Workflow workflow) { + if (workflow != null) { + Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow); + try { + maskPasswordFields(workflowConverted); + } catch (Exception e) { + throw new EntityMaskException(String.format("Failed to mask workflow instance [%s]", workflow.getName())); + } + return workflowConverted; + } + return null; + } + public Object unmaskServiceConnectionConfig( Object connectionConfig, Object originalConnectionConfig, String connectionType, ServiceType serviceType) { if (originalConnectionConfig != null && connectionConfig != null) { @@ -89,7 +109,7 @@ public class PasswordEntityMasker extends EntityMasker { unmaskPasswordFields(toUnmaskConfig, NEW_KEY, passwordsMap); return toUnmaskConfig; } catch (Exception e) { - throw new EntityMaskException(String.format("Failed to mask connection instance of %s", connectionType)); + throw new EntityMaskException(String.format("Failed to unmask connection instance of %s", connectionType)); } } return connectionConfig; @@ -97,15 +117,17 @@ public class PasswordEntityMasker extends EntityMasker { public void unmaskIngestionPipeline( IngestionPipeline ingestionPipeline, IngestionPipeline originalIngestionPipeline) { - IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline); - IngestionPipelineBuilder.addDefinedConfig(originalIngestionPipeline); - try { - Map passwordsMap = new HashMap<>(); - buildPasswordsMap(originalIngestionPipeline, NEW_KEY, passwordsMap); - unmaskPasswordFields(ingestionPipeline, NEW_KEY, passwordsMap); - } catch (Exception e) { - throw new EntityMaskException( - String.format("Failed to mask ingestion pipeline instance [%s]", ingestionPipeline.getName())); + if (ingestionPipeline != null && originalIngestionPipeline != null) { + IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline); + IngestionPipelineBuilder.addDefinedConfig(originalIngestionPipeline); + try { + Map passwordsMap = new HashMap<>(); + buildPasswordsMap(originalIngestionPipeline, NEW_KEY, passwordsMap); + unmaskPasswordFields(ingestionPipeline, NEW_KEY, passwordsMap); + } catch (Exception e) { + throw new EntityMaskException( + String.format("Failed to unmask ingestion pipeline instance [%s]", ingestionPipeline.getName())); + } } } @@ -113,7 +135,7 @@ public class PasswordEntityMasker extends EntityMasker { String name, AuthenticationMechanism authenticationMechanism, AuthenticationMechanism originalAuthenticationMechanism) { - if (authenticationMechanism != null) { + if (authenticationMechanism != null && originalAuthenticationMechanism != null) { AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism); AuthenticationMechanismBuilder.addDefinedConfig(originalAuthenticationMechanism); try { @@ -121,11 +143,29 @@ public class PasswordEntityMasker extends EntityMasker { buildPasswordsMap(originalAuthenticationMechanism, NEW_KEY, passwordsMap); unmaskPasswordFields(authenticationMechanism, NEW_KEY, passwordsMap); } catch (Exception e) { - throw new EntityMaskException(String.format("Failed to mask user bot instance [%s]", name)); + throw new EntityMaskException(String.format("Failed to unmask auth mechanism instance [%s]", name)); } } } + @Override + public Workflow unmaskWorkflow(Workflow workflow, Workflow originalWorkflow) { + if (workflow != null && originalWorkflow != null) { + Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow); + Workflow origWorkflowConverted = + (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(originalWorkflow); + try { + Map passwordsMap = new HashMap<>(); + buildPasswordsMap(origWorkflowConverted, NEW_KEY, passwordsMap); + unmaskPasswordFields(workflowConverted, NEW_KEY, passwordsMap); + return workflowConverted; + } catch (Exception e) { + throw new EntityMaskException(String.format("Failed to unmask workflow instance [%s]", workflow.getName())); + } + } + return workflow; + } + private void maskPasswordFields(Object toMaskObject) { if (!DO_NOT_MASK_CLASSES.contains(toMaskObject.getClass())) { // for each get method diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/operations/WorkflowResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/operations/WorkflowResourceTest.java index da3ca68123a..790d2e69e8b 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/operations/WorkflowResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/operations/WorkflowResourceTest.java @@ -9,6 +9,7 @@ import static org.openmetadata.service.util.TestUtils.assertListNull; import java.io.IOException; import java.util.Map; import org.apache.http.client.HttpResponseException; +import org.openmetadata.schema.api.services.DatabaseConnection; import org.openmetadata.schema.entity.automations.CreateWorkflow; import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; import org.openmetadata.schema.entity.automations.Workflow; @@ -41,10 +42,12 @@ public class WorkflowResourceTest extends EntityResourceTest authHeaders) { assertEquals(createRequest.getAirflowConfig().getConcurrency(), ingestion.getAirflowConfig().getConcurrency()); validateSourceConfig(createRequest.getSourceConfig(), ingestion.getSourceConfig(), ingestion); + assertNotNull(ingestion.getOpenMetadataServerConnection()); } @Override diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java index d66bf8157a5..a9f3bf384e1 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java @@ -20,7 +20,10 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; import org.openmetadata.schema.api.services.CreateDatabaseService; +import org.openmetadata.schema.api.services.DatabaseConnection; import org.openmetadata.schema.auth.SSOAuthMechanism; +import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; +import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.entity.services.DatabaseService; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; @@ -29,14 +32,17 @@ import org.openmetadata.schema.entity.teams.AuthenticationMechanism; import org.openmetadata.schema.metadataIngestion.DbtPipeline; import org.openmetadata.schema.metadataIngestion.SourceConfig; import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtS3Config; +import org.openmetadata.schema.security.client.GoogleSSOClientConfig; import org.openmetadata.schema.security.client.OktaSSOClientConfig; import org.openmetadata.schema.security.credentials.AWSCredentials; import org.openmetadata.schema.security.secrets.Parameters; import org.openmetadata.schema.security.secrets.SecretsManagerConfiguration; import org.openmetadata.schema.security.secrets.SecretsManagerProvider; import org.openmetadata.schema.services.connections.database.MysqlConnection; +import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; import org.openmetadata.service.Entity; import org.openmetadata.service.fernet.Fernet; +import org.openmetadata.service.util.JsonUtils; @ExtendWith(MockitoExtension.class) public abstract class ExternalSecretsManagerTest { @@ -89,6 +95,16 @@ public abstract class ExternalSecretsManagerTest { testEncryptDecryptDBTConfig(ENCRYPT); } + @Test + void testDecryptWorkflow() { + testEncryptWorkflowObject(DECRYPT); + } + + @Test + void testEncryptWorkflow() { + testEncryptWorkflowObject(ENCRYPT); + } + @Test void testReturnsExpectedSecretManagerProvider() { assertEquals(expectedSecretManagerProvider(), secretsManager.getSecretsManagerProvider()); @@ -127,12 +143,7 @@ public abstract class ExternalSecretsManagerTest { new SSOAuthMechanism().withAuthConfig(config).withSsoServiceType(SSOAuthMechanism.SsoServiceType.OKTA)); AuthenticationMechanism actualAuthenticationMechanism = - new AuthenticationMechanism() - .withAuthType(AuthenticationMechanism.AuthType.SSO) - .withConfig( - new SSOAuthMechanism() - .withSsoServiceType(SSOAuthMechanism.SsoServiceType.OKTA) - .withAuthConfig(Map.of("privateKey", "this-is-a-test"))); + JsonUtils.convertValue(expectedAuthenticationMechanism, AuthenticationMechanism.class); secretsManager.encryptOrDecryptAuthenticationMechanism("bot", actualAuthenticationMechanism, decrypt); @@ -165,20 +176,7 @@ public abstract class ExternalSecretsManagerTest { .withAwsRegion("eu-west-1"))))); IngestionPipeline actualIngestionPipeline = - new IngestionPipeline() - .withName("my-pipeline") - .withPipelineType(PipelineType.DBT) - .withService(new DatabaseService().getEntityReference().withType(Entity.DATABASE_SERVICE)) - .withSourceConfig( - new SourceConfig() - .withConfig( - Map.of( - "dbtConfigSource", - Map.of( - "dbtSecurityConfig", - Map.of( - "awsSecretAccessKey", "secret-password", - "awsRegion", "eu-west-1"))))); + JsonUtils.convertValue(expectedIngestionPipeline, IngestionPipeline.class); secretsManager.encryptOrDecryptIngestionPipeline(actualIngestionPipeline, decrypt); @@ -202,5 +200,49 @@ public abstract class ExternalSecretsManagerTest { assertEquals(expectedIngestionPipeline, actualIngestionPipeline); } + void testEncryptWorkflowObject(boolean encrypt) { + Workflow expectedWorkflow = + new Workflow() + .withName("my-workflow") + .withOpenMetadataServerConnection( + new OpenMetadataConnection() + .withSecretsManagerCredentials(new AWSCredentials().withAwsSecretAccessKey("aws-secret")) + .withSecurityConfig(new GoogleSSOClientConfig().withSecretKey("google-secret"))) + .withRequest( + new TestServiceConnectionRequest() + .withConnection( + new DatabaseConnection().withConfig(new MysqlConnection().withPassword("openmetadata-test"))) + .withServiceType(ServiceType.DATABASE) + .withConnectionType("Mysql")); + + Workflow workflow = JsonUtils.convertValue(expectedWorkflow, Workflow.class); + + Workflow actualWorkflow = secretsManager.encryptOrDecryptWorkflow(workflow, encrypt); + + if (encrypt) { + ((MysqlConnection) + ((DatabaseConnection) ((TestServiceConnectionRequest) expectedWorkflow.getRequest()).getConnection()) + .getConfig()) + .setPassword("secret:/openmetadata/workflow/my-workflow/request/connection/config/password"); + MysqlConnection mysqlConnection = + (MysqlConnection) + ((DatabaseConnection) ((TestServiceConnectionRequest) actualWorkflow.getRequest()).getConnection()) + .getConfig(); + mysqlConnection.setPassword(Fernet.getInstance().decrypt(mysqlConnection.getPassword())); + ((GoogleSSOClientConfig) (expectedWorkflow.getOpenMetadataServerConnection()).getSecurityConfig()) + .setSecretKey("secret:/openmetadata/serverconnection/securityconfig/secretkey"); + GoogleSSOClientConfig googleSSOClientConfig = + ((GoogleSSOClientConfig) (actualWorkflow.getOpenMetadataServerConnection()).getSecurityConfig()); + googleSSOClientConfig.setSecretKey(Fernet.getInstance().decrypt(googleSSOClientConfig.getSecretKey())); + ((AWSCredentials) (expectedWorkflow.getOpenMetadataServerConnection()).getSecretsManagerCredentials()) + .setAwsSecretAccessKey("secret:/openmetadata/serverconnection/secretsmanagercredentials/awssecretaccesskey"); + AWSCredentials awsCredentials = + ((AWSCredentials) (actualWorkflow.getOpenMetadataServerConnection()).getSecretsManagerCredentials()); + awsCredentials.setAwsSecretAccessKey(Fernet.getInstance().decrypt(awsCredentials.getAwsSecretAccessKey())); + } + + assertEquals(expectedWorkflow, actualWorkflow); + } + protected abstract SecretsManagerProvider expectedSecretManagerProvider(); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java index 9aa613452a9..512598ff7bc 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java @@ -7,6 +7,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.openmetadata.schema.auth.SSOAuthMechanism; +import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; +import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.metadataIngestion.DbtPipeline; import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtGCSConfig; import org.openmetadata.schema.security.credentials.GCSCredentials; @@ -33,7 +35,9 @@ public class ClassConverterFactoryTest { GcsConnection.class, GCSConfig.class, BigQueryConnection.class, - DbtGCSConfig.class + DbtGCSConfig.class, + TestServiceConnectionRequest.class, + Workflow.class }) void testClassConverterIsSet(Class clazz) { assertFalse(ClassConverterFactory.getConverter(clazz) instanceof DefaultConnectionClassConverter); @@ -41,6 +45,6 @@ public class ClassConverterFactoryTest { @Test void testClassConvertedMapIsNotModified() { - assertEquals(ClassConverterFactory.getConverterMap().size(), 11); + assertEquals(ClassConverterFactory.getConverterMap().size(), 13); } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/TestEntityMasker.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/TestEntityMasker.java index b1f43cf2ebc..2cf0e62cd8c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/TestEntityMasker.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/TestEntityMasker.java @@ -6,8 +6,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; +import org.openmetadata.schema.api.services.DatabaseConnection; import org.openmetadata.schema.auth.JWTAuthMechanism; import org.openmetadata.schema.auth.SSOAuthMechanism; +import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest; +import org.openmetadata.schema.entity.automations.Workflow; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; @@ -172,6 +175,46 @@ abstract class TestEntityMasker { assertEquals(((MysqlConnection) unmasked.getConnection()).getPassword(), PASSWORD); } + @Test + void testWorkflowMasker() { + Workflow workflow = + new Workflow() + .withRequest( + new TestServiceConnectionRequest() + .withConnection(new DatabaseConnection().withConfig(buildMysqlConnection())) + .withServiceType(ServiceType.DATABASE) + .withConnectionType("Mysql")) + .withOpenMetadataServerConnection(buildOpenMetadataConnection()); + Workflow masked = EntityMaskerFactory.createEntityMasker(config).maskWorkflow(workflow); + assertNotNull(masked); + assertEquals( + ((MysqlConnection) + ((DatabaseConnection) ((TestServiceConnectionRequest) masked.getRequest()).getConnection()).getConfig()) + .getPassword(), + getMaskedPassword()); + assertEquals( + ((AWSCredentials) masked.getOpenMetadataServerConnection().getSecretsManagerCredentials()) + .getAwsSecretAccessKey(), + getMaskedPassword()); + assertEquals( + ((GoogleSSOClientConfig) masked.getOpenMetadataServerConnection().getSecurityConfig()).getSecretKey(), + getMaskedPassword()); + Workflow unmasked = EntityMaskerFactory.createEntityMasker(config).unmaskWorkflow(masked, workflow); + assertEquals( + ((MysqlConnection) + ((DatabaseConnection) ((TestServiceConnectionRequest) unmasked.getRequest()).getConnection()) + .getConfig()) + .getPassword(), + PASSWORD); + assertEquals( + ((AWSCredentials) unmasked.getOpenMetadataServerConnection().getSecretsManagerCredentials()) + .getAwsSecretAccessKey(), + PASSWORD); + assertEquals( + ((GoogleSSOClientConfig) unmasked.getOpenMetadataServerConnection().getSecurityConfig()).getSecretKey(), + PASSWORD); + } + @Test void testObjectMaskerWithoutACustomClassConverter() { MysqlConnection mysqlConnection = buildMysqlConnection(); @@ -216,10 +259,13 @@ abstract class TestEntityMasker { .withConfig( new DbtPipeline() .withDbtConfigSource(new DbtGCSConfig().withDbtSecurityConfig(buildGcsCredentials())))) - .withOpenMetadataServerConnection( - new OpenMetadataConnection() - .withSecretsManagerCredentials(new AWSCredentials().withAwsSecretAccessKey(PASSWORD)) - .withSecurityConfig(buildGoogleSSOClientConfig())); + .withOpenMetadataServerConnection(buildOpenMetadataConnection()); + } + + private OpenMetadataConnection buildOpenMetadataConnection() { + return new OpenMetadataConnection() + .withSecretsManagerCredentials(new AWSCredentials().withAwsSecretAccessKey(PASSWORD)) + .withSecurityConfig(buildGoogleSSOClientConfig()); } private GoogleSSOClientConfig buildGoogleSSOClientConfig() { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json b/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json index b590ff24017..4c8bebc3726 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json @@ -103,5 +103,5 @@ } }, "additionalProperties": false, - "required": ["id", "name", "workflowType", "request", "openMetadataServerConnection"] + "required": ["id", "name", "workflowType", "request"] } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json index 233ca8288a8..d76c8026377 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json @@ -208,7 +208,6 @@ "name", "pipelineType", "sourceConfig", - "openMetadataServerConnection", "airflowConfig" ], "additionalProperties": false