yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-01 19:18:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Don't store the OM connection in the Ingestion Pipeline or Workflow (#10448)

Browse Source 
* Do not store OM connection

* Migration to remove the server connection

* Update tests

* Add workflow masking and secrets manager

* Fix failing test

---------

Co-authored-by: Nahuel Verdugo Revigliono <nahuel@getcollate.io>
This commit is contained in:
Pere Miquel Brull 2023-03-09 17:32:40 +01:00 committed by GitHub
parent d9e4fbdebb
commit 81dec813a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 510 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql
View File

@ -73,3 +73,7 @@ CREATE TABLE IF NOT EXISTS automations_workflow (
PRIMARY KEY (id),
UNIQUE (name)
);
-- Do not store OM server connection, we'll set it dynamically on the resource
UPDATE ingestion_pipeline_entity
SET json = JSON_REMOVE(json, '$.openMetadataServerConnection');

4
bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql
View File

@ -73,3 +73,7 @@ CREATE TABLE IF NOT EXISTS automations_workflow (
PRIMARY KEY (id),
UNIQUE (name)
);
-- Do not store OM server connection, we'll set it dynamically on the resource
UPDATE ingestion_pipeline_entity
SET json = json::jsonb #- '{openMetadataServerConnection}';

3
ingestion/tests/integration/ometa/test_ometa_workflow_api.py
View File

@ -28,6 +28,7 @@ from metadata.generated.schema.entity.automations.workflow import (
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
MySQLType,
)
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
@ -72,6 +73,7 @@ class OMetaWorkflowTest(TestCase):
fullyQualifiedName="test",
request=TestServiceConnectionRequest(
serviceType=ServiceType.Database,
connectionType=MySQLType.Mysql.value,
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
@ -91,6 +93,7 @@ class OMetaWorkflowTest(TestCase):
workflowType=WorkflowType.TEST_CONNECTION,
request=TestServiceConnectionRequest(
serviceType=ServiceType.Database,
connectionType=MySQLType.Mysql.value,
connection=DatabaseConnection(
config=MysqlConnection(
username="username",

23
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java
View File

@ -85,19 +85,25 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
// Relationships and fields such as href are derived and not stored as part of json
EntityReference owner = ingestionPipeline.getOwner();
EntityReference service = ingestionPipeline.getService();
OpenMetadataConnection openmetadataConnection = ingestionPipeline.getOpenMetadataServerConnection();
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
if (secretsManager != null) {
secretsManager.encryptOrDecryptIngestionPipeline(ingestionPipeline, true);
// We store the OM sensitive values in SM separately
openmetadataConnection =
secretsManager.encryptOrDecryptOpenMetadataConnection(openmetadataConnection, true, true);
}
// Don't store owner. Build it on the fly based on relationships
ingestionPipeline.withOwner(null).withService(null).withHref(null);
// We don't want to store the OM connection.
ingestionPipeline.withOwner(null).withService(null).withHref(null).withOpenMetadataServerConnection(null);
store(ingestionPipeline, update);
// Restore the relationships
ingestionPipeline.withOwner(owner).withService(service);
ingestionPipeline.withOwner(owner).withService(service).withOpenMetadataServerConnection(openmetadataConnection);
}
@Override
@ -230,8 +236,6 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
public void entitySpecificUpdate() throws IOException {
updateSourceConfig();
updateAirflowConfig(original.getAirflowConfig(), updated.getAirflowConfig());
updateOpenMetadataServerConnection(
original.getOpenMetadataServerConnection(), updated.getOpenMetadataServerConnection());
updateLogLevel(original.getLoggerLevel(), updated.getLoggerLevel());
updateEnabled(original.getEnabled(), updated.getEnabled());
updateDeployed(original.getDeployed(), updated.getDeployed());
@ -254,17 +258,6 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
}
}
private void updateOpenMetadataServerConnection(
OpenMetadataConnection origConfig, OpenMetadataConnection updatedConfig) throws JsonProcessingException {
JSONObject origConfigJson = new JSONObject(JsonUtils.pojoToJson(origConfig));
JSONObject updatedConfigJson = new JSONObject(JsonUtils.pojoToJson(updatedConfig));
if (!origConfigJson.similar(updatedConfigJson)) {
recordChange("openMetadataServerConnection", origConfig, updatedConfig);
}
}
private void updateLogLevel(LogLevels origLevel, LogLevels updatedLevel) throws JsonProcessingException {
if (updatedLevel != null && !origLevel.equals(updatedLevel)) {
recordChange("loggerLevel", origLevel, updatedLevel);

21
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowRepository.java
View File

@ -4,10 +4,12 @@ import static org.openmetadata.service.Entity.WORKFLOW;
import java.io.IOException;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.sdk.PipelineServiceClient;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.operations.WorkflowResource;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.util.EntityUtil;
public class WorkflowRepository extends EntityRepository<Workflow> {
@ -15,8 +17,6 @@ public class WorkflowRepository extends EntityRepository<Workflow> {
private static final String UPDATE_FIELDS = "owner";
private static final String PATCH_FIELDS = "owner";
private static PipelineServiceClient pipelineServiceClient;
public WorkflowRepository(CollectionDAO dao) {
super(
WorkflowResource.COLLECTION_PATH,
@ -28,10 +28,6 @@ public class WorkflowRepository extends EntityRepository<Workflow> {
UPDATE_FIELDS);
}
public void setPipelineServiceClient(PipelineServiceClient client) {
pipelineServiceClient = client;
}
@Override
public Workflow setFields(Workflow entity, EntityUtil.Fields fields) throws IOException {
return entity.withOwner(fields.contains(Entity.FIELD_OWNER) ? getOwner(entity) : null);
@ -48,12 +44,19 @@ public class WorkflowRepository extends EntityRepository<Workflow> {
@Override
public void storeEntity(Workflow entity, boolean update) throws IOException {
EntityReference owner = entity.getOwner();
OpenMetadataConnection openmetadataConnection = entity.getOpenMetadataServerConnection();
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
if (secretsManager != null) {
entity = secretsManager.encryptOrDecryptWorkflow(entity, true);
}
// Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
entity.withOwner(null).withHref(null);
entity.withOwner(null).withHref(null).withOpenMetadataServerConnection(null);
store(entity, update);
// Restore the relationships
entity.withOwner(owner);
entity.withOwner(owner).withOpenMetadataServerConnection(openmetadataConnection);
}
@Override

92
openmetadata-service/src/main/java/org/openmetadata/service/resources/operations/WorkflowResource.java
View File

@ -1,5 +1,6 @@
package org.openmetadata.service.resources.operations;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import com.google.inject.Inject;
@ -14,6 +15,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
@ -35,14 +37,17 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.ServiceConnectionEntityInterface;
import org.openmetadata.schema.api.data.RestoreEntity;
import org.openmetadata.schema.entity.automations.CreateWorkflow;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.automations.WorkflowStatus;
import org.openmetadata.schema.entity.automations.WorkflowType;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.sdk.PipelineServiceClient;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
@ -52,7 +57,13 @@ import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.WorkflowRepository;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.secrets.converter.ClassConverterFactory;
import org.openmetadata.service.secrets.masker.EntityMaskerFactory;
import org.openmetadata.service.security.AuthorizationException;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.OpenMetadataConnectionBuilder;
import org.openmetadata.service.util.RestUtil;
@ -90,7 +101,6 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
this.pipelineServiceClient =
PipelineServiceClientFactory.createPipelineServiceClient(config.getPipelineServiceClientConfiguration());
dao.setPipelineServiceClient(pipelineServiceClient);
}
public static class WorkflowList extends ResultList<Workflow> {
@ -162,7 +172,13 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
if (status != null) {
filter.addQueryParam("status", status);
}
return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after);
ResultList<Workflow> workflows =
super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after);
workflows.setData(
listOrEmpty(workflows.getData()).stream()
.map(service -> decryptOrNullify(securityContext, service))
.collect(Collectors.toList()));
return workflows;
}
@GET
@ -215,7 +231,7 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getInternal(uriInfo, securityContext, id, fieldsParam, include);
return decryptOrNullify(securityContext, getInternal(uriInfo, securityContext, id, fieldsParam, include));
}
@GET
@ -249,7 +265,7 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include);
return decryptOrNullify(securityContext, getByNameInternal(uriInfo, securityContext, name, fieldsParam, include));
}
@GET
@ -278,7 +294,7 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
@PathParam("version")
String version)
throws IOException {
return super.getVersionInternal(securityContext, id, version);
return decryptOrNullify(securityContext, super.getVersionInternal(securityContext, id, version));
}
@POST
@ -298,8 +314,10 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateWorkflow create)
throws IOException {
Workflow workflow = getWorkflow(create, securityContext.getUserPrincipal().getName());
// TODO: Create Trigger workflow endpoint using the pipelineServiceClient
return create(uriInfo, securityContext, workflow);
Response response = create(uriInfo, securityContext, workflow);
return Response.fromResponse(response)
.entity(decryptOrNullify(securityContext, (Workflow) response.getEntity()))
.build();
}
@POST
@ -323,6 +341,7 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
throws IOException {
EntityUtil.Fields fields = getFields(FIELD_OWNER);
Workflow workflow = dao.get(uriInfo, id, fields);
workflow.setOpenMetadataServerConnection(new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build());
return pipelineServiceClient.runAutomationsWorkflow(workflow);
}
@ -349,7 +368,10 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
}))
JsonPatch patch)
throws IOException {
return patchInternal(uriInfo, securityContext, id, patch);
Response response = patchInternal(uriInfo, securityContext, id, patch);
return Response.fromResponse(response)
.entity(decryptOrNullify(securityContext, (Workflow) response.getEntity()))
.build();
}
@PUT
@ -368,7 +390,11 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateWorkflow create)
throws IOException {
Workflow workflow = getWorkflow(create, securityContext.getUserPrincipal().getName());
return createOrUpdate(uriInfo, securityContext, workflow);
workflow = unmask(workflow);
Response response = createOrUpdate(uriInfo, securityContext, workflow);
return Response.fromResponse(response)
.entity(decryptOrNullify(securityContext, (Workflow) response.getEntity()))
.build();
}
@DELETE
@ -391,7 +417,10 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
boolean hardDelete,
@Parameter(description = "Id of the Workflow", schema = @Schema(type = "UUID")) @PathParam("id") UUID id)
throws IOException {
return delete(uriInfo, securityContext, id, false, hardDelete);
Response response = delete(uriInfo, securityContext, id, false, hardDelete);
return Response.fromResponse(response)
.entity(decryptOrNullify(securityContext, (Workflow) response.getEntity()))
.build();
}
@DELETE
@ -415,7 +444,10 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
@Parameter(description = "Name of the Workflow", schema = @Schema(type = "string")) @PathParam("name")
String name)
throws IOException {
return deleteByName(uriInfo, securityContext, name, false, hardDelete);
Response response = deleteByName(uriInfo, securityContext, name, false, hardDelete);
return Response.fromResponse(response)
.entity(decryptOrNullify(securityContext, (Workflow) response.getEntity()))
.build();
}
@PUT
@ -434,7 +466,10 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
public Response restoreWorkflow(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid RestoreEntity restore)
throws IOException {
return restoreEntity(uriInfo, securityContext, restore.getId());
Response response = restoreEntity(uriInfo, securityContext, restore.getId());
return Response.fromResponse(response)
.entity(decryptOrNullify(securityContext, (Workflow) response.getEntity()))
.build();
}
private Workflow getWorkflow(CreateWorkflow create, String user) throws IOException {
@ -448,4 +483,37 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
.withOpenMetadataServerConnection(openMetadataServerConnection)
.withName(create.getName());
}
private Workflow unmask(Workflow workflow) {
dao.setFullyQualifiedName(workflow);
Workflow originalWorkflow = dao.findByNameOrNull(workflow.getFullyQualifiedName(), null, Include.NON_DELETED);
return EntityMaskerFactory.getEntityMasker().unmaskWorkflow(workflow, originalWorkflow);
}
private Workflow decryptOrNullify(SecurityContext securityContext, Workflow workflow) {
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
try {
authorizer.authorize(
securityContext,
new OperationContext(entityType, MetadataOperation.VIEW_ALL),
getResourceContextById(workflow.getId()));
} catch (AuthorizationException | IOException e) {
Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow);
if (workflowConverted.getRequest() instanceof TestServiceConnectionRequest) {
((ServiceConnectionEntityInterface)
((TestServiceConnectionRequest) workflowConverted.getRequest()).getConnection())
.setConfig(null);
}
return workflowConverted;
}
Workflow workflowDecrypted = secretsManager.encryptOrDecryptWorkflow(workflow, false);
OpenMetadataConnection openMetadataServerConnection =
new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build();
workflowDecrypted.setOpenMetadataServerConnection(
secretsManager.encryptOrDecryptOpenMetadataConnection(openMetadataServerConnection, true, false));
if (authorizer.shouldMaskPasswords(securityContext)) {
workflowDecrypted = EntityMaskerFactory.getEntityMasker().maskWorkflow(workflowDecrypted);
}
return workflowDecrypted;
}
}

8
openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java
View File

@ -273,7 +273,9 @@ public class IngestionPipelineResource extends EntityResource<IngestionPipeline,
@PathParam("version")
String version)
throws IOException {
return super.getVersionInternal(securityContext, id, version);
IngestionPipeline ingestionPipeline = super.getVersionInternal(securityContext, id, version);
decryptOrNullify(securityContext, ingestionPipeline, false);
return ingestionPipeline;
}
@GET
@ -821,6 +823,10 @@ public class IngestionPipelineResource extends EntityResource<IngestionPipeline,
ingestionPipeline.getSourceConfig().setConfig(null);
}
secretsManager.encryptOrDecryptIngestionPipeline(ingestionPipeline, false);
OpenMetadataConnection openMetadataServerConnection =
new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build();
ingestionPipeline.setOpenMetadataServerConnection(
secretsManager.encryptOrDecryptOpenMetadataConnection(openMetadataServerConnection, true, false));
if (authorizer.shouldMaskPasswords(securityContext) && !forceNotMask) {
EntityMaskerFactory.getEntityMasker().maskIngestionPipeline(ingestionPipeline);
}

6
openmetadata-service/src/main/java/org/openmetadata/service/secrets/ExternalSecretsManager.java
View File

@ -29,11 +29,13 @@ public abstract class ExternalSecretsManager extends SecretsManager {
}
@Override
protected String storeValue(String fieldName, String value, String secretId) {
protected String storeValue(String fieldName, String value, String secretId, boolean store) {
String fieldSecretId = buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT));
// check if value does not start with 'config:' only String can have password annotation
if (!value.startsWith(SECRET_FIELD_PREFIX)) {
upsertSecret(fieldSecretId, value);
if (store) {
upsertSecret(fieldSecretId, value);
}
return SECRET_FIELD_PREFIX + fieldSecretId;
} else {
return value;

2
openmetadata-service/src/main/java/org/openmetadata/service/secrets/NoopSecretsManager.java
View File

@ -29,7 +29,7 @@ public class NoopSecretsManager extends SecretsManager {
}
@Override
protected String storeValue(String fieldName, String value, String secretId) {
protected String storeValue(String fieldName, String value, String secretId, boolean store) {
return value;
}
}

74
openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java
View File

@ -20,14 +20,18 @@ import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Locale;
import java.util.Set;
import javax.ws.rs.core.Response;
import lombok.Getter;
import org.openmetadata.annotations.PasswordField;
import org.openmetadata.schema.auth.BasicAuthMechanism;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.security.client.OpenMetadataJWTClientConfig;
import org.openmetadata.schema.security.secrets.SecretsManagerProvider;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.service.exception.CustomExceptionMessage;
import org.openmetadata.service.exception.InvalidServiceConnectionException;
import org.openmetadata.service.exception.SecretsManagerException;
import org.openmetadata.service.fernet.Fernet;
@ -56,7 +60,7 @@ public abstract class SecretsManager {
Class<?> clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType);
Object newConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig);
return encryptOrDecryptPasswordFields(
newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt);
newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt, true);
} catch (Exception e) {
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to encrypt connection instance of %s", connectionType));
@ -68,36 +72,76 @@ public abstract class SecretsManager {
if (authenticationMechanism != null) {
AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism);
try {
encryptOrDecryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), encrypt);
encryptOrDecryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), encrypt, true);
} catch (Exception e) {
throw InvalidServiceConnectionException.byMessage(
name, String.format("Failed to encrypt user bot instance [%s]", name));
throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, String.format("Failed to encrypt user bot instance [%s]", name));
}
}
}
public void encryptOrDecryptIngestionPipeline(IngestionPipeline ingestionPipeline, boolean encrypt) {
OpenMetadataConnection openMetadataConnection =
encryptOrDecryptOpenMetadataConnection(ingestionPipeline.getOpenMetadataServerConnection(), encrypt, true);
ingestionPipeline.setOpenMetadataServerConnection(null);
// we don't store OM conn sensitive data
IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline);
try {
encryptOrDecryptPasswordFields(
ingestionPipeline, buildSecretId(true, "pipeline", ingestionPipeline.getName()), encrypt);
ingestionPipeline, buildSecretId(true, "pipeline", ingestionPipeline.getName()), encrypt, true);
} catch (Exception e) {
throw InvalidServiceConnectionException.byMessage(
ingestionPipeline.getName(),
throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST,
String.format("Failed to encrypt ingestion pipeline instance [%s]", ingestionPipeline.getName()));
}
ingestionPipeline.setOpenMetadataServerConnection(openMetadataConnection);
}
private Object encryptOrDecryptPasswordFields(Object targetObject, String name, boolean encrypt) {
public Workflow encryptOrDecryptWorkflow(Workflow workflow, boolean encrypt) {
OpenMetadataConnection openMetadataConnection =
encryptOrDecryptOpenMetadataConnection(workflow.getOpenMetadataServerConnection(), encrypt, true);
Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow);
// we don't store OM conn sensitive data
workflowConverted.setOpenMetadataServerConnection(null);
try {
encryptOrDecryptPasswordFields(
workflowConverted, buildSecretId(true, "workflow", workflow.getName()), encrypt, true);
} catch (Exception e) {
throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, String.format("Failed to encrypt workflow instance [%s]", workflow.getName()));
}
workflowConverted.setOpenMetadataServerConnection(openMetadataConnection);
return workflowConverted;
}
public OpenMetadataConnection encryptOrDecryptOpenMetadataConnection(
OpenMetadataConnection openMetadataConnection, boolean encrypt, boolean store) {
if (openMetadataConnection != null) {
OpenMetadataConnection openMetadataConnectionConverted =
(OpenMetadataConnection)
ClassConverterFactory.getConverter(OpenMetadataConnection.class).convert(openMetadataConnection);
try {
encryptOrDecryptPasswordFields(
openMetadataConnectionConverted, buildSecretId(true, "serverconnection"), encrypt, store);
} catch (Exception e) {
throw new CustomExceptionMessage(
Response.Status.BAD_REQUEST, "Failed to encrypt OpenMetadataConnection instance.");
}
return openMetadataConnectionConverted;
}
return null;
}
private Object encryptOrDecryptPasswordFields(Object targetObject, String name, boolean encrypt, boolean store) {
if (encrypt) {
encryptPasswordFields(targetObject, name);
encryptPasswordFields(targetObject, name, store);
} else {
decryptPasswordFields(targetObject);
}
return targetObject;
}
private void encryptPasswordFields(Object toEncryptObject, String secretId) {
private void encryptPasswordFields(Object toEncryptObject, String secretId, boolean store) {
if (!DO_NOT_ENCRYPT_CLASSES.contains(toEncryptObject.getClass())) {
// for each get method
Arrays.stream(toEncryptObject.getClass().getMethods())
@ -109,17 +153,19 @@ public abstract class SecretsManager {
// if the object matches the package of openmetadata
if (obj != null && obj.getClass().getPackageName().startsWith("org.openmetadata")) {
// encryptPasswordFields
encryptPasswordFields(obj, buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT)));
encryptPasswordFields(obj, buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT)), store);
// check if it has annotation
} else if (obj != null && method.getAnnotation(PasswordField.class) != null) {
// store value if proceed
String newFieldValue = storeValue(fieldName, fernet.decryptIfApplies((String) obj), secretId);
String newFieldValue = storeValue(fieldName, fernet.decryptIfApplies((String) obj), secretId, store);
// get setMethod
Method toSet = ReflectionUtil.getToSetMethod(toEncryptObject, obj, fieldName);
// set new value
ReflectionUtil.setValueInMethod(
toEncryptObject,
Fernet.isTokenized(newFieldValue) ? newFieldValue : fernet.encrypt(newFieldValue),
Fernet.isTokenized(newFieldValue)
? newFieldValue
: store ? fernet.encrypt(newFieldValue) : newFieldValue,
toSet);
}
});
@ -150,7 +196,7 @@ public abstract class SecretsManager {
});
}
protected abstract String storeValue(String fieldName, String value, String secretId);
protected abstract String storeValue(String fieldName, String value, String secretId, boolean store);
protected String getSecretSeparator() {
return "/";

4
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java
View File

@ -18,6 +18,8 @@ import java.util.HashMap;
import java.util.Map;
import lombok.Getter;
import org.openmetadata.schema.auth.SSOAuthMechanism;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.metadataIngestion.DbtPipeline;
import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtGCSConfig;
import org.openmetadata.schema.security.credentials.GCSCredentials;
@ -50,6 +52,8 @@ public class ClassConverterFactory {
put(GCSConfig.class, new GCSConfigClassConverter());
put(BigQueryConnection.class, new BigQueryConnectionClassConverter());
put(DbtGCSConfig.class, new DbtGCSConfigClassConverter());
put(TestServiceConnectionRequest.class, new TestServiceConnectionRequestClassConverter());
put(Workflow.class, new WorkflowClassConverter());
}
});
}

70
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestServiceConnectionRequestClassConverter.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.secrets.converter;
import java.util.List;
import org.openmetadata.schema.ServiceConnectionEntityInterface;
import org.openmetadata.schema.api.services.DatabaseConnection;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
import org.openmetadata.schema.entity.services.MetadataConnection;
import org.openmetadata.schema.type.DashboardConnection;
import org.openmetadata.schema.type.MessagingConnection;
import org.openmetadata.schema.type.MlModelConnection;
import org.openmetadata.schema.type.PipelineConnection;
import org.openmetadata.service.exception.InvalidServiceConnectionException;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ReflectionUtil;
/** Converter class to get an `TestServiceConnectionRequest` object. */
public class TestServiceConnectionRequestClassConverter extends ClassConverter {
private static final List<Class<?>> CONNECTION_CLASSES =
List.of(
DatabaseConnection.class,
DashboardConnection.class,
MessagingConnection.class,
PipelineConnection.class,
MlModelConnection.class,
MetadataConnection.class);
public TestServiceConnectionRequestClassConverter() {
super(TestServiceConnectionRequest.class);
}
@Override
public Object convert(Object object) {
TestServiceConnectionRequest testServiceConnectionRequest =
(TestServiceConnectionRequest) JsonUtils.convertValue(object, this.clazz);
try {
Class<?> clazz =
ReflectionUtil.createConnectionConfigClass(
testServiceConnectionRequest.getConnectionType(), testServiceConnectionRequest.getServiceType());
tryToConvertOrFail(testServiceConnectionRequest.getConnection(), CONNECTION_CLASSES)
.ifPresent(testServiceConnectionRequest::setConnection);
Object newConnectionConfig =
ClassConverterFactory.getConverter(clazz)
.convert(((ServiceConnectionEntityInterface) testServiceConnectionRequest.getConnection()).getConfig());
((ServiceConnectionEntityInterface) testServiceConnectionRequest.getConnection()).setConfig(newConnectionConfig);
} catch (Exception e) {
throw InvalidServiceConnectionException.byMessage(
testServiceConnectionRequest.getConnectionType(),
String.format("Failed to convert class instance of %s", testServiceConnectionRequest.getConnectionType()));
}
return testServiceConnectionRequest;
}
}

45
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/WorkflowClassConverter.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.secrets.converter;
import java.util.List;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.service.util.JsonUtils;
/** Converter class to get an `Workflow` object. */
public class WorkflowClassConverter extends ClassConverter {
public WorkflowClassConverter() {
super(Workflow.class);
}
@Override
public Object convert(Object object) {
Workflow workflow = (Workflow) JsonUtils.convertValue(object, this.clazz);
tryToConvertOrFail(workflow.getRequest(), List.of(TestServiceConnectionRequest.class))
.ifPresent(workflow::setRequest);
if (workflow.getOpenMetadataServerConnection() != null) {
workflow.setOpenMetadataServerConnection(
(OpenMetadataConnection)
ClassConverterFactory.getConverter(OpenMetadataConnection.class)
.convert(workflow.getOpenMetadataServerConnection()));
}
return workflow;
}
}

5
openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/EntityMasker.java
View File

@ -15,6 +15,7 @@ package org.openmetadata.service.secrets.masker;
import java.util.Set;
import org.openmetadata.schema.auth.BasicAuthMechanism;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
@ -32,6 +33,8 @@ public abstract class EntityMasker {
public abstract void maskIngestionPipeline(IngestionPipeline ingestionPipeline);
public abstract Workflow maskWorkflow(Workflow workflow);
public abstract Object unmaskServiceConnectionConfig(
Object connectionConfig, Object originalConnectionConfig, String connectionType, ServiceType serviceType);
@ -42,4 +45,6 @@ public abstract class EntityMasker {
String name,
AuthenticationMechanism authenticationMechanism,
AuthenticationMechanism originalAuthenticationMechanism);
public abstract Workflow unmaskWorkflow(Workflow workflow, Workflow originalWorkflow);
}

11
openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/NoopEntityMasker.java
View File

@ -13,6 +13,7 @@
package org.openmetadata.service.secrets.masker;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
@ -45,6 +46,11 @@ public class NoopEntityMasker extends EntityMasker {
// do nothing
}
@Override
public Workflow maskWorkflow(Workflow workflow) {
return workflow;
}
@Override
public Object unmaskServiceConnectionConfig(
Object connectionConfig, Object originalConnectionConfig, String connectionType, ServiceType serviceType) {
@ -64,4 +70,9 @@ public class NoopEntityMasker extends EntityMasker {
AuthenticationMechanism originalAuthenticationMechanism) {
// do nothing
}
@Override
public Workflow unmaskWorkflow(Workflow workflow, Workflow originalWorkflow) {
return workflow;
}
}

90
openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/PasswordEntityMasker.java
View File

@ -18,6 +18,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import org.openmetadata.annotations.PasswordField;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
@ -46,14 +47,17 @@ public class PasswordEntityMasker extends EntityMasker {
}
public Object maskServiceConnectionConfig(Object connectionConfig, String connectionType, ServiceType serviceType) {
try {
Class<?> clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType);
Object newConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig);
maskPasswordFields(newConnectionConfig);
return newConnectionConfig;
} catch (Exception e) {
throw new EntityMaskException(String.format("Failed to mask connection instance of %s", connectionType));
if (connectionConfig != null) {
try {
Class<?> clazz = ReflectionUtil.createConnectionConfigClass(connectionType, serviceType);
Object convertedConnectionConfig = ClassConverterFactory.getConverter(clazz).convert(connectionConfig);
maskPasswordFields(convertedConnectionConfig);
return convertedConnectionConfig;
} catch (Exception e) {
throw new EntityMaskException(String.format("Failed to mask connection instance of %s", connectionType));
}
}
return null;
}
public void maskAuthenticationMechanism(String name, AuthenticationMechanism authenticationMechanism) {
@ -68,15 +72,31 @@ public class PasswordEntityMasker extends EntityMasker {
}
public void maskIngestionPipeline(IngestionPipeline ingestionPipeline) {
IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline);
try {
maskPasswordFields(ingestionPipeline);
} catch (Exception e) {
throw new EntityMaskException(
String.format("Failed to mask ingestion pipeline instance [%s]", ingestionPipeline.getName()));
if (ingestionPipeline != null) {
IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline);
try {
maskPasswordFields(ingestionPipeline);
} catch (Exception e) {
throw new EntityMaskException(
String.format("Failed to mask ingestion pipeline instance [%s]", ingestionPipeline.getName()));
}
}
}
@Override
public Workflow maskWorkflow(Workflow workflow) {
if (workflow != null) {
Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow);
try {
maskPasswordFields(workflowConverted);
} catch (Exception e) {
throw new EntityMaskException(String.format("Failed to mask workflow instance [%s]", workflow.getName()));
}
return workflowConverted;
}
return null;
}
public Object unmaskServiceConnectionConfig(
Object connectionConfig, Object originalConnectionConfig, String connectionType, ServiceType serviceType) {
if (originalConnectionConfig != null && connectionConfig != null) {
@ -89,7 +109,7 @@ public class PasswordEntityMasker extends EntityMasker {
unmaskPasswordFields(toUnmaskConfig, NEW_KEY, passwordsMap);
return toUnmaskConfig;
} catch (Exception e) {
throw new EntityMaskException(String.format("Failed to mask connection instance of %s", connectionType));
throw new EntityMaskException(String.format("Failed to unmask connection instance of %s", connectionType));
}
}
return connectionConfig;
@ -97,15 +117,17 @@ public class PasswordEntityMasker extends EntityMasker {
public void unmaskIngestionPipeline(
IngestionPipeline ingestionPipeline, IngestionPipeline originalIngestionPipeline) {
IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline);
IngestionPipelineBuilder.addDefinedConfig(originalIngestionPipeline);
try {
Map<String, String> passwordsMap = new HashMap<>();
buildPasswordsMap(originalIngestionPipeline, NEW_KEY, passwordsMap);
unmaskPasswordFields(ingestionPipeline, NEW_KEY, passwordsMap);
} catch (Exception e) {
throw new EntityMaskException(
String.format("Failed to mask ingestion pipeline instance [%s]", ingestionPipeline.getName()));
if (ingestionPipeline != null && originalIngestionPipeline != null) {
IngestionPipelineBuilder.addDefinedConfig(ingestionPipeline);
IngestionPipelineBuilder.addDefinedConfig(originalIngestionPipeline);
try {
Map<String, String> passwordsMap = new HashMap<>();
buildPasswordsMap(originalIngestionPipeline, NEW_KEY, passwordsMap);
unmaskPasswordFields(ingestionPipeline, NEW_KEY, passwordsMap);
} catch (Exception e) {
throw new EntityMaskException(
String.format("Failed to unmask ingestion pipeline instance [%s]", ingestionPipeline.getName()));
}
}
}
@ -113,7 +135,7 @@ public class PasswordEntityMasker extends EntityMasker {
String name,
AuthenticationMechanism authenticationMechanism,
AuthenticationMechanism originalAuthenticationMechanism) {
if (authenticationMechanism != null) {
if (authenticationMechanism != null && originalAuthenticationMechanism != null) {
AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism);
AuthenticationMechanismBuilder.addDefinedConfig(originalAuthenticationMechanism);
try {
@ -121,11 +143,29 @@ public class PasswordEntityMasker extends EntityMasker {
buildPasswordsMap(originalAuthenticationMechanism, NEW_KEY, passwordsMap);
unmaskPasswordFields(authenticationMechanism, NEW_KEY, passwordsMap);
} catch (Exception e) {
throw new EntityMaskException(String.format("Failed to mask user bot instance [%s]", name));
throw new EntityMaskException(String.format("Failed to unmask auth mechanism instance [%s]", name));
}
}
}
@Override
public Workflow unmaskWorkflow(Workflow workflow, Workflow originalWorkflow) {
if (workflow != null && originalWorkflow != null) {
Workflow workflowConverted = (Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(workflow);
Workflow origWorkflowConverted =
(Workflow) ClassConverterFactory.getConverter(Workflow.class).convert(originalWorkflow);
try {
Map<String, String> passwordsMap = new HashMap<>();
buildPasswordsMap(origWorkflowConverted, NEW_KEY, passwordsMap);
unmaskPasswordFields(workflowConverted, NEW_KEY, passwordsMap);
return workflowConverted;
} catch (Exception e) {
throw new EntityMaskException(String.format("Failed to unmask workflow instance [%s]", workflow.getName()));
}
}
return workflow;
}
private void maskPasswordFields(Object toMaskObject) {
if (!DO_NOT_MASK_CLASSES.contains(toMaskObject.getClass())) {
// for each get method

11
openmetadata-service/src/test/java/org/openmetadata/service/resources/operations/WorkflowResourceTest.java
View File

@ -9,6 +9,7 @@ import static org.openmetadata.service.util.TestUtils.assertListNull;
import java.io.IOException;
import java.util.Map;
import org.apache.http.client.HttpResponseException;
import org.openmetadata.schema.api.services.DatabaseConnection;
import org.openmetadata.schema.entity.automations.CreateWorkflow;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
import org.openmetadata.schema.entity.automations.Workflow;
@ -41,10 +42,12 @@ public class WorkflowResourceTest extends EntityResourceTest<Workflow, CreateWor
.withServiceType(ServiceType.DATABASE)
.withConnectionType("Mysql")
.withConnection(
new MysqlConnection()
.withHostPort("mysql:3306")
.withUsername("openmetadata_user")
.withPassword("openmetadata_password")));
new DatabaseConnection()
.withConfig(
new MysqlConnection()
.withHostPort("mysql:3306")
.withUsername("openmetadata_user")
.withPassword("openmetadata_password"))));
}
@Override

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java
View File

@ -137,6 +137,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
IngestionPipeline ingestion, CreateIngestionPipeline createRequest, Map<String, String> authHeaders) {
assertEquals(createRequest.getAirflowConfig().getConcurrency(), ingestion.getAirflowConfig().getConcurrency());
validateSourceConfig(createRequest.getSourceConfig(), ingestion.getSourceConfig(), ingestion);
assertNotNull(ingestion.getOpenMetadataServerConnection());
}
@Override

82
openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java
View File

@ -20,7 +20,10 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import org.openmetadata.schema.api.services.CreateDatabaseService;
import org.openmetadata.schema.api.services.DatabaseConnection;
import org.openmetadata.schema.auth.SSOAuthMechanism;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.services.DatabaseService;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
@ -29,14 +32,17 @@ import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.metadataIngestion.DbtPipeline;
import org.openmetadata.schema.metadataIngestion.SourceConfig;
import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtS3Config;
import org.openmetadata.schema.security.client.GoogleSSOClientConfig;
import org.openmetadata.schema.security.client.OktaSSOClientConfig;
import org.openmetadata.schema.security.credentials.AWSCredentials;
import org.openmetadata.schema.security.secrets.Parameters;
import org.openmetadata.schema.security.secrets.SecretsManagerConfiguration;
import org.openmetadata.schema.security.secrets.SecretsManagerProvider;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.service.Entity;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.util.JsonUtils;
@ExtendWith(MockitoExtension.class)
public abstract class ExternalSecretsManagerTest {
@ -89,6 +95,16 @@ public abstract class ExternalSecretsManagerTest {
testEncryptDecryptDBTConfig(ENCRYPT);
}
@Test
void testDecryptWorkflow() {
testEncryptWorkflowObject(DECRYPT);
}
@Test
void testEncryptWorkflow() {
testEncryptWorkflowObject(ENCRYPT);
}
@Test
void testReturnsExpectedSecretManagerProvider() {
assertEquals(expectedSecretManagerProvider(), secretsManager.getSecretsManagerProvider());
@ -127,12 +143,7 @@ public abstract class ExternalSecretsManagerTest {
new SSOAuthMechanism().withAuthConfig(config).withSsoServiceType(SSOAuthMechanism.SsoServiceType.OKTA));
AuthenticationMechanism actualAuthenticationMechanism =
new AuthenticationMechanism()
.withAuthType(AuthenticationMechanism.AuthType.SSO)
.withConfig(
new SSOAuthMechanism()
.withSsoServiceType(SSOAuthMechanism.SsoServiceType.OKTA)
.withAuthConfig(Map.of("privateKey", "this-is-a-test")));
JsonUtils.convertValue(expectedAuthenticationMechanism, AuthenticationMechanism.class);
secretsManager.encryptOrDecryptAuthenticationMechanism("bot", actualAuthenticationMechanism, decrypt);
@ -165,20 +176,7 @@ public abstract class ExternalSecretsManagerTest {
.withAwsRegion("eu-west-1")))));
IngestionPipeline actualIngestionPipeline =
new IngestionPipeline()
.withName("my-pipeline")
.withPipelineType(PipelineType.DBT)
.withService(new DatabaseService().getEntityReference().withType(Entity.DATABASE_SERVICE))
.withSourceConfig(
new SourceConfig()
.withConfig(
Map.of(
"dbtConfigSource",
Map.of(
"dbtSecurityConfig",
Map.of(
"awsSecretAccessKey", "secret-password",
"awsRegion", "eu-west-1")))));
JsonUtils.convertValue(expectedIngestionPipeline, IngestionPipeline.class);
secretsManager.encryptOrDecryptIngestionPipeline(actualIngestionPipeline, decrypt);
@ -202,5 +200,49 @@ public abstract class ExternalSecretsManagerTest {
assertEquals(expectedIngestionPipeline, actualIngestionPipeline);
}
void testEncryptWorkflowObject(boolean encrypt) {
Workflow expectedWorkflow =
new Workflow()
.withName("my-workflow")
.withOpenMetadataServerConnection(
new OpenMetadataConnection()
.withSecretsManagerCredentials(new AWSCredentials().withAwsSecretAccessKey("aws-secret"))
.withSecurityConfig(new GoogleSSOClientConfig().withSecretKey("google-secret")))
.withRequest(
new TestServiceConnectionRequest()
.withConnection(
new DatabaseConnection().withConfig(new MysqlConnection().withPassword("openmetadata-test")))
.withServiceType(ServiceType.DATABASE)
.withConnectionType("Mysql"));
Workflow workflow = JsonUtils.convertValue(expectedWorkflow, Workflow.class);
Workflow actualWorkflow = secretsManager.encryptOrDecryptWorkflow(workflow, encrypt);
if (encrypt) {
((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) expectedWorkflow.getRequest()).getConnection())
.getConfig())
.setPassword("secret:/openmetadata/workflow/my-workflow/request/connection/config/password");
MysqlConnection mysqlConnection =
(MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) actualWorkflow.getRequest()).getConnection())
.getConfig();
mysqlConnection.setPassword(Fernet.getInstance().decrypt(mysqlConnection.getPassword()));
((GoogleSSOClientConfig) (expectedWorkflow.getOpenMetadataServerConnection()).getSecurityConfig())
.setSecretKey("secret:/openmetadata/serverconnection/securityconfig/secretkey");
GoogleSSOClientConfig googleSSOClientConfig =
((GoogleSSOClientConfig) (actualWorkflow.getOpenMetadataServerConnection()).getSecurityConfig());
googleSSOClientConfig.setSecretKey(Fernet.getInstance().decrypt(googleSSOClientConfig.getSecretKey()));
((AWSCredentials) (expectedWorkflow.getOpenMetadataServerConnection()).getSecretsManagerCredentials())
.setAwsSecretAccessKey("secret:/openmetadata/serverconnection/secretsmanagercredentials/awssecretaccesskey");
AWSCredentials awsCredentials =
((AWSCredentials) (actualWorkflow.getOpenMetadataServerConnection()).getSecretsManagerCredentials());
awsCredentials.setAwsSecretAccessKey(Fernet.getInstance().decrypt(awsCredentials.getAwsSecretAccessKey()));
}
assertEquals(expectedWorkflow, actualWorkflow);
}
protected abstract SecretsManagerProvider expectedSecretManagerProvider();
}

8
openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java
View File

@ -7,6 +7,8 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.openmetadata.schema.auth.SSOAuthMechanism;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.metadataIngestion.DbtPipeline;
import org.openmetadata.schema.metadataIngestion.dbtconfig.DbtGCSConfig;
import org.openmetadata.schema.security.credentials.GCSCredentials;
@ -33,7 +35,9 @@ public class ClassConverterFactoryTest {
GcsConnection.class,
GCSConfig.class,
BigQueryConnection.class,
DbtGCSConfig.class
DbtGCSConfig.class,
TestServiceConnectionRequest.class,
Workflow.class
})
void testClassConverterIsSet(Class<?> clazz) {
assertFalse(ClassConverterFactory.getConverter(clazz) instanceof DefaultConnectionClassConverter);
@ -41,6 +45,6 @@ public class ClassConverterFactoryTest {
@Test
void testClassConvertedMapIsNotModified() {
assertEquals(ClassConverterFactory.getConverterMap().size(), 11);
assertEquals(ClassConverterFactory.getConverterMap().size(), 13);
}
}

54
openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/TestEntityMasker.java
View File

@ -6,8 +6,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.openmetadata.schema.api.services.DatabaseConnection;
import org.openmetadata.schema.auth.JWTAuthMechanism;
import org.openmetadata.schema.auth.SSOAuthMechanism;
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
@ -172,6 +175,46 @@ abstract class TestEntityMasker {
assertEquals(((MysqlConnection) unmasked.getConnection()).getPassword(), PASSWORD);
}
@Test
void testWorkflowMasker() {
Workflow workflow =
new Workflow()
.withRequest(
new TestServiceConnectionRequest()
.withConnection(new DatabaseConnection().withConfig(buildMysqlConnection()))
.withServiceType(ServiceType.DATABASE)
.withConnectionType("Mysql"))
.withOpenMetadataServerConnection(buildOpenMetadataConnection());
Workflow masked = EntityMaskerFactory.createEntityMasker(config).maskWorkflow(workflow);
assertNotNull(masked);
assertEquals(
((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) masked.getRequest()).getConnection()).getConfig())
.getPassword(),
getMaskedPassword());
assertEquals(
((AWSCredentials) masked.getOpenMetadataServerConnection().getSecretsManagerCredentials())
.getAwsSecretAccessKey(),
getMaskedPassword());
assertEquals(
((GoogleSSOClientConfig) masked.getOpenMetadataServerConnection().getSecurityConfig()).getSecretKey(),
getMaskedPassword());
Workflow unmasked = EntityMaskerFactory.createEntityMasker(config).unmaskWorkflow(masked, workflow);
assertEquals(
((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) unmasked.getRequest()).getConnection())
.getConfig())
.getPassword(),
PASSWORD);
assertEquals(
((AWSCredentials) unmasked.getOpenMetadataServerConnection().getSecretsManagerCredentials())
.getAwsSecretAccessKey(),
PASSWORD);
assertEquals(
((GoogleSSOClientConfig) unmasked.getOpenMetadataServerConnection().getSecurityConfig()).getSecretKey(),
PASSWORD);
}
@Test
void testObjectMaskerWithoutACustomClassConverter() {
MysqlConnection mysqlConnection = buildMysqlConnection();
@ -216,10 +259,13 @@ abstract class TestEntityMasker {
.withConfig(
new DbtPipeline()
.withDbtConfigSource(new DbtGCSConfig().withDbtSecurityConfig(buildGcsCredentials()))))
.withOpenMetadataServerConnection(
new OpenMetadataConnection()
.withSecretsManagerCredentials(new AWSCredentials().withAwsSecretAccessKey(PASSWORD))
.withSecurityConfig(buildGoogleSSOClientConfig()));
.withOpenMetadataServerConnection(buildOpenMetadataConnection());
}
private OpenMetadataConnection buildOpenMetadataConnection() {
return new OpenMetadataConnection()
.withSecretsManagerCredentials(new AWSCredentials().withAwsSecretAccessKey(PASSWORD))
.withSecurityConfig(buildGoogleSSOClientConfig());
}
private GoogleSSOClientConfig buildGoogleSSOClientConfig() {

2
openmetadata-spec/src/main/resources/json/schema/entity/automations/workflow.json
View File

@ -103,5 +103,5 @@
}
},
"additionalProperties": false,
"required": ["id", "name", "workflowType", "request", "openMetadataServerConnection"]
"required": ["id", "name", "workflowType", "request"]
}

1
openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json
View File

@ -208,7 +208,6 @@
"name",
"pipelineType",
"sourceConfig",
"openMetadataServerConnection",
"airflowConfig"
],
"additionalProperties": false