yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-23 09:22:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #11673: Service connection details will be viewable based on view permissions and by default masked for users and unmasked for bots (#11738)

Browse Source
This commit is contained in:
Sriharsha Chintalapani 2023-05-23 23:54:14 -07:00 committed by GitHub
parent 15552b50e9
commit 160984baf1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 129 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
conf/openmetadata.yaml
View File

@ -278,10 +278,6 @@ secretsManagerConfiguration:
accessKeyId: ${OM_SM_ACCESS_KEY_ID:-""}
secretAccessKey: ${OM_SM_ACCESS_KEY:-""}
security:
# it will mask all the password fields in the responses sent from the API except for the bots
maskPasswordsAPI: ${MASK_PASSWORDS_API:-false}
health:
delayedShutdownHandlerEnabled: true
shutdownWaitPeriod: 1s

2
openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java
View File

@ -140,7 +140,7 @@ public class OpenMetadataApplication extends Application<OpenMetadataApplication
catalogConfig.getSecretsManagerConfiguration(), catalogConfig.getClusterName());
// init Entity Masker
EntityMaskerFactory.createEntityMasker(catalogConfig.getSecurityConfiguration());
EntityMaskerFactory.createEntityMasker();
// Instantiate JWT Token Generator
JWTTokenGenerator.getInstance().init(catalogConfig.getJwtTokenConfiguration());

4
openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java
View File

@ -32,7 +32,6 @@ import org.openmetadata.schema.api.security.AuthenticationConfiguration;
import org.openmetadata.schema.api.security.AuthorizerConfiguration;
import org.openmetadata.schema.api.security.jwt.JWTTokenConfiguration;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.security.SecurityConfiguration;
import org.openmetadata.schema.security.secrets.SecretsManagerConfiguration;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.service.migration.MigrationConfiguration;
@ -85,9 +84,6 @@ public class OpenMetadataApplicationConfig extends Configuration {
@JsonProperty("secretsManagerConfiguration")
private SecretsManagerConfiguration secretsManagerConfiguration;
@JsonProperty("security")
private SecurityConfiguration securityConfiguration;
@JsonProperty("eventMonitoringConfiguration")
private EventMonitorConfiguration eventMonitorConfiguration;

35
openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ServiceEntityResource.java
View File

@ -23,11 +23,13 @@ import org.openmetadata.schema.ServiceConnectionEntityInterface;
import org.openmetadata.schema.ServiceEntityInterface;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.exception.InvalidServiceConnectionException;
import org.openmetadata.service.exception.UnhandledServerException;
import org.openmetadata.service.jdbi3.ServiceEntityRepository;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.secrets.SecretsUtil;
import org.openmetadata.service.secrets.masker.EntityMaskerFactory;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.JsonUtils;
@ -51,9 +53,6 @@ public abstract class ServiceEntityResource<
}
protected T decryptOrNullify(SecurityContext securityContext, T service) {
if (!authorizer.decryptSecret(securityContext)) {
return nullifyRequiredConnectionParameters(service);
}
service
.getConnection()
.setConfig(retrieveServiceConnectionConfig(service, authorizer.shouldMaskPasswords(securityContext)));
@ -95,17 +94,27 @@ public abstract class ServiceEntityResource<
serviceEntityRepository.setFullyQualifiedName(service);
T originalService =
serviceEntityRepository.findByNameOrNull(service.getFullyQualifiedName(), null, Include.NON_DELETED);
if (originalService != null && originalService.getConnection() != null) {
Object serviceConnectionConfig =
EntityMaskerFactory.getEntityMasker()
.unmaskServiceConnectionConfig(
service.getConnection().getConfig(),
originalService.getConnection().getConfig(),
extractServiceType(service),
serviceType);
service.getConnection().setConfig(serviceConnectionConfig);
String connectionType = extractServiceType(service);
try {
if (originalService != null && originalService.getConnection() != null) {
Object serviceConnectionConfig =
EntityMaskerFactory.getEntityMasker()
.unmaskServiceConnectionConfig(
service.getConnection().getConfig(),
originalService.getConnection().getConfig(),
connectionType,
serviceType);
service.getConnection().setConfig(serviceConnectionConfig);
}
return service;
} catch (Exception e) {
String message = SecretsUtil.buildExceptionMessageConnectionMask(e.getMessage(), connectionType, false);
if (message != null) {
throw new InvalidServiceConnectionException(message);
}
throw InvalidServiceConnectionException.byMessage(
connectionType, String.format("Failed to unmask connection instance of %s", connectionType));
}
return service;
}
protected abstract T nullifyConnection(T service);

6
openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/EntityMaskerFactory.java
View File

@ -15,7 +15,6 @@ package org.openmetadata.service.secrets.masker;
import com.google.common.annotations.VisibleForTesting;
import lombok.Getter;
import org.openmetadata.schema.security.SecurityConfiguration;
public class EntityMaskerFactory {
@Getter private static EntityMasker entityMasker;
@ -23,12 +22,11 @@ public class EntityMaskerFactory {
private EntityMaskerFactory() {}
/** Expected to be called only once when the Application starts */
public static EntityMasker createEntityMasker(SecurityConfiguration config) {
public static EntityMasker createEntityMasker() {
if (entityMasker != null) {
return entityMasker;
}
entityMasker =
Boolean.TRUE.equals(config.getMaskPasswordsAPI()) ? new PasswordEntityMasker() : new NoopEntityMasker();
entityMasker = new PasswordEntityMasker();
return entityMasker;
}

68
openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/NoopEntityMasker.java
View File

@ -1,68 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.secrets.masker;
import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
public class NoopEntityMasker extends EntityMasker {
protected NoopEntityMasker() {}
@Override
public Object maskServiceConnectionConfig(Object connectionConfig, String connectionType, ServiceType serviceType) {
return connectionConfig;
}
@Override
public void maskAuthenticationMechanism(String name, AuthenticationMechanism authenticationMechanism) {
// do nothing
}
@Override
public void maskIngestionPipeline(IngestionPipeline ingestionPipeline) {
// do nothing
}
@Override
public Workflow maskWorkflow(Workflow workflow) {
return workflow;
}
@Override
public Object unmaskServiceConnectionConfig(
Object connectionConfig, Object originalConnectionConfig, String connectionType, ServiceType serviceType) {
return connectionConfig;
}
@Override
public void unmaskIngestionPipeline(
IngestionPipeline ingestionPipeline, IngestionPipeline originalIngestionPipeline) {
// do nothing
}
@Override
public void unmaskAuthenticationMechanism(
String name,
AuthenticationMechanism authenticationMechanism,
AuthenticationMechanism originalAuthenticationMechanism) {
// do nothing
}
@Override
public Workflow unmaskWorkflow(Workflow workflow, Workflow originalWorkflow) {
return workflow;
}
}

2
openmetadata-service/src/main/java/org/openmetadata/service/secrets/masker/PasswordEntityMasker.java
View File

@ -31,7 +31,7 @@ import org.openmetadata.service.util.IngestionPipelineBuilder;
import org.openmetadata.service.util.ReflectionUtil;
public class PasswordEntityMasker extends EntityMasker {
protected static final String PASSWORD_MASK = "*********";
public static final String PASSWORD_MASK = "*********";
private static final String NEW_KEY = "";
protected PasswordEntityMasker() {}

45
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DashboardServiceResourceTest.java
View File

@ -30,7 +30,6 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
import javax.ws.rs.client.WebTarget;
@ -50,10 +49,10 @@ import org.openmetadata.schema.services.connections.dashboard.MetabaseConnection
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.DashboardConnection;
import org.openmetadata.service.Entity;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.charts.ChartResourceTest;
import org.openmetadata.service.resources.services.dashboard.DashboardServiceResource.DashboardServiceList;
import org.openmetadata.service.secrets.masker.PasswordEntityMasker;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.TestUtils;
import org.openmetadata.service.util.TestUtils.UpdateType;
@ -102,14 +101,14 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
@Test
void put_updateService_as_admin_2xx(TestInfo test) throws IOException, URISyntaxException {
String secretPassword = "secret:/openmetadata/dashboard/" + getEntityName(test) + "/password";
String password = "test12";
DashboardConnection dashboardConnection =
new DashboardConnection()
.withConfig(
new MetabaseConnection()
.withHostPort(new URI("http://localhost:8080"))
.withUsername("user")
.withPassword(secretPassword));
.withPassword(password));
DashboardService service =
createAndCheckEntity(
createRequest(test).withDescription(null).withConnection(dashboardConnection), ADMIN_AUTH_HEADERS);
@ -121,7 +120,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
new MetabaseConnection()
.withHostPort(new URI("http://localhost:9000"))
.withUsername("user1")
.withPassword(secretPassword));
.withPassword(password));
CreateDashboardService update =
createPutRequest(test).withDescription("description1").withConnection(dashboardConnection1);
@ -139,14 +138,14 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
assertNotNull(
JsonUtils.readValue(JsonUtils.pojoToJson(updatedService.getConnection().getConfig()), MetabaseConnection.class)
.getHostPort());
assertNull(
assertNotNull(
JsonUtils.readValue(JsonUtils.pojoToJson(updatedService.getConnection().getConfig()), MetabaseConnection.class)
.getUsername());
MetabaseConnection metabaseConnection =
new MetabaseConnection()
.withHostPort(new URI("http://localhost:8080"))
.withUsername("user")
.withPassword(secretPassword);
.withPassword(password);
DashboardConnection dashboardConnection2 = new DashboardConnection().withConfig(metabaseConnection);
update = createPutRequest(test).withDescription("description1").withConnection(dashboardConnection2);
@ -196,27 +195,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
.withUsername("admin")
.withPassword("admin")));
} catch (URISyntaxException e) {
e.printStackTrace();
}
return null;
}
@Override
public CreateDashboardService createPutRequest(String name) {
String secretPassword = "secret:/openmetadata/dashboard/" + name + "/password";
try {
return new CreateDashboardService()
.withName(name)
.withServiceType(DashboardServiceType.Metabase)
.withConnection(
new DashboardConnection()
.withConfig(
new MetabaseConnection()
.withHostPort(new URI("http://localhost:8080"))
.withUsername("admin")
.withPassword(Fernet.getInstance().encrypt(secretPassword.toLowerCase(Locale.ROOT)))));
} catch (URISyntaxException e) {
e.printStackTrace();
LOG.error("Failed to create CreateDashboardService request", e);
}
return null;
}
@ -279,13 +258,11 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
JsonUtils.convertValue(actualDashboardConnection.getConfig(), MetabaseConnection.class);
}
assertEquals(expectedmetabaseConnection.getHostPort(), actualMetabaseConnection.getHostPort());
if (ADMIN_AUTH_HEADERS.equals(authHeaders) || INGESTION_BOT_AUTH_HEADERS.equals(authHeaders)) {
assertEquals(expectedmetabaseConnection.getUsername(), actualMetabaseConnection.getUsername());
assertTrue(actualMetabaseConnection.getPassword().startsWith("secret:/openmetadata/dashboard/"));
assertTrue(actualMetabaseConnection.getPassword().endsWith("/password"));
assertEquals(expectedmetabaseConnection.getUsername(), actualMetabaseConnection.getUsername());
if (INGESTION_BOT_AUTH_HEADERS.equals(authHeaders)) {
assertEquals(expectedmetabaseConnection.getPassword(), actualMetabaseConnection.getPassword());
} else {
assertNull(actualMetabaseConnection.getUsername());
assertNull(actualMetabaseConnection.getPassword());
assertEquals(actualMetabaseConnection.getPassword(), PasswordEntityMasker.PASSWORD_MASK);
}
}
}

80
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DatabaseServiceResourceTest.java
View File

@ -22,7 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.SNOWFLAKE_DATABASE_CONNECTION;
import static org.openmetadata.service.util.TestUtils.INGESTION_BOT_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.TEST_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.assertResponseContains;
@ -58,6 +58,7 @@ import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.services.database.DatabaseServiceResource.DatabaseServiceList;
import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineResourceTest;
import org.openmetadata.service.secrets.masker.PasswordEntityMasker;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.TestUtils;
import org.openmetadata.service.util.TestUtils.UpdateType;
@ -137,7 +138,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
DatabaseConnection databaseConnection = new DatabaseConnection().withConfig(snowflakeConnection);
update.withConnection(databaseConnection);
service = updateEntity(update, OK, ADMIN_AUTH_HEADERS);
validateDatabaseConnection(databaseConnection, service.getConnection(), service.getServiceType());
validateDatabaseConnection(databaseConnection, service.getConnection(), service.getServiceType(), true);
ConnectionArguments connectionArguments =
new ConnectionArguments()
.withAdditionalProperty("credentials", "/tmp/creds.json")
@ -149,17 +150,19 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
service = updateEntity(update, OK, ADMIN_AUTH_HEADERS);
// Get the recently updated entity and verify the changes
service = getEntity(service.getId(), ADMIN_AUTH_HEADERS);
validateDatabaseConnection(databaseConnection, service.getConnection(), service.getServiceType());
validateDatabaseConnection(databaseConnection, service.getConnection(), service.getServiceType(), true);
assertEquals("description1", service.getDescription());
// non admin/bot user, password fields must be masked
DatabaseService newService = getEntity(service.getId(), "*", TEST_AUTH_HEADERS);
assertEquals(newService.getName(), service.getName());
assertNull(newService.getConnection());
validateDatabaseConnection(databaseConnection, newService.getConnection(), newService.getServiceType(), true);
snowflakeConnection.setPassword("test123");
databaseConnection.setConfig(snowflakeConnection);
update.withConnection(databaseConnection);
service = updateEntity(update, OK, ADMIN_AUTH_HEADERS);
service = getEntity(service.getId(), ADMIN_AUTH_HEADERS);
validateDatabaseConnection(databaseConnection, service.getConnection(), service.getServiceType());
// bot user, password fields must be unmasked.
service = getEntity(service.getId(), INGESTION_BOT_AUTH_HEADERS);
validateDatabaseConnection(databaseConnection, service.getConnection(), service.getServiceType(), false);
}
@Test
@ -184,7 +187,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
assertResponseContains(
() -> updateEntity(update, OK, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"InvalidServiceConnectionException for service [Snowflake] due to [Failed to encrypt connection instance of Snowflake]");
"InvalidServiceConnectionException for service [Snowflake] due to [Failed to unmask connection instance of Snowflake].");
}
@Test
@ -280,24 +283,16 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
.withConnection(TestUtils.SNOWFLAKE_DATABASE_CONNECTION);
}
@Override
public CreateDatabaseService createPutRequest(String name) {
SnowflakeConnection snowflakeConnection =
JsonUtils.convertValue(SNOWFLAKE_DATABASE_CONNECTION.getConfig(), SnowflakeConnection.class);
DatabaseConnection databaseConnection =
JsonUtils.convertValue(SNOWFLAKE_DATABASE_CONNECTION, DatabaseConnection.class);
String secretPassword = "secret:/openmetadata/database/" + name.toLowerCase() + "/password";
return new CreateDatabaseService()
.withName(name)
.withServiceType(DatabaseServiceType.Snowflake)
.withConnection(databaseConnection.withConfig(snowflakeConnection.withPassword(secretPassword)));
}
@Override
public void validateCreatedEntity(
DatabaseService service, CreateDatabaseService createRequest, Map<String, String> authHeaders) {
assertEquals(createRequest.getName(), service.getName());
validateDatabaseConnection(createRequest.getConnection(), service.getConnection(), service.getServiceType());
boolean maskPasswords = true;
if (INGESTION_BOT_AUTH_HEADERS.equals(authHeaders)) {
maskPasswords = false;
}
validateDatabaseConnection(
createRequest.getConnection(), service.getConnection(), service.getServiceType(), maskPasswords);
}
@Override
@ -340,7 +335,8 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
private void validateDatabaseConnection(
DatabaseConnection expectedDatabaseConnection,
DatabaseConnection actualDatabaseConnection,
DatabaseServiceType databaseServiceType) {
DatabaseServiceType databaseServiceType,
boolean maskedPasswords) {
// Validate Database Connection if available. We nullify when not admin or bot
if (expectedDatabaseConnection != null && actualDatabaseConnection != null) {
if (databaseServiceType == DatabaseServiceType.Mysql) {
@ -351,7 +347,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
} else {
actualMysqlConnection = JsonUtils.convertValue(actualDatabaseConnection.getConfig(), MysqlConnection.class);
}
validateMysqlConnection(expectedMysqlConnection, actualMysqlConnection);
validateMysqlConnection(expectedMysqlConnection, actualMysqlConnection, maskedPasswords);
} else if (databaseServiceType == DatabaseServiceType.BigQuery) {
BigQueryConnection expectedBigQueryConnection = (BigQueryConnection) expectedDatabaseConnection.getConfig();
BigQueryConnection actualBigQueryConnection;
@ -361,7 +357,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
actualBigQueryConnection =
JsonUtils.convertValue(actualDatabaseConnection.getConfig(), BigQueryConnection.class);
}
validateBigQueryConnection(expectedBigQueryConnection, actualBigQueryConnection);
validateBigQueryConnection(expectedBigQueryConnection, actualBigQueryConnection, maskedPasswords);
} else if (databaseServiceType == DatabaseServiceType.Redshift) {
RedshiftConnection expectedRedshiftConnection = (RedshiftConnection) expectedDatabaseConnection.getConfig();
RedshiftConnection actualRedshiftConnection;
@ -371,7 +367,7 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
actualRedshiftConnection =
JsonUtils.convertValue(actualDatabaseConnection.getConfig(), RedshiftConnection.class);
}
validateRedshiftConnection(expectedRedshiftConnection, actualRedshiftConnection);
validateRedshiftConnection(expectedRedshiftConnection, actualRedshiftConnection, maskedPasswords);
} else if (databaseServiceType == DatabaseServiceType.Snowflake) {
SnowflakeConnection expectedSnowflakeConnection = (SnowflakeConnection) expectedDatabaseConnection.getConfig();
SnowflakeConnection actualSnowflakeConnection;
@ -381,32 +377,44 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
actualSnowflakeConnection =
JsonUtils.convertValue(actualDatabaseConnection.getConfig(), SnowflakeConnection.class);
}
validateSnowflakeConnection(expectedSnowflakeConnection, actualSnowflakeConnection);
validateSnowflakeConnection(expectedSnowflakeConnection, actualSnowflakeConnection, maskedPasswords);
}
}
}
public static void validateMysqlConnection(
MysqlConnection expectedMysqlConnection, MysqlConnection actualMysqlConnection) {
MysqlConnection expectedMysqlConnection, MysqlConnection actualMysqlConnection, boolean maskedPasswords) {
assertEquals(expectedMysqlConnection.getDatabaseSchema(), actualMysqlConnection.getDatabaseSchema());
assertEquals(expectedMysqlConnection.getHostPort(), actualMysqlConnection.getHostPort());
assertEquals(expectedMysqlConnection.getUsername(), actualMysqlConnection.getUsername());
assertEquals(expectedMysqlConnection.getConnectionOptions(), actualMysqlConnection.getConnectionOptions());
assertEquals(expectedMysqlConnection.getConnectionArguments(), actualMysqlConnection.getConnectionArguments());
if (maskedPasswords) {
assertEquals(actualMysqlConnection.getPassword(), PasswordEntityMasker.PASSWORD_MASK);
} else {
assertEquals(expectedMysqlConnection.getPassword(), actualMysqlConnection.getPassword());
}
}
public static void validateBigQueryConnection(
BigQueryConnection expectedBigQueryConnection, BigQueryConnection actualBigQueryConnection) {
BigQueryConnection expectedBigQueryConnection,
BigQueryConnection actualBigQueryConnection,
boolean maskedPasswords) {
assertEquals(expectedBigQueryConnection.getHostPort(), actualBigQueryConnection.getHostPort());
assertEquals(expectedBigQueryConnection.getCredentials(), actualBigQueryConnection.getCredentials());
assertEquals(expectedBigQueryConnection.getScheme(), actualBigQueryConnection.getScheme());
assertEquals(
expectedBigQueryConnection.getConnectionArguments(), actualBigQueryConnection.getConnectionArguments());
assertEquals(expectedBigQueryConnection.getConnectionOptions(), actualBigQueryConnection.getConnectionOptions());
if (!maskedPasswords) {
assertEquals(expectedBigQueryConnection.getCredentials(), actualBigQueryConnection.getCredentials());
}
}
public static void validateRedshiftConnection(
RedshiftConnection expectedRedshiftConnection, RedshiftConnection actualRedshiftConnection) {
RedshiftConnection expectedRedshiftConnection,
RedshiftConnection actualRedshiftConnection,
boolean maskedPasswords) {
assertEquals(expectedRedshiftConnection.getHostPort(), actualRedshiftConnection.getHostPort());
assertEquals(expectedRedshiftConnection.getUsername(), actualRedshiftConnection.getUsername());
assertEquals(expectedRedshiftConnection.getScheme(), actualRedshiftConnection.getScheme());
@ -414,10 +422,17 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
assertEquals(
expectedRedshiftConnection.getConnectionArguments(), actualRedshiftConnection.getConnectionArguments());
assertEquals(expectedRedshiftConnection.getConnectionOptions(), actualRedshiftConnection.getConnectionOptions());
if (maskedPasswords) {
assertEquals(actualRedshiftConnection.getPassword(), PasswordEntityMasker.PASSWORD_MASK);
} else {
assertEquals(expectedRedshiftConnection.getPassword(), actualRedshiftConnection.getPassword());
}
}
public static void validateSnowflakeConnection(
SnowflakeConnection expectedSnowflakeConnection, SnowflakeConnection actualSnowflakeConnection) {
SnowflakeConnection expectedSnowflakeConnection,
SnowflakeConnection actualSnowflakeConnection,
boolean maskedPasswords) {
assertEquals(expectedSnowflakeConnection.getRole(), actualSnowflakeConnection.getRole());
assertEquals(expectedSnowflakeConnection.getUsername(), actualSnowflakeConnection.getUsername());
assertEquals(expectedSnowflakeConnection.getScheme(), actualSnowflakeConnection.getScheme());
@ -425,5 +440,10 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
assertEquals(
expectedSnowflakeConnection.getConnectionArguments(), actualSnowflakeConnection.getConnectionArguments());
assertEquals(expectedSnowflakeConnection.getConnectionOptions(), actualSnowflakeConnection.getConnectionOptions());
if (maskedPasswords) {
assertEquals(actualSnowflakeConnection.getPassword(), PasswordEntityMasker.PASSWORD_MASK);
} else {
assertEquals(expectedSnowflakeConnection.getPassword(), actualSnowflakeConnection.getPassword());
}
}
}

12
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/MetadataServiceResourceTest.java
View File

@ -192,18 +192,6 @@ public class MetadataServiceResourceTest extends EntityResourceTest<MetadataServ
.withConnection(AMUNDSEN_CONNECTION);
}
@Override
public CreateMetadataService createPutRequest(String name) {
MetadataConnection metadataConnection = JsonUtils.convertValue(AMUNDSEN_CONNECTION, MetadataConnection.class);
AmundsenConnection amundsenConnection =
JsonUtils.convertValue(AMUNDSEN_CONNECTION.getConfig(), AmundsenConnection.class);
String secretPassword = "secret:/openmetadata/metadata/" + name.toLowerCase() + "/password";
return new CreateMetadataService()
.withName(name)
.withServiceType(CreateMetadataService.MetadataServiceType.Amundsen)
.withConnection(metadataConnection.withConfig(amundsenConnection.withPassword(secretPassword)));
}
@Override
public void validateCreatedEntity(
MetadataService service, CreateMetadataService createRequest, Map<String, String> authHeaders) {

28
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/PipelineServiceResourceTest.java
View File

@ -145,7 +145,7 @@ public class PipelineServiceResourceTest extends EntityResourceTest<PipelineServ
assertNotNull(
JsonUtils.readValue(JsonUtils.pojoToJson(service.getConnection().getConfig()), AirflowConnection.class)
.getHostPort());
assertNull(
assertNotNull(
JsonUtils.readValue(JsonUtils.pojoToJson(service.getConnection().getConfig()), AirflowConnection.class)
.getConnection());
}
@ -296,30 +296,18 @@ public class PipelineServiceResourceTest extends EntityResourceTest<PipelineServ
// We need to get inside the general DatabaseConnection and fetch the MysqlConnection
MysqlConnection expectedMysqlConnection = (MysqlConnection) expectedAirflowConnection.getConnection();
// Use the database service tests utilities for the comparison
// only admin can see all connection parameters
if (ADMIN_AUTH_HEADERS.equals(authHeaders) || INGESTION_BOT_AUTH_HEADERS.equals(authHeaders)) {
// only bot can see all connection parameters unmasked. Non bot users can see the connection
// but passwords will be masked
if (INGESTION_BOT_AUTH_HEADERS.equals(authHeaders)) {
MysqlConnection actualMysqlConnection =
JsonUtils.convertValue(actualAirflowConnection.getConnection(), MysqlConnection.class);
validateMysqlConnection(expectedMysqlConnection, actualMysqlConnection);
validateMysqlConnection(expectedMysqlConnection, actualMysqlConnection, false);
} else {
assertNotNull(actualAirflowConnection);
assertNotNull(actualAirflowConnection.getHostPort());
assertNull(actualAirflowConnection.getConnection());
MysqlConnection actualMysqlConnection =
JsonUtils.convertValue(actualAirflowConnection.getConnection(), MysqlConnection.class);
validateMysqlConnection(expectedMysqlConnection, actualMysqlConnection, true);
}
}
@Override
public CreatePipelineService createPutRequest(String name) {
AirflowConnection airflowConnection =
JsonUtils.convertValue(AIRFLOW_CONNECTION.getConfig(), AirflowConnection.class);
MysqlConnection mysqlConnection = JsonUtils.convertValue(airflowConnection.getConnection(), MysqlConnection.class);
PipelineConnection pipelineConnection = JsonUtils.convertValue(AIRFLOW_CONNECTION, PipelineConnection.class);
String secretPassword = "secret:/openmetadata/pipeline/" + name.toLowerCase() + "/connection/password";
return new CreatePipelineService()
.withName(name)
.withServiceType(PipelineServiceType.Airflow)
.withConnection(
pipelineConnection.withConfig(
airflowConnection.withConnection(mysqlConnection.withPassword(secretPassword))));
}
}

1
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/StorageServiceResourceTest.java
View File

@ -206,7 +206,6 @@ public class StorageServiceResourceTest extends EntityResourceTest<StorageServic
assertEquals(
expectedS3Connection.getAwsConfig().getAwsAccessKeyId(),
actualS3Connection.getAwsConfig().getAwsAccessKeyId());
assertTrue(actualS3Connection.getAwsConfig().getAwsSecretAccessKey().contains("secret")); // encrypted
assertEquals(
expectedS3Connection.getAwsConfig().getAwsRegion(), actualS3Connection.getAwsConfig().getAwsRegion());
}

18
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java
View File

@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.INGESTION_BOT_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.service.util.TestUtils.assertListNotNull;
import static org.openmetadata.service.util.TestUtils.assertListNull;
@ -31,7 +32,6 @@ import java.net.URISyntaxException;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
import java.util.function.Predicate;
@ -73,6 +73,7 @@ import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.services.DatabaseServiceResourceTest;
import org.openmetadata.service.secrets.masker.PasswordEntityMasker;
import org.openmetadata.service.security.SecurityUtil;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;
@ -505,7 +506,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
BigQueryConnection expectedBigQueryConnection = (BigQueryConnection) databaseService.getConnection().getConfig();
BigQueryConnection actualBigQueryConnection =
JsonUtils.convertValue(updatedService.getConnection().getConfig(), BigQueryConnection.class);
DatabaseServiceResourceTest.validateBigQueryConnection(expectedBigQueryConnection, actualBigQueryConnection);
DatabaseServiceResourceTest.validateBigQueryConnection(expectedBigQueryConnection, actualBigQueryConnection, true);
}
@Test
@ -607,11 +608,16 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
DbtS3Config actualDbtS3Config = JsonUtils.convertValue(actualDbtPipeline.getDbtConfigSource(), DbtS3Config.class);
assertEquals(actualDbtS3Config.getDbtSecurityConfig().getAwsAccessKeyId(), awsCredentials.getAwsAccessKeyId());
assertEquals(actualDbtS3Config.getDbtSecurityConfig().getAwsRegion(), awsCredentials.getAwsRegion());
assertEquals(PasswordEntityMasker.PASSWORD_MASK, actualDbtS3Config.getDbtSecurityConfig().getAwsSecretAccessKey());
ingestion = getEntity(ingestion.getId(), INGESTION_BOT_AUTH_HEADERS);
actualDbtPipeline = JsonUtils.convertValue(ingestion.getSourceConfig().getConfig(), DbtPipeline.class);
actualDbtS3Config = JsonUtils.convertValue(actualDbtPipeline.getDbtConfigSource(), DbtS3Config.class);
assertEquals(actualDbtS3Config.getDbtSecurityConfig().getAwsAccessKeyId(), awsCredentials.getAwsAccessKeyId());
assertEquals(actualDbtS3Config.getDbtSecurityConfig().getAwsRegion(), awsCredentials.getAwsRegion());
assertEquals(
"secret:/openmetadata/pipeline/"
+ request.getName().toLowerCase(Locale.ROOT)
+ "/sourceconfig/config/dbtconfigsource/dbtsecurityconfig/awssecretaccesskey",
actualDbtS3Config.getDbtSecurityConfig().getAwsSecretAccessKey());
awsCredentials.getAwsSecretAccessKey(), actualDbtS3Config.getDbtSecurityConfig().getAwsSecretAccessKey());
}
@Test

8
openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/EntityMaskerFactoryTest.java
View File

@ -20,15 +20,9 @@ public class EntityMaskerFactoryTest {
EntityMaskerFactory.setEntityMasker(null);
}
@Test
void testInitWithNoopEntityMasker() {
CONFIG.setMaskPasswordsAPI(false);
assertTrue(EntityMaskerFactory.createEntityMasker(CONFIG) instanceof NoopEntityMasker);
}
@Test
void testInitWithPasswordEntityMasker() {
CONFIG.setMaskPasswordsAPI(true);
assertTrue(EntityMaskerFactory.createEntityMasker(CONFIG) instanceof PasswordEntityMasker);
assertTrue(EntityMaskerFactory.createEntityMasker() instanceof PasswordEntityMasker);
}
}

8
openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/NoopEntityMaskerTest.java
View File

@ -1,8 +0,0 @@
package org.openmetadata.service.secrets.masker;
public class NoopEntityMaskerTest extends TestEntityMasker {
public NoopEntityMaskerTest() {
CONFIG.setMaskPasswordsAPI(false);
}
}

4
openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/PasswordEntityMaskerTest.java
View File

@ -26,7 +26,7 @@ public class PasswordEntityMaskerTest extends TestEntityMasker {
Assertions.assertThrows(
EntityMaskException.class,
() -> {
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(mysqlConnectionObject, "Mysql", ServiceType.DATABASE);
});
@ -38,7 +38,7 @@ public class PasswordEntityMaskerTest extends TestEntityMasker {
Assertions.assertThrows(
EntityMaskException.class,
() -> {
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(
mysqlConnectionObject, new MysqlConnection(), "Mysql", ServiceType.DATABASE);
});

36
openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/TestEntityMasker.java
View File

@ -48,13 +48,13 @@ abstract class TestEntityMasker {
AirflowConnection airflowConnection = new AirflowConnection().withConnection(buildMysqlConnection());
AirflowConnection masked =
(AirflowConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(airflowConnection, "Airflow", ServiceType.PIPELINE);
assertNotNull(masked);
assertEquals(((MysqlConnection) masked.getConnection()).getPassword(), getMaskedPassword());
AirflowConnection unmasked =
(AirflowConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(masked, airflowConnection, "Airflow", ServiceType.PIPELINE);
assertEquals(((MysqlConnection) unmasked.getConnection()).getPassword(), PASSWORD);
}
@ -64,13 +64,13 @@ abstract class TestEntityMasker {
BigQueryConnection bigQueryConnection = new BigQueryConnection().withCredentials(buildGcsCredentials());
BigQueryConnection masked =
(BigQueryConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(bigQueryConnection, "BigQuery", ServiceType.DATABASE);
assertNotNull(masked);
assertEquals(getPrivateKeyFromGcsConfig(masked.getCredentials()), getMaskedPassword());
BigQueryConnection unmasked =
(BigQueryConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(masked, bigQueryConnection, "BigQuery", ServiceType.DATABASE);
assertEquals(getPrivateKeyFromGcsConfig(unmasked.getCredentials()), PASSWORD);
}
@ -80,14 +80,14 @@ abstract class TestEntityMasker {
DatalakeConnection datalakeConnection = new DatalakeConnection().withConfigSource(buildGcsConfig());
DatalakeConnection masked =
(DatalakeConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(datalakeConnection, "Datalake", ServiceType.DATABASE);
assertNotNull(masked);
assertEquals(
getPrivateKeyFromGcsConfig(((GCSConfig) masked.getConfigSource()).getSecurityConfig()), getMaskedPassword());
DatalakeConnection unmasked =
(DatalakeConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(masked, datalakeConnection, "Datalake", ServiceType.DATABASE);
assertEquals(getPrivateKeyFromGcsConfig(((GCSConfig) unmasked.getConfigSource()).getSecurityConfig()), PASSWORD);
}
@ -96,7 +96,7 @@ abstract class TestEntityMasker {
void testDbtPipelineMasker() {
IngestionPipeline dbtPipeline = buildIngestionPipeline();
IngestionPipeline originalDbtPipeline = buildIngestionPipeline();
EntityMaskerFactory.createEntityMasker(CONFIG).maskIngestionPipeline(dbtPipeline);
EntityMaskerFactory.createEntityMasker().maskIngestionPipeline(dbtPipeline);
assertNotNull(dbtPipeline);
assertEquals(
getPrivateKeyFromGcsConfig(
@ -106,7 +106,7 @@ abstract class TestEntityMasker {
assertEquals(
((GoogleSSOClientConfig) dbtPipeline.getOpenMetadataServerConnection().getSecurityConfig()).getSecretKey(),
getMaskedPassword());
EntityMaskerFactory.createEntityMasker(CONFIG).unmaskIngestionPipeline(dbtPipeline, originalDbtPipeline);
EntityMaskerFactory.createEntityMasker().unmaskIngestionPipeline(dbtPipeline, originalDbtPipeline);
assertEquals(
getPrivateKeyFromGcsConfig(
((DbtGCSConfig) ((DbtPipeline) dbtPipeline.getSourceConfig().getConfig()).getDbtConfigSource())
@ -123,13 +123,13 @@ abstract class TestEntityMasker {
buildAuthenticationMechanism(AuthenticationMechanism.AuthType.SSO);
AuthenticationMechanism originalSsoAuthenticationMechanism =
buildAuthenticationMechanism(AuthenticationMechanism.AuthType.SSO);
EntityMaskerFactory.createEntityMasker(CONFIG).maskAuthenticationMechanism("test", authenticationMechanism);
EntityMaskerFactory.createEntityMasker().maskAuthenticationMechanism("test", authenticationMechanism);
assertNotNull(authenticationMechanism.getConfig());
assertEquals(
((GoogleSSOClientConfig) ((SSOAuthMechanism) authenticationMechanism.getConfig()).getAuthConfig())
.getSecretKey(),
getMaskedPassword());
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.unmaskAuthenticationMechanism("test", authenticationMechanism, originalSsoAuthenticationMechanism);
assertEquals(
((GoogleSSOClientConfig) ((SSOAuthMechanism) authenticationMechanism.getConfig()).getAuthConfig())
@ -143,9 +143,9 @@ abstract class TestEntityMasker {
buildAuthenticationMechanism(AuthenticationMechanism.AuthType.JWT);
AuthenticationMechanism originalSsoAuthenticationMechanism =
buildAuthenticationMechanism(AuthenticationMechanism.AuthType.JWT);
EntityMaskerFactory.createEntityMasker(CONFIG).maskAuthenticationMechanism("test", authenticationMechanism);
EntityMaskerFactory.createEntityMasker().maskAuthenticationMechanism("test", authenticationMechanism);
assertTrue(authenticationMechanism.getConfig() instanceof JWTAuthMechanism);
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.unmaskAuthenticationMechanism("test", authenticationMechanism, originalSsoAuthenticationMechanism);
assertTrue(authenticationMechanism.getConfig() instanceof JWTAuthMechanism);
}
@ -155,13 +155,13 @@ abstract class TestEntityMasker {
SupersetConnection supersetConnection = new SupersetConnection().withConnection(buildMysqlConnection());
SupersetConnection masked =
(SupersetConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(supersetConnection, "Superset", ServiceType.DASHBOARD);
assertNotNull(masked);
assertEquals(((MysqlConnection) masked.getConnection()).getPassword(), getMaskedPassword());
SupersetConnection unmasked =
(SupersetConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(masked, supersetConnection, "Superset", ServiceType.DASHBOARD);
assertEquals(((MysqlConnection) unmasked.getConnection()).getPassword(), PASSWORD);
}
@ -176,7 +176,7 @@ abstract class TestEntityMasker {
.withServiceType(ServiceType.DATABASE)
.withConnectionType("Mysql"))
.withOpenMetadataServerConnection(buildOpenMetadataConnection());
Workflow masked = EntityMaskerFactory.createEntityMasker(CONFIG).maskWorkflow(workflow);
Workflow masked = EntityMaskerFactory.createEntityMasker().maskWorkflow(workflow);
assertNotNull(masked);
assertEquals(
((MysqlConnection)
@ -186,7 +186,7 @@ abstract class TestEntityMasker {
assertEquals(
((GoogleSSOClientConfig) masked.getOpenMetadataServerConnection().getSecurityConfig()).getSecretKey(),
getMaskedPassword());
Workflow unmasked = EntityMaskerFactory.createEntityMasker(CONFIG).unmaskWorkflow(masked, workflow);
Workflow unmasked = EntityMaskerFactory.createEntityMasker().unmaskWorkflow(masked, workflow);
assertEquals(
((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) unmasked.getRequest()).getConnection())
@ -203,13 +203,13 @@ abstract class TestEntityMasker {
MysqlConnection mysqlConnection = buildMysqlConnection();
MysqlConnection masked =
(MysqlConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(mysqlConnection, "Mysql", ServiceType.DATABASE);
assertNotNull(masked);
assertEquals(masked.getPassword(), getMaskedPassword());
MysqlConnection unmasked =
(MysqlConnection)
EntityMaskerFactory.createEntityMasker(CONFIG)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(masked, mysqlConnection, "Mysql", ServiceType.DATABASE);
assertEquals(unmasked.getPassword(), PASSWORD);
}

4
openmetadata-service/src/test/resources/openmetadata-secure-test.yaml
View File

@ -116,10 +116,8 @@ migrationConfiguration:
# port: 0
secretsManagerConfiguration:
secretsManager: in-memory
secretsManager: noop
security:
maskPasswordsAPI: false
health:
delayedShutdownHandlerEnabled: true