mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-11 16:58:38 +00:00
Fix: Improvements on secret manager implementation (#7282)
* Change local secret manager by noop * Update openmetadata-secure-test.yaml
This commit is contained in:
parent
3373e9da9d
commit
b829a2cbf3
@ -13,7 +13,7 @@
|
|||||||
|
|
||||||
package org.openmetadata.catalog.secrets;
|
package org.openmetadata.catalog.secrets;
|
||||||
|
|
||||||
import static org.openmetadata.catalog.services.connections.metadata.SecretsManagerProvider.LOCAL;
|
import static org.openmetadata.catalog.services.connections.metadata.SecretsManagerProvider.NOOP;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import java.lang.reflect.InvocationTargetException;
|
import java.lang.reflect.InvocationTargetException;
|
||||||
@ -27,14 +27,14 @@ import org.openmetadata.catalog.fernet.Fernet;
|
|||||||
import org.openmetadata.catalog.services.connections.metadata.OpenMetadataServerConnection;
|
import org.openmetadata.catalog.services.connections.metadata.OpenMetadataServerConnection;
|
||||||
import org.openmetadata.catalog.util.JsonUtils;
|
import org.openmetadata.catalog.util.JsonUtils;
|
||||||
|
|
||||||
public class LocalSecretsManager extends SecretsManager {
|
public class NoopSecretsManager extends SecretsManager {
|
||||||
|
|
||||||
private static LocalSecretsManager INSTANCE;
|
private static NoopSecretsManager INSTANCE;
|
||||||
|
|
||||||
private Fernet fernet;
|
private Fernet fernet;
|
||||||
|
|
||||||
private LocalSecretsManager(String clusterPrefix) {
|
private NoopSecretsManager(String clusterPrefix) {
|
||||||
super(LOCAL, clusterPrefix);
|
super(NOOP, clusterPrefix);
|
||||||
this.fernet = Fernet.getInstance();
|
this.fernet = Fernet.getInstance();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -113,8 +113,8 @@ public class LocalSecretsManager extends SecretsManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static LocalSecretsManager getInstance(String clusterPrefix) {
|
public static NoopSecretsManager getInstance(String clusterPrefix) {
|
||||||
if (INSTANCE == null) INSTANCE = new LocalSecretsManager(clusterPrefix);
|
if (INSTANCE == null) INSTANCE = new NoopSecretsManager(clusterPrefix);
|
||||||
return INSTANCE;
|
return INSTANCE;
|
||||||
}
|
}
|
||||||
|
|
@ -22,7 +22,7 @@ import org.openmetadata.catalog.services.connections.metadata.SecretsManagerProv
|
|||||||
@Setter
|
@Setter
|
||||||
public class SecretsManagerConfiguration {
|
public class SecretsManagerConfiguration {
|
||||||
|
|
||||||
public static final SecretsManagerProvider DEFAULT_SECRET_MANAGER = SecretsManagerProvider.LOCAL;
|
public static final SecretsManagerProvider DEFAULT_SECRET_MANAGER = SecretsManagerProvider.NOOP;
|
||||||
|
|
||||||
private SecretsManagerProvider secretsManager;
|
private SecretsManagerProvider secretsManager;
|
||||||
|
|
||||||
|
@ -23,8 +23,8 @@ public class SecretsManagerFactory {
|
|||||||
? config.getSecretsManager()
|
? config.getSecretsManager()
|
||||||
: SecretsManagerConfiguration.DEFAULT_SECRET_MANAGER;
|
: SecretsManagerConfiguration.DEFAULT_SECRET_MANAGER;
|
||||||
switch (secretManager) {
|
switch (secretManager) {
|
||||||
case LOCAL:
|
case NOOP:
|
||||||
return LocalSecretsManager.getInstance(clusterName);
|
return NoopSecretsManager.getInstance(clusterName);
|
||||||
case AWS:
|
case AWS:
|
||||||
return AWSSecretsManager.getInstance(config, clusterName);
|
return AWSSecretsManager.getInstance(config, clusterName);
|
||||||
case AWS_SSM:
|
case AWS_SSM:
|
||||||
|
@ -5,6 +5,6 @@
|
|||||||
"description": "OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager providers as the ones configured on the OpenMetadata server.",
|
"description": "OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager providers as the ones configured on the OpenMetadata server.",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"javaType": "org.openmetadata.catalog.services.connections.metadata.SecretsManagerProvider",
|
"javaType": "org.openmetadata.catalog.services.connections.metadata.SecretsManagerProvider",
|
||||||
"enum": ["local", "aws", "aws-ssm"],
|
"enum": ["noop", "aws", "aws-ssm"],
|
||||||
"additionalProperties": false
|
"additionalProperties": false
|
||||||
}
|
}
|
||||||
|
@ -85,7 +85,7 @@ public class AWSSSMSecretsManagerTest extends ExternalSecretsManagerTest {
|
|||||||
verifySecretIdGetCalls(expectedSecretId, 1);
|
verifySecretIdGetCalls(expectedSecretId, 1);
|
||||||
assertEquals(expectedSecretId, createSecretCaptor.getValue().name());
|
assertEquals(expectedSecretId, createSecretCaptor.getValue().name());
|
||||||
assertEquals(
|
assertEquals(
|
||||||
"{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true}",
|
"{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true,\"supportsQueryComment\":true}",
|
||||||
createSecretCaptor.getValue().value());
|
createSecretCaptor.getValue().value());
|
||||||
assertNull(serviceConnection);
|
assertNull(serviceConnection);
|
||||||
}
|
}
|
||||||
|
@ -67,7 +67,7 @@ public class AWSSecretsManagerTest extends ExternalSecretsManagerTest {
|
|||||||
verifySecretIdGetCalls(expectedSecretId, 1);
|
verifySecretIdGetCalls(expectedSecretId, 1);
|
||||||
assertEquals(expectedSecretId, createSecretCaptor.getValue().name());
|
assertEquals(expectedSecretId, createSecretCaptor.getValue().name());
|
||||||
assertEquals(
|
assertEquals(
|
||||||
"{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true}",
|
"{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true,\"supportsQueryComment\":true}",
|
||||||
createSecretCaptor.getValue().secretString());
|
createSecretCaptor.getValue().secretString());
|
||||||
assertNull(serviceConnection);
|
assertNull(serviceConnection);
|
||||||
}
|
}
|
||||||
|
@ -66,7 +66,7 @@ public abstract class ExternalSecretsManagerTest {
|
|||||||
static final String TEST_CONNECTION_SECRET_ID_PREFIX = "test-connection-temp";
|
static final String TEST_CONNECTION_SECRET_ID_PREFIX = "test-connection-temp";
|
||||||
static final boolean DECRYPT = false;
|
static final boolean DECRYPT = false;
|
||||||
static final String EXPECTED_CONNECTION_JSON =
|
static final String EXPECTED_CONNECTION_JSON =
|
||||||
"{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"password\":\"openmetadata-test\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true}";
|
"{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"password\":\"openmetadata-test\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true,\"supportsQueryComment\":true}";
|
||||||
static final String EXPECTED_SECRET_ID = "/openmetadata/service/database/mysql/test";
|
static final String EXPECTED_SECRET_ID = "/openmetadata/service/database/mysql/test";
|
||||||
|
|
||||||
AWSBasedSecretsManager secretsManager;
|
AWSBasedSecretsManager secretsManager;
|
||||||
|
@ -62,17 +62,17 @@ import org.openmetadata.catalog.services.connections.mlModel.SklearnConnection;
|
|||||||
import org.openmetadata.catalog.type.EntityReference;
|
import org.openmetadata.catalog.type.EntityReference;
|
||||||
|
|
||||||
@ExtendWith(MockitoExtension.class)
|
@ExtendWith(MockitoExtension.class)
|
||||||
public class LocalSecretsManagerTest {
|
public class NoopSecretsManagerTest {
|
||||||
|
|
||||||
private static final boolean ENCRYPT = true;
|
private static final boolean ENCRYPT = true;
|
||||||
private static final boolean DECRYPT = false;
|
private static final boolean DECRYPT = false;
|
||||||
private static final String ENCRYPTED_VALUE = "fernet:abcdef";
|
private static final String ENCRYPTED_VALUE = "fernet:abcdef";
|
||||||
private static final String DECRYPTED_VALUE = "123456";
|
private static final String DECRYPTED_VALUE = "123456";
|
||||||
private static LocalSecretsManager secretsManager;
|
private static NoopSecretsManager secretsManager;
|
||||||
|
|
||||||
@BeforeAll
|
@BeforeAll
|
||||||
static void setUp() {
|
static void setUp() {
|
||||||
secretsManager = LocalSecretsManager.getInstance("openmetadata");
|
secretsManager = NoopSecretsManager.getInstance("openmetadata");
|
||||||
Fernet fernet = Mockito.mock(Fernet.class);
|
Fernet fernet = Mockito.mock(Fernet.class);
|
||||||
lenient().when(fernet.decrypt(anyString())).thenReturn(DECRYPTED_VALUE);
|
lenient().when(fernet.decrypt(anyString())).thenReturn(DECRYPTED_VALUE);
|
||||||
lenient().when(fernet.encrypt(anyString())).thenReturn(ENCRYPTED_VALUE);
|
lenient().when(fernet.encrypt(anyString())).thenReturn(ENCRYPTED_VALUE);
|
||||||
@ -146,7 +146,7 @@ public class LocalSecretsManagerTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testReturnsExpectedSecretManagerProvider() {
|
void testReturnsExpectedSecretManagerProvider() {
|
||||||
assertEquals(SecretsManagerProvider.LOCAL, secretsManager.getSecretsManagerProvider());
|
assertEquals(SecretsManagerProvider.NOOP, secretsManager.getSecretsManagerProvider());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@ParameterizedTest
|
@ -33,18 +33,18 @@ public class SecretsManagerFactoryTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testDefaultIsCreatedIfNullConfig() {
|
void testDefaultIsCreatedIfNullConfig() {
|
||||||
assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof LocalSecretsManager);
|
assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof NoopSecretsManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testDefaultIsCreatedIfMissingSecretManager() {
|
void testDefaultIsCreatedIfMissingSecretManager() {
|
||||||
assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof LocalSecretsManager);
|
assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof NoopSecretsManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testIsCreatedIfLocalSecretsManager() {
|
void testIsCreatedIfLocalSecretsManager() {
|
||||||
config.setSecretsManager(SecretsManagerProvider.LOCAL);
|
config.setSecretsManager(SecretsManagerProvider.NOOP);
|
||||||
assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof LocalSecretsManager);
|
assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof NoopSecretsManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -114,7 +114,7 @@ migrationConfiguration:
|
|||||||
# port: 0
|
# port: 0
|
||||||
|
|
||||||
secretsManagerConfiguration:
|
secretsManagerConfiguration:
|
||||||
secretsManager: local
|
secretsManager: noop
|
||||||
|
|
||||||
health:
|
health:
|
||||||
delayedShutdownHandlerEnabled: true
|
delayedShutdownHandlerEnabled: true
|
||||||
|
@ -210,7 +210,7 @@ fernetConfiguration:
|
|||||||
fernetKey: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
fernetKey: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||||
|
|
||||||
secretsManagerConfiguration:
|
secretsManagerConfiguration:
|
||||||
secretsManager: ${SECRET_MANAGER:-local} # Possible values are "local", "aws", "aws-ssm"
|
secretsManager: ${SECRET_MANAGER:-noop} # Possible values are "noop", "aws", "aws-ssm"
|
||||||
# it will use the default auth provider for the secrets' manager service if parameters are not set
|
# it will use the default auth provider for the secrets' manager service if parameters are not set
|
||||||
parameters:
|
parameters:
|
||||||
region: ${OM_SM_REGION:-""}
|
region: ${OM_SM_REGION:-""}
|
||||||
|
@ -12,40 +12,23 @@
|
|||||||
"""
|
"""
|
||||||
Abstract class for AWS based secrets manager implementations
|
Abstract class for AWS based secrets manager implementations
|
||||||
"""
|
"""
|
||||||
import json
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from metadata.clients.aws_client import AWSClient
|
from metadata.clients.aws_client import AWSClient
|
||||||
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
|
|
||||||
AuthProvider,
|
|
||||||
OpenMetadataConnection,
|
|
||||||
)
|
|
||||||
from metadata.generated.schema.entity.services.connections.metadata.secretsManagerProvider import (
|
from metadata.generated.schema.entity.services.connections.metadata.secretsManagerProvider import (
|
||||||
SecretsManagerProvider,
|
SecretsManagerProvider,
|
||||||
)
|
)
|
||||||
from metadata.generated.schema.entity.services.connections.serviceConnection import (
|
|
||||||
ServiceConnection,
|
|
||||||
)
|
|
||||||
from metadata.generated.schema.metadataIngestion.workflow import SourceConfig
|
|
||||||
from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials
|
from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials
|
||||||
from metadata.utils.logger import utils_logger
|
from metadata.utils.logger import utils_logger
|
||||||
from metadata.utils.secrets.secrets_manager import (
|
from metadata.utils.secrets.external_secrets_manager import ExternalSecretsManager
|
||||||
AUTH_PROVIDER_MAPPING,
|
|
||||||
AUTH_PROVIDER_SECRET_PREFIX,
|
|
||||||
DBT_SOURCE_CONFIG_SECRET_PREFIX,
|
|
||||||
TEST_CONNECTION_TEMP_SECRET_PREFIX,
|
|
||||||
SecretsManager,
|
|
||||||
ServiceConnectionType,
|
|
||||||
ServiceWithConnectionType,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = utils_logger()
|
logger = utils_logger()
|
||||||
|
|
||||||
NULL_VALUE = "null"
|
NULL_VALUE = "null"
|
||||||
|
|
||||||
|
|
||||||
class AWSBasedSecretsManager(SecretsManager, ABC):
|
class AWSBasedSecretsManager(ExternalSecretsManager, ABC):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
credentials: Optional[AWSCredentials],
|
credentials: Optional[AWSCredentials],
|
||||||
@ -53,94 +36,8 @@ class AWSBasedSecretsManager(SecretsManager, ABC):
|
|||||||
provider: SecretsManagerProvider,
|
provider: SecretsManagerProvider,
|
||||||
cluster_prefix: str,
|
cluster_prefix: str,
|
||||||
):
|
):
|
||||||
super().__init__(cluster_prefix)
|
super().__init__(cluster_prefix, provider)
|
||||||
self.client = AWSClient(credentials).get_client(client)
|
self.client = AWSClient(credentials).get_client(client)
|
||||||
self.provider = provider.name
|
|
||||||
|
|
||||||
def retrieve_service_connection(
|
|
||||||
self,
|
|
||||||
service: ServiceWithConnectionType,
|
|
||||||
service_type: str,
|
|
||||||
) -> ServiceConnection:
|
|
||||||
"""
|
|
||||||
Retrieve the service connection from the AWS client store from a given service connection object.
|
|
||||||
:param service: Service connection object e.g. DatabaseConnection
|
|
||||||
:param service_type: Service type e.g. databaseService
|
|
||||||
"""
|
|
||||||
logger.debug(
|
|
||||||
f"Retrieving service connection from {self.provider} secrets' manager for {service_type} - {service.name}"
|
|
||||||
)
|
|
||||||
service_connection_type = service.serviceType.value
|
|
||||||
service_name = service.name.__root__
|
|
||||||
secret_id = self.build_secret_id(
|
|
||||||
"service", service_type, service_connection_type, service_name
|
|
||||||
)
|
|
||||||
connection_class = self.get_connection_class(
|
|
||||||
service_type, service_connection_type
|
|
||||||
)
|
|
||||||
service_conn_class = self.get_service_connection_class(service_type)
|
|
||||||
service_connection = service_conn_class(
|
|
||||||
config=connection_class.parse_obj(
|
|
||||||
json.loads(self.get_string_value(secret_id))
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return ServiceConnection(__root__=service_connection)
|
|
||||||
|
|
||||||
def add_auth_provider_security_config(self, config: OpenMetadataConnection) -> None:
|
|
||||||
"""
|
|
||||||
Add the auth provider security config from the AWS client store to a given OpenMetadata connection object.
|
|
||||||
:param config: OpenMetadataConnection object
|
|
||||||
"""
|
|
||||||
logger.debug(
|
|
||||||
f"Adding auth provider security config using {self.provider} secrets' manager"
|
|
||||||
)
|
|
||||||
if config.authProvider != AuthProvider.no_auth:
|
|
||||||
secret_id = self.build_secret_id(
|
|
||||||
AUTH_PROVIDER_SECRET_PREFIX, config.authProvider.value.lower()
|
|
||||||
)
|
|
||||||
auth_config_json = self.get_string_value(secret_id)
|
|
||||||
try:
|
|
||||||
config.securityConfig = AUTH_PROVIDER_MAPPING.get(
|
|
||||||
config.authProvider
|
|
||||||
).parse_obj(json.loads(auth_config_json))
|
|
||||||
except KeyError as err:
|
|
||||||
msg = f"No client implemented for auth provider [{config.authProvider}]: {err}"
|
|
||||||
raise NotImplementedError(msg)
|
|
||||||
|
|
||||||
def retrieve_dbt_source_config(
|
|
||||||
self, source_config: SourceConfig, pipeline_name: str
|
|
||||||
) -> object:
|
|
||||||
"""
|
|
||||||
Retrieve the DBT source config from the AWS client store from a source config object.
|
|
||||||
:param source_config: SourceConfig object
|
|
||||||
:param pipeline_name: the pipeline's name
|
|
||||||
:return:
|
|
||||||
"""
|
|
||||||
logger.debug(
|
|
||||||
f"Retrieving source_config from {self.provider} secrets' manager for {pipeline_name}"
|
|
||||||
)
|
|
||||||
secret_id = self.build_secret_id(DBT_SOURCE_CONFIG_SECRET_PREFIX, pipeline_name)
|
|
||||||
source_config_json = self.get_string_value(secret_id)
|
|
||||||
return json.loads(source_config_json) if source_config_json else None
|
|
||||||
|
|
||||||
def retrieve_temp_service_test_connection(
|
|
||||||
self,
|
|
||||||
connection: ServiceConnectionType,
|
|
||||||
service_type: str,
|
|
||||||
) -> ServiceConnectionType:
|
|
||||||
"""
|
|
||||||
Retrieve the service connection from the AWS client stored in a temporary secret depending on the service
|
|
||||||
type.
|
|
||||||
:param connection: Connection of the service
|
|
||||||
:param service_type: Service type e.g. Database
|
|
||||||
"""
|
|
||||||
secret_id = self.build_secret_id(
|
|
||||||
TEST_CONNECTION_TEMP_SECRET_PREFIX, service_type
|
|
||||||
)
|
|
||||||
service_conn_class = self.get_service_connection_class(service_type)
|
|
||||||
stored_connection = service_conn_class()
|
|
||||||
source_config_json = self.get_string_value(secret_id)
|
|
||||||
return stored_connection.parse_raw(source_config_json)
|
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_string_value(self, name: str) -> str:
|
def get_string_value(self, name: str) -> str:
|
||||||
|
147
ingestion/src/metadata/utils/secrets/external_secrets_manager.py
Normal file
147
ingestion/src/metadata/utils/secrets/external_secrets_manager.py
Normal file
@ -0,0 +1,147 @@
|
|||||||
|
# Copyright 2022 Collate
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Abstract class for third party secrets' manager implementations
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
from metadata.clients.aws_client import AWSClient
|
||||||
|
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
|
||||||
|
AuthProvider,
|
||||||
|
OpenMetadataConnection,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.entity.services.connections.metadata.secretsManagerProvider import (
|
||||||
|
SecretsManagerProvider,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.entity.services.connections.serviceConnection import (
|
||||||
|
ServiceConnection,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.metadataIngestion.workflow import SourceConfig
|
||||||
|
from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials
|
||||||
|
from metadata.utils.logger import utils_logger
|
||||||
|
from metadata.utils.secrets.secrets_manager import (
|
||||||
|
AUTH_PROVIDER_MAPPING,
|
||||||
|
AUTH_PROVIDER_SECRET_PREFIX,
|
||||||
|
DBT_SOURCE_CONFIG_SECRET_PREFIX,
|
||||||
|
TEST_CONNECTION_TEMP_SECRET_PREFIX,
|
||||||
|
SecretsManager,
|
||||||
|
ServiceConnectionType,
|
||||||
|
ServiceWithConnectionType,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = utils_logger()
|
||||||
|
|
||||||
|
NULL_VALUE = "null"
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalSecretsManager(SecretsManager, ABC):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
cluster_prefix: str,
|
||||||
|
provider: SecretsManagerProvider,
|
||||||
|
):
|
||||||
|
super().__init__(cluster_prefix)
|
||||||
|
self.provider = provider.name
|
||||||
|
|
||||||
|
def retrieve_service_connection(
|
||||||
|
self,
|
||||||
|
service: ServiceWithConnectionType,
|
||||||
|
service_type: str,
|
||||||
|
) -> ServiceConnection:
|
||||||
|
"""
|
||||||
|
Retrieve the service connection from the AWS client store from a given service connection object.
|
||||||
|
:param service: Service connection object e.g. DatabaseConnection
|
||||||
|
:param service_type: Service type e.g. databaseService
|
||||||
|
"""
|
||||||
|
logger.debug(
|
||||||
|
f"Retrieving service connection from {self.provider} secrets' manager for {service_type} - {service.name}"
|
||||||
|
)
|
||||||
|
service_connection_type = service.serviceType.value
|
||||||
|
service_name = service.name.__root__
|
||||||
|
secret_id = self.build_secret_id(
|
||||||
|
"service", service_type, service_connection_type, service_name
|
||||||
|
)
|
||||||
|
connection_class = self.get_connection_class(
|
||||||
|
service_type, service_connection_type
|
||||||
|
)
|
||||||
|
service_conn_class = self.get_service_connection_class(service_type)
|
||||||
|
service_connection = service_conn_class(
|
||||||
|
config=connection_class.parse_obj(
|
||||||
|
json.loads(self.get_string_value(secret_id))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return ServiceConnection(__root__=service_connection)
|
||||||
|
|
||||||
|
def add_auth_provider_security_config(self, config: OpenMetadataConnection) -> None:
|
||||||
|
"""
|
||||||
|
Add the auth provider security config from the AWS client store to a given OpenMetadata connection object.
|
||||||
|
:param config: OpenMetadataConnection object
|
||||||
|
"""
|
||||||
|
logger.debug(
|
||||||
|
f"Adding auth provider security config using {self.provider} secrets' manager"
|
||||||
|
)
|
||||||
|
if config.authProvider != AuthProvider.no_auth:
|
||||||
|
secret_id = self.build_secret_id(
|
||||||
|
AUTH_PROVIDER_SECRET_PREFIX, config.authProvider.value.lower()
|
||||||
|
)
|
||||||
|
auth_config_json = self.get_string_value(secret_id)
|
||||||
|
try:
|
||||||
|
config.securityConfig = AUTH_PROVIDER_MAPPING.get(
|
||||||
|
config.authProvider
|
||||||
|
).parse_obj(json.loads(auth_config_json))
|
||||||
|
except KeyError as err:
|
||||||
|
msg = f"No client implemented for auth provider [{config.authProvider}]: {err}"
|
||||||
|
raise NotImplementedError(msg)
|
||||||
|
|
||||||
|
def retrieve_dbt_source_config(
|
||||||
|
self, source_config: SourceConfig, pipeline_name: str
|
||||||
|
) -> object:
|
||||||
|
"""
|
||||||
|
Retrieve the DBT source config from the AWS client store from a source config object.
|
||||||
|
:param source_config: SourceConfig object
|
||||||
|
:param pipeline_name: the pipeline's name
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
logger.debug(
|
||||||
|
f"Retrieving source_config from {self.provider} secrets' manager for {pipeline_name}"
|
||||||
|
)
|
||||||
|
secret_id = self.build_secret_id(DBT_SOURCE_CONFIG_SECRET_PREFIX, pipeline_name)
|
||||||
|
source_config_json = self.get_string_value(secret_id)
|
||||||
|
return json.loads(source_config_json) if source_config_json else None
|
||||||
|
|
||||||
|
def retrieve_temp_service_test_connection(
|
||||||
|
self,
|
||||||
|
connection: ServiceConnectionType,
|
||||||
|
service_type: str,
|
||||||
|
) -> ServiceConnectionType:
|
||||||
|
"""
|
||||||
|
Retrieve the service connection from the AWS client stored in a temporary secret depending on the service
|
||||||
|
type.
|
||||||
|
:param connection: Connection of the service
|
||||||
|
:param service_type: Service type e.g. Database
|
||||||
|
"""
|
||||||
|
secret_id = self.build_secret_id(
|
||||||
|
TEST_CONNECTION_TEMP_SECRET_PREFIX, service_type
|
||||||
|
)
|
||||||
|
service_conn_class = self.get_service_connection_class(service_type)
|
||||||
|
stored_connection = service_conn_class()
|
||||||
|
source_config_json = self.get_string_value(secret_id)
|
||||||
|
return stored_connection.parse_raw(source_config_json)
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_string_value(self, name: str) -> str:
|
||||||
|
"""
|
||||||
|
:param name: The secret name to retrieve
|
||||||
|
:return: The value of the secret
|
||||||
|
"""
|
||||||
|
pass
|
@ -30,12 +30,12 @@ from metadata.utils.secrets.secrets_manager import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class LocalSecretsManager(SecretsManager):
|
class NoopSecretsManager(SecretsManager):
|
||||||
"""
|
"""
|
||||||
LocalSecretsManager is used when there is not a secrets' manager configured.
|
LocalSecretsManager is used when there is not a secrets' manager configured.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
provider: str = SecretsManagerProvider.local.name
|
provider: str = SecretsManagerProvider.noop.name
|
||||||
|
|
||||||
def add_auth_provider_security_config(
|
def add_auth_provider_security_config(
|
||||||
self, open_metadata_connection: OpenMetadataConnection
|
self, open_metadata_connection: OpenMetadataConnection
|
@ -23,7 +23,7 @@ from metadata.generated.schema.entity.services.connections.metadata.secretsManag
|
|||||||
from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials
|
from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials
|
||||||
from metadata.utils.secrets.aws_secrets_manager import AWSSecretsManager
|
from metadata.utils.secrets.aws_secrets_manager import AWSSecretsManager
|
||||||
from metadata.utils.secrets.aws_ssm_secrets_manager import AWSSSMSecretsManager
|
from metadata.utils.secrets.aws_ssm_secrets_manager import AWSSSMSecretsManager
|
||||||
from metadata.utils.secrets.local_secrets_manager import LocalSecretsManager
|
from metadata.utils.secrets.noop_secrets_manager import NoopSecretsManager
|
||||||
from metadata.utils.secrets.secrets_manager import SecretsManager
|
from metadata.utils.secrets.secrets_manager import SecretsManager
|
||||||
|
|
||||||
|
|
||||||
@ -58,9 +58,9 @@ def get_secrets_manager(
|
|||||||
"""
|
"""
|
||||||
if (
|
if (
|
||||||
secrets_manager_provider is None
|
secrets_manager_provider is None
|
||||||
or secrets_manager_provider == SecretsManagerProvider.local
|
or secrets_manager_provider == SecretsManagerProvider.noop
|
||||||
):
|
):
|
||||||
return LocalSecretsManager(cluster_name)
|
return NoopSecretsManager(cluster_name)
|
||||||
elif secrets_manager_provider == SecretsManagerProvider.aws:
|
elif secrets_manager_provider == SecretsManagerProvider.aws:
|
||||||
return AWSSecretsManager(credentials, cluster_name)
|
return AWSSecretsManager(credentials, cluster_name)
|
||||||
elif secrets_manager_provider == SecretsManagerProvider.aws_ssm:
|
elif secrets_manager_provider == SecretsManagerProvider.aws_ssm:
|
||||||
|
@ -30,7 +30,7 @@ from metadata.ingestion.ometa.auth_provider import (
|
|||||||
)
|
)
|
||||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||||
from metadata.utils.secrets.aws_secrets_manager import AWSSecretsManager
|
from metadata.utils.secrets.aws_secrets_manager import AWSSecretsManager
|
||||||
from metadata.utils.secrets.local_secrets_manager import LocalSecretsManager
|
from metadata.utils.secrets.noop_secrets_manager import NoopSecretsManager
|
||||||
|
|
||||||
|
|
||||||
class OMetaSecretManagerTest(TestCase):
|
class OMetaSecretManagerTest(TestCase):
|
||||||
@ -55,7 +55,7 @@ class OMetaSecretManagerTest(TestCase):
|
|||||||
|
|
||||||
def test_ometa_with_local_secret_manager(self):
|
def test_ometa_with_local_secret_manager(self):
|
||||||
self._init_local_secret_manager()
|
self._init_local_secret_manager()
|
||||||
assert type(self.metadata.secrets_manager_client) is LocalSecretsManager
|
assert type(self.metadata.secrets_manager_client) is NoopSecretsManager
|
||||||
assert type(self.metadata._auth_provider) is NoOpAuthenticationProvider
|
assert type(self.metadata._auth_provider) is NoOpAuthenticationProvider
|
||||||
|
|
||||||
def test_ometa_with_local_secret_manager_with_google_auth(self):
|
def test_ometa_with_local_secret_manager_with_google_auth(self):
|
||||||
@ -64,7 +64,7 @@ class OMetaSecretManagerTest(TestCase):
|
|||||||
secretKey="/fake/path"
|
secretKey="/fake/path"
|
||||||
)
|
)
|
||||||
self._init_local_secret_manager()
|
self._init_local_secret_manager()
|
||||||
assert type(self.metadata.secrets_manager_client) is LocalSecretsManager
|
assert type(self.metadata.secrets_manager_client) is NoopSecretsManager
|
||||||
assert type(self.metadata._auth_provider) is GoogleAuthenticationProvider
|
assert type(self.metadata._auth_provider) is GoogleAuthenticationProvider
|
||||||
|
|
||||||
def test_ometa_with_aws_secret_manager(self):
|
def test_ometa_with_aws_secret_manager(self):
|
||||||
|
@ -33,14 +33,14 @@ from .test_secrets_manager import TestSecretsManager
|
|||||||
|
|
||||||
|
|
||||||
class TestLocalSecretsManager(TestSecretsManager.External):
|
class TestLocalSecretsManager(TestSecretsManager.External):
|
||||||
def test_local_manager_add_service_config_connection(self):
|
def test_noop_manager_add_service_config_connection(self):
|
||||||
local_manager = get_secrets_manager_from_om_connection(
|
noop_manager = get_secrets_manager_from_om_connection(
|
||||||
self.build_open_metadata_connection(SecretsManagerProvider.local), None
|
self.build_open_metadata_connection(SecretsManagerProvider.noop), None
|
||||||
)
|
)
|
||||||
expected_service_connection = self.service_connection
|
expected_service_connection = self.service_connection
|
||||||
|
|
||||||
actual_service_connection: ServiceConnection = (
|
actual_service_connection: ServiceConnection = (
|
||||||
local_manager.retrieve_service_connection(self.service, self.service_type)
|
noop_manager.retrieve_service_connection(self.service, self.service_type)
|
||||||
)
|
)
|
||||||
|
|
||||||
self.assertEqual(actual_service_connection, expected_service_connection)
|
self.assertEqual(actual_service_connection, expected_service_connection)
|
||||||
@ -48,41 +48,41 @@ class TestLocalSecretsManager(TestSecretsManager.External):
|
|||||||
expected_service_connection.__root__.config
|
expected_service_connection.__root__.config
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_local_manager_add_auth_provider_security_config(self):
|
def test_noop_manager_add_auth_provider_security_config(self):
|
||||||
local_manager = get_secrets_manager_from_om_connection(
|
noop_manager = get_secrets_manager_from_om_connection(
|
||||||
self.build_open_metadata_connection(SecretsManagerProvider.local), None
|
self.build_open_metadata_connection(SecretsManagerProvider.noop), None
|
||||||
)
|
)
|
||||||
actual_om_connection = deepcopy(self.om_connection)
|
actual_om_connection = deepcopy(self.om_connection)
|
||||||
actual_om_connection.securityConfig = self.auth_provider_config
|
actual_om_connection.securityConfig = self.auth_provider_config
|
||||||
|
|
||||||
local_manager.add_auth_provider_security_config(actual_om_connection)
|
noop_manager.add_auth_provider_security_config(actual_om_connection)
|
||||||
|
|
||||||
self.assertEqual(self.auth_provider_config, actual_om_connection.securityConfig)
|
self.assertEqual(self.auth_provider_config, actual_om_connection.securityConfig)
|
||||||
assert id(self.auth_provider_config) == id(actual_om_connection.securityConfig)
|
assert id(self.auth_provider_config) == id(actual_om_connection.securityConfig)
|
||||||
|
|
||||||
def test_local_manager_retrieve_dbt_source_config(self):
|
def test_noop_manager_retrieve_dbt_source_config(self):
|
||||||
local_manager = get_secrets_manager_from_om_connection(
|
noop_manager = get_secrets_manager_from_om_connection(
|
||||||
self.build_open_metadata_connection(SecretsManagerProvider.local), None
|
self.build_open_metadata_connection(SecretsManagerProvider.noop), None
|
||||||
)
|
)
|
||||||
source_config = SourceConfig()
|
source_config = SourceConfig()
|
||||||
source_config.config = DatabaseServiceMetadataPipeline(
|
source_config.config = DatabaseServiceMetadataPipeline(
|
||||||
dbtConfigSource=self.dbt_source_config
|
dbtConfigSource=self.dbt_source_config
|
||||||
)
|
)
|
||||||
|
|
||||||
actual_dbt_source_config = local_manager.retrieve_dbt_source_config(
|
actual_dbt_source_config = noop_manager.retrieve_dbt_source_config(
|
||||||
source_config, "test-pipeline"
|
source_config, "test-pipeline"
|
||||||
)
|
)
|
||||||
|
|
||||||
self.assertEqual(self.dbt_source_config.dict(), actual_dbt_source_config)
|
self.assertEqual(self.dbt_source_config.dict(), actual_dbt_source_config)
|
||||||
|
|
||||||
def test_local_manager_retrieve_temp_service_test_connection(self):
|
def test_noop_manager_retrieve_temp_service_test_connection(self):
|
||||||
local_manager = get_secrets_manager_from_om_connection(
|
noop_manager = get_secrets_manager_from_om_connection(
|
||||||
self.build_open_metadata_connection(SecretsManagerProvider.local), None
|
self.build_open_metadata_connection(SecretsManagerProvider.noop), None
|
||||||
)
|
)
|
||||||
expected_service_connection = self.service.connection
|
expected_service_connection = self.service.connection
|
||||||
|
|
||||||
actual_service_connection: DatabaseConnection = (
|
actual_service_connection: DatabaseConnection = (
|
||||||
local_manager.retrieve_temp_service_test_connection(
|
noop_manager.retrieve_temp_service_test_connection(
|
||||||
self.service.connection, "Database"
|
self.service.connection, "Database"
|
||||||
)
|
)
|
||||||
)
|
)
|
@ -39,7 +39,7 @@ class TestSecretsManagerFactory(TestCase):
|
|||||||
with self.assertRaises(NotImplementedError) as not_implemented_error:
|
with self.assertRaises(NotImplementedError) as not_implemented_error:
|
||||||
om_connection: OpenMetadataConnection = (
|
om_connection: OpenMetadataConnection = (
|
||||||
TestSecretsManager.External.build_open_metadata_connection(
|
TestSecretsManager.External.build_open_metadata_connection(
|
||||||
SecretsManagerProvider.local
|
SecretsManagerProvider.noop
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
om_connection.secretsManagerProvider = "aws"
|
om_connection.secretsManagerProvider = "aws"
|
||||||
@ -51,7 +51,7 @@ class TestSecretsManagerFactory(TestCase):
|
|||||||
def test_get_none_secret_manager(self):
|
def test_get_none_secret_manager(self):
|
||||||
om_connection: OpenMetadataConnection = (
|
om_connection: OpenMetadataConnection = (
|
||||||
TestSecretsManager.External.build_open_metadata_connection(
|
TestSecretsManager.External.build_open_metadata_connection(
|
||||||
SecretsManagerProvider.local
|
SecretsManagerProvider.noop
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
om_connection.secretsManagerProvider = None
|
om_connection.secretsManagerProvider = None
|
||||||
|
@ -5,6 +5,6 @@
|
|||||||
"description": "OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager providers as the ones configured on the OpenMetadata server.",
|
"description": "OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager providers as the ones configured on the OpenMetadata server.",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"javaType": "org.openmetadata.catalog.services.connections.metadata.SecretsManagerProvider",
|
"javaType": "org.openmetadata.catalog.services.connections.metadata.SecretsManagerProvider",
|
||||||
"enum": ["local", "aws", "aws-ssm"],
|
"enum": ["noop", "aws", "aws-ssm"],
|
||||||
"additionalProperties": false
|
"additionalProperties": false
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user