From cf2cb6d531a75a7f5086dac0828fa8d9c34aec67 Mon Sep 17 00:00:00 2001 From: Nahuel Date: Tue, 9 Aug 2022 09:00:43 +0200 Subject: [PATCH] Fix#6517: Add clusterName property to the application config yaml (#6610) * Add cluster name in the app configuration and start using it to create secrets id * Update secret manager client in openmetadata for using default auth provider * Add missing property in test config file --- .../catalog/CatalogApplication.java | 3 +- .../catalog/CatalogApplicationConfig.java | 3 + .../IngestionPipelineResource.java | 10 ++-- .../catalog/secrets/AWSSecretsManager.java | 13 +++-- .../catalog/secrets/LocalSecretsManager.java | 9 +-- .../catalog/secrets/SecretsManager.java | 28 +++++++--- .../secrets/SecretsManagerFactory.java | 6 +- .../metadata/openMetadataConnection.json | 5 ++ .../secrets/AWSSecretsManagerTest.java | 7 +-- .../secrets/LocalSecretsManagerTest.java | 2 +- .../secrets/SecretsManagerFactoryTest.java | 10 ++-- .../resources/openmetadata-secure-test.yaml | 2 + conf/openmetadata.yaml | 2 + ingestion/airflow.cfg | 3 +- ingestion/examples/airflow/airflow.cfg | 3 +- .../src/metadata/ingestion/ometa/ometa_api.py | 2 +- .../src/metadata/utils/secrets_manager.py | 56 +++++++++++++------ .../metadata/utils/test_secrets_manager.py | 41 ++++++++++++-- .../ingestion/credentials_builder.py | 22 ++++---- .../metadata/openMetadataConnection.json | 5 ++ 20 files changed, 161 insertions(+), 71 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java index 2f08f580caa..cc5a4ee55f8 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java @@ -96,7 +96,8 @@ public class CatalogApplication extends Application { jdbi.setTimingCollector(new MicrometerJdbiTimingCollector()); final SecretsManager secretsManager = - SecretsManagerFactory.createSecretsManager(catalogConfig.getSecretsManagerConfiguration()); + SecretsManagerFactory.createSecretsManager( + catalogConfig.getSecretsManagerConfiguration(), catalogConfig.getClusterName()); secretsManager.encryptAirflowConnection(catalogConfig.getAirflowConfiguration()); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplicationConfig.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplicationConfig.java index f66ed901bf2..422275f09f1 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplicationConfig.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplicationConfig.java @@ -92,6 +92,9 @@ public class CatalogApplicationConfig extends Configuration { @JsonProperty("secretsManagerConfiguration") private SecretsManagerConfiguration secretsManagerConfiguration; + @JsonProperty("clusterName") + private String clusterName; + @Override public String toString() { return "catalogConfig{" diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java index febcf1ecdda..df59ffdd460 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java @@ -53,7 +53,6 @@ import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; import org.openmetadata.catalog.CatalogApplicationConfig; import org.openmetadata.catalog.Entity; -import org.openmetadata.catalog.airflow.AirflowConfiguration; import org.openmetadata.catalog.airflow.AirflowRESTClient; import org.openmetadata.catalog.api.services.ingestionPipelines.CreateIngestionPipeline; import org.openmetadata.catalog.api.services.ingestionPipelines.TestServiceConnection; @@ -81,7 +80,7 @@ import org.openmetadata.catalog.util.ResultList; public class IngestionPipelineResource extends EntityResource { public static final String COLLECTION_PATH = "v1/services/ingestionPipelines/"; private PipelineServiceClient pipelineServiceClient; - private AirflowConfiguration airflowConfiguration; + private CatalogApplicationConfig catalogApplicationConfig; private final SecretsManager secretsManager; @Override @@ -97,8 +96,8 @@ public class IngestionPipelineResource extends EntityResource createConnectionConfigClass(String connectionType, String connectionPackage) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManagerFactory.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManagerFactory.java index c204513dbaf..1483d4d27e5 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManagerFactory.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManagerFactory.java @@ -4,16 +4,16 @@ import org.openmetadata.catalog.services.connections.metadata.OpenMetadataServer public class SecretsManagerFactory { - public static SecretsManager createSecretsManager(SecretsManagerConfiguration config) { + public static SecretsManager createSecretsManager(SecretsManagerConfiguration config, String clusterName) { SecretsManagerProvider secretManager = config != null && config.getSecretsManager() != null ? config.getSecretsManager() : SecretsManagerConfiguration.DEFAULT_SECRET_MANAGER; switch (secretManager) { case LOCAL: - return LocalSecretsManager.getInstance(); + return LocalSecretsManager.getInstance(clusterName); case AWS: - return AWSSecretsManager.getInstance(config); + return AWSSecretsManager.getInstance(config, clusterName); default: throw new IllegalArgumentException("Not implemented secret manager store: " + secretManager); } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json index 4ca99d53a58..e759d2a8ef8 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json @@ -14,6 +14,11 @@ } }, "properties": { + "clusterName": { + "description": "Cluster name to differentiate OpenMetadata Server instance", + "type": "string", + "default": "openmetadata" + }, "type": { "description": "Service Type", "$ref": "#/definitions/openmetadataType", diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java index 2751158928a..8de3e550883 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java @@ -51,12 +51,11 @@ import software.amazon.awssdk.services.secretsmanager.model.UpdateSecretRequest; public class AWSSecretsManagerTest { private static final String AUTH_PROVIDER_SECRET_ID_SUFFIX = "auth-provider"; - private static final boolean ENCRYPT = true; private static final boolean DECRYPT = false; private static final String EXPECTED_CONNECTION_JSON = "{\"type\":\"Mysql\",\"scheme\":\"mysql+pymysql\",\"password\":\"openmetadata-test\",\"supportsMetadataExtraction\":true,\"supportsProfiler\":true}"; - private static final String EXPECTED_SECRET_ID = "openmetadata-database-mysql-test"; + private static final String EXPECTED_SECRET_ID = "/openmetadata/service/database/mysql/test"; @Mock private SecretsManagerClient secretsManagerClient; @@ -70,7 +69,7 @@ public class AWSSecretsManagerTest { parameters.put("secretAccessKey", "654321"); SecretsManagerConfiguration config = new SecretsManagerConfiguration(); config.setParameters(parameters); - secretsManager = AWSSecretsManager.getInstance(config); + secretsManager = AWSSecretsManager.getInstance(config, "openmetadata"); secretsManager.setSecretsClient(secretsManagerClient); reset(secretsManagerClient); } @@ -136,7 +135,7 @@ public class AWSSecretsManagerTest { OpenMetadataServerConnection.AuthProvider authProvider, AuthConfiguration authConfig) throws JsonProcessingException { - String expectedSecretId = String.format("openmetadata-%s-%s", AUTH_PROVIDER_SECRET_ID_SUFFIX, authProvider); + String expectedSecretId = String.format("/openmetadata/%s/%s", AUTH_PROVIDER_SECRET_ID_SUFFIX, authProvider); AirflowConfiguration airflowConfiguration = ConfigurationFixtures.buildAirflowConfig(authProvider); airflowConfiguration.setAuthConfig(authConfig); AirflowConfiguration expectedAirflowConfiguration = ConfigurationFixtures.buildAirflowConfig(authProvider); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java index a8d05f664cf..334c66b006f 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java @@ -47,7 +47,7 @@ public class LocalSecretsManagerTest { @BeforeAll static void setUp() { - secretsManager = LocalSecretsManager.getInstance(); + secretsManager = LocalSecretsManager.getInstance("openmetadata"); Fernet fernet = Mockito.mock(Fernet.class); lenient().when(fernet.decrypt(anyString())).thenReturn(DECRYPTED_VALUE); lenient().when(fernet.encrypt(anyString())).thenReturn(ENCRYPTED_VALUE); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java index 0c03ec2704c..53d8fac5fd2 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java @@ -11,6 +11,8 @@ public class SecretsManagerFactoryTest { private SecretsManagerConfiguration config; + private static final String CLUSTER_NAME = "openmetadata"; + @BeforeEach void setUp() { config = new SecretsManagerConfiguration(); @@ -19,18 +21,18 @@ public class SecretsManagerFactoryTest { @Test void testDefaultIsCreatedIfNullConfig() { - assertTrue(SecretsManagerFactory.createSecretsManager(config) instanceof LocalSecretsManager); + assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof LocalSecretsManager); } @Test void testDefaultIsCreatedIfMissingSecretManager() { - assertTrue(SecretsManagerFactory.createSecretsManager(config) instanceof LocalSecretsManager); + assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof LocalSecretsManager); } @Test void testIsCreatedIfLocalSecretsManager() { config.setSecretsManager(SecretsManagerProvider.LOCAL); - assertTrue(SecretsManagerFactory.createSecretsManager(config) instanceof LocalSecretsManager); + assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof LocalSecretsManager); } @Test @@ -39,6 +41,6 @@ public class SecretsManagerFactoryTest { config.getParameters().put("region", "eu-west-1"); config.getParameters().put("accessKeyId", "123456"); config.getParameters().put("secretAccessKey", "654321"); - assertTrue(SecretsManagerFactory.createSecretsManager(config) instanceof AWSSecretsManager); + assertTrue(SecretsManagerFactory.createSecretsManager(config, CLUSTER_NAME) instanceof AWSSecretsManager); } } diff --git a/catalog-rest-service/src/test/resources/openmetadata-secure-test.yaml b/catalog-rest-service/src/test/resources/openmetadata-secure-test.yaml index d2e62a8a28f..c6982eb81fc 100644 --- a/catalog-rest-service/src/test/resources/openmetadata-secure-test.yaml +++ b/catalog-rest-service/src/test/resources/openmetadata-secure-test.yaml @@ -9,6 +9,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +clusterName: openmetadata + swagger: resourcePackage: org.openmetadata.catalog.webservice.resources diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index 9ce4a2eba4e..e279a75c208 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -9,6 +9,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +clusterName: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} + swagger: resourcePackage: org.openmetadata.catalog.resources diff --git a/ingestion/airflow.cfg b/ingestion/airflow.cfg index fafb1cf9119..060e5ffe37a 100644 --- a/ingestion/airflow.cfg +++ b/ingestion/airflow.cfg @@ -426,7 +426,8 @@ access_control_allow_origin = [openmetadata_airflow_apis] dag_generated_configs = /airflow/dag_generated_configs -[openmetadata_secrets_manager] +# this section is optional, the default auth provider for the secrets' manager service will be used if it is not set +# [openmetadata_secrets_manager] # aws_access_key_id = # aws_secret_access_key = # aws_region = diff --git a/ingestion/examples/airflow/airflow.cfg b/ingestion/examples/airflow/airflow.cfg index 16927d2a17a..4f651662ab3 100644 --- a/ingestion/examples/airflow/airflow.cfg +++ b/ingestion/examples/airflow/airflow.cfg @@ -423,7 +423,8 @@ auth_provider_type = no-auth dag_runner_template = /airflow/dag_templates/dag_runner.j2 dag_generated_configs = /airflow/dag_generated_configs -[openmetadata_secrets_manager] +# this section is optional, the default auth provider for the secrets' manager service will be used if it is not set +# [openmetadata_secrets_manager] # aws_access_key_id = # aws_secret_access_key = # aws_region = diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 5479c4496eb..7162db97b3f 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -168,7 +168,7 @@ class OpenMetadata( # Load the secrets' manager client self.secrets_manager_client = get_secrets_manager( - config.secretsManagerProvider, config.secretsManagerCredentials + config, config.secretsManagerCredentials ) # Load auth provider config from Secret Manager if necessary diff --git a/ingestion/src/metadata/utils/secrets_manager.py b/ingestion/src/metadata/utils/secrets_manager.py index 30fab119bed..4885aa91f74 100644 --- a/ingestion/src/metadata/utils/secrets_manager.py +++ b/ingestion/src/metadata/utils/secrets_manager.py @@ -84,6 +84,11 @@ class SecretsManager(metaclass=Singleton): providers. """ + cluster_prefix: str + + def __init__(self, cluster_prefix: str): + self.cluster_prefix = cluster_prefix + @abstractmethod def retrieve_service_connection( self, @@ -105,8 +110,15 @@ class SecretsManager(metaclass=Singleton): """ pass - @staticmethod - def build_secret_id(*args: str) -> str: + @property + def secret_id_separator(self) -> str: + return "/" + + @property + def starts_with_separator(self) -> bool: + return True + + def build_secret_id(self, *args: str) -> str: """ Returns a secret_id used by the secrets' manager providers for retrieving a secret. For example: @@ -114,8 +126,8 @@ class SecretsManager(metaclass=Singleton): :param args: sorted parameters for building the secret_id :return: the secret_id """ - secret_suffix = "-".join([arg.lower() for arg in args]) - return f"openmetadata-{secret_suffix}" + secret_id = self.secret_id_separator.join([arg.lower() for arg in args]) + return f"{self.secret_id_separator if self.starts_with_separator else ''}{self.cluster_prefix}{self.secret_id_separator}{secret_id}" @staticmethod def get_service_connection_class(service_type: str) -> object: @@ -181,13 +193,19 @@ class LocalSecretsManager(SecretsManager): class AWSSecretsManager(SecretsManager): - def __init__(self, credentials: AWSCredentials): - session = boto3.Session( - aws_access_key_id=credentials.awsAccessKeyId, - aws_secret_access_key=credentials.awsSecretAccessKey.get_secret_value(), - region_name=credentials.awsRegion, - ) - self.secretsmanager_client = session.client("secretsmanager") + def __init__(self, credentials: AWSCredentials, cluster_prefix: str): + super().__init__(cluster_prefix) + # initialize the secret client depending on the SecretsManagerConfiguration passed + if credentials: + session = boto3.Session( + aws_access_key_id=credentials.awsAccessKeyId, + aws_secret_access_key=credentials.awsSecretAccessKey.get_secret_value(), + region_name=credentials.awsRegion, + ) + self.secretsmanager_client = session.client("secretsmanager") + else: + # initialized with the credentials loaded from running machine + self.secretsmanager_client = boto3.client("secretsmanager") def retrieve_service_connection( self, @@ -197,7 +215,7 @@ class AWSSecretsManager(SecretsManager): service_connection_type = service.serviceType.value service_name = service.name.__root__ secret_id = self.build_secret_id( - service_type, service_connection_type, service_name + "service", service_type, service_connection_type, service_name ) connection_class = self.get_connection_class( service_type, service_connection_type @@ -251,12 +269,14 @@ class AWSSecretsManager(SecretsManager): def get_secrets_manager( - secret_manager: SecretsManagerProvider, + open_metadata_config: OpenMetadataConnection, credentials: Optional[Union[AWSCredentials]] = None, ) -> SecretsManager: - if secret_manager == SecretsManagerProvider.local: - return LocalSecretsManager() - elif secret_manager == SecretsManagerProvider.aws: - return AWSSecretsManager(credentials) + if open_metadata_config.secretsManagerProvider == SecretsManagerProvider.local: + return LocalSecretsManager(open_metadata_config.clusterName) + elif open_metadata_config.secretsManagerProvider == SecretsManagerProvider.aws: + return AWSSecretsManager(credentials, open_metadata_config.clusterName) else: - raise NotImplementedError(f"[{secret_manager}] is not implemented.") + raise NotImplementedError( + f"[{open_metadata_config.secretsManagerProvider}] is not implemented." + ) diff --git a/ingestion/tests/unit/metadata/utils/test_secrets_manager.py b/ingestion/tests/unit/metadata/utils/test_secrets_manager.py index 6e37b57363f..382fe5e74de 100644 --- a/ingestion/tests/unit/metadata/utils/test_secrets_manager.py +++ b/ingestion/tests/unit/metadata/utils/test_secrets_manager.py @@ -41,6 +41,7 @@ from metadata.generated.schema.security.client.googleSSOClientConfig import ( from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials from metadata.utils.secrets_manager import ( AUTH_PROVIDER_MAPPING, + SecretsManager, Singleton, get_secrets_manager, ) @@ -80,7 +81,9 @@ class TestSecretsManager(TestCase): Singleton.clear_all() def test_local_manager_add_service_config_connection(self): - local_manager = get_secrets_manager(SecretsManagerProvider.local, None) + local_manager = get_secrets_manager( + self._build_open_metadata_connection(SecretsManagerProvider.local), None + ) expected_service_connection = self.service_connection actual_service_connection: ServiceConnection = ( @@ -93,7 +96,9 @@ class TestSecretsManager(TestCase): ) def test_local_manager_add_auth_provider_security_config(self): - local_manager = get_secrets_manager(SecretsManagerProvider.local, None) + local_manager = get_secrets_manager( + self._build_open_metadata_connection(SecretsManagerProvider.local), None + ) actual_om_connection = deepcopy(self.om_connection) actual_om_connection.securityConfig = self.auth_provider_config @@ -113,6 +118,12 @@ class TestSecretsManager(TestCase): aws_manager.retrieve_service_connection(self.service, self.service_type) ) + expected_call = { + "SecretId": "/openmetadata/service/database/mysql/test_service" + } + aws_manager.secretsmanager_client.get_secret_value.assert_called_once_with( + **expected_call + ) self.assertEqual(expected_service_connection, actual_service_connection) assert id(actual_service_connection.__root__.config) != id( expected_service_connection.__root__.config @@ -138,6 +149,10 @@ class TestSecretsManager(TestCase): aws_manager.add_auth_provider_security_config(actual_om_connection) + expected_call = {"SecretId": "/openmetadata/auth-provider/google"} + aws_manager.secretsmanager_client.get_secret_value.assert_called_once_with( + **expected_call + ) self.assertEqual(self.auth_provider_config, actual_om_connection.securityConfig) assert id(self.auth_provider_config) != id(actual_om_connection.securityConfig) @@ -155,7 +170,11 @@ class TestSecretsManager(TestCase): def test_get_not_implemented_secret_manager(self): with self.assertRaises(NotImplementedError) as not_implemented_error: - get_secrets_manager("any") + om_connection: OpenMetadataConnection = ( + self._build_open_metadata_connection(SecretsManagerProvider.local) + ) + om_connection.secretsManagerProvider = "aws" + get_secrets_manager(om_connection) self.assertEqual( "[any] is not implemented.", not_implemented_error.exception ) @@ -167,10 +186,12 @@ class TestSecretsManager(TestCase): for auth_provider in auth_provider_with_client: assert AUTH_PROVIDER_MAPPING.get(auth_provider, None) is not None - def _build_secret_manager(self, mocked_boto3: Mock, expected_json: Dict[str, Any]): + def _build_secret_manager( + self, mocked_boto3: Mock, expected_json: Dict[str, Any] + ) -> SecretsManager: self._init_boto3_mock(mocked_boto3, expected_json) aws_manager = get_secrets_manager( - SecretsManagerProvider.aws, + self._build_open_metadata_connection(SecretsManagerProvider.aws), AWSCredentials( awsAccessKeyId="fake_key", awsSecretAccessKey="fake_access", @@ -179,6 +200,16 @@ class TestSecretsManager(TestCase): ) return aws_manager + @staticmethod + def _build_open_metadata_connection( + secret_manager_provider: SecretsManagerProvider, + ) -> OpenMetadataConnection: + return OpenMetadataConnection( + secretsManagerProvider=secret_manager_provider, + clusterName="openmetadata", + hostPort="http://localhost:8585/api", + ) + @staticmethod def _init_boto3_mock(boto3_mock: Mock, client_return: Dict[str, Any]): mocked_client = Mock() diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/credentials_builder.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/credentials_builder.py index bcc05b97f0e..52993ad07eb 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/credentials_builder.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/credentials_builder.py @@ -9,16 +9,18 @@ from metadata.utils.secrets_manager import SECRET_MANAGER_AIRFLOW_CONF def build_aws_credentials(): - credentials = AWSCredentials( - awsRegion=conf.get(SECRET_MANAGER_AIRFLOW_CONF, "aws_region", fallback="") - ) - credentials.awsAccessKeyId = conf.get( - SECRET_MANAGER_AIRFLOW_CONF, "aws_access_key_id", fallback="" - ) - credentials.awsSecretAccessKey = SecretStr( - conf.get(SECRET_MANAGER_AIRFLOW_CONF, "aws_secret_access_key", fallback="") - ) - return credentials + if conf.has_section(SECRET_MANAGER_AIRFLOW_CONF): + credentials = AWSCredentials( + awsRegion=conf.get(SECRET_MANAGER_AIRFLOW_CONF, "aws_region", fallback="") + ) + credentials.awsAccessKeyId = conf.get( + SECRET_MANAGER_AIRFLOW_CONF, "aws_access_key_id", fallback="" + ) + credentials.awsSecretAccessKey = SecretStr( + conf.get(SECRET_MANAGER_AIRFLOW_CONF, "aws_secret_access_key", fallback="") + ) + return credentials + return None def build_secrets_manager_credentials(secrets_manager: SecretsManagerProvider): diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json b/openmetadata-core/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json index 4ca99d53a58..e759d2a8ef8 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/connections/metadata/openMetadataConnection.json @@ -14,6 +14,11 @@ } }, "properties": { + "clusterName": { + "description": "Cluster name to differentiate OpenMetadata Server instance", + "type": "string", + "default": "openmetadata" + }, "type": { "description": "Service Type", "$ref": "#/definitions/openmetadataType",