mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-27 07:28:30 +00:00
FIX CL-548 - Encrypt JWT Token w/ Secrets Manager (#16861)
* encrypt jwt * encrypt jwt * Handle token revokes with SM * #10192 - Validate both include and excludes filters * #10192 - Validate both include and excludes filters * docs
This commit is contained in:
Pere Miquel Brull
GitHub
parent
c3cc1a5e5b
commit
8d739563f2
@ -20,6 +20,7 @@ import com.sun.codemodel.JDefinedClass;
|
||||
import com.sun.codemodel.JFieldVar;
|
||||
import com.sun.codemodel.JMethod;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.TreeMap;
|
||||
import org.jsonschema2pojo.AbstractAnnotator;
|
||||
|
||||
/** Add {@link ExposedField} annotation to generated Java classes */
|
||||
@ -59,8 +60,13 @@ public class ExposedAnnotator extends AbstractAnnotator {
|
||||
Field outerClassField = JMethod.class.getDeclaredField("outer");
|
||||
outerClassField.setAccessible(true);
|
||||
JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod);
|
||||
if (outerClass.fields().containsKey(propertyName)
|
||||
&& outerClass.fields().get(propertyName).annotations().stream()
|
||||
|
||||
TreeMap<String, JFieldVar> insensitiveFieldsMap =
|
||||
new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
|
||||
insensitiveFieldsMap.putAll(outerClass.fields());
|
||||
|
||||
if (insensitiveFieldsMap.containsKey(propertyName)
|
||||
&& insensitiveFieldsMap.get(propertyName).annotations().stream()
|
||||
.anyMatch(
|
||||
annotation ->
|
||||
ExposedField.class.getName().equals(getAnnotationClassName(annotation)))) {
|
||||
|
||||
@ -20,6 +20,7 @@ import com.sun.codemodel.JDefinedClass;
|
||||
import com.sun.codemodel.JFieldVar;
|
||||
import com.sun.codemodel.JMethod;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.TreeMap;
|
||||
import org.jsonschema2pojo.AbstractAnnotator;
|
||||
|
||||
/** Add {@link MaskedField} annotation to generated Java classes */
|
||||
@ -59,8 +60,13 @@ public class MaskedAnnotator extends AbstractAnnotator {
|
||||
Field outerClassField = JMethod.class.getDeclaredField("outer");
|
||||
outerClassField.setAccessible(true);
|
||||
JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod);
|
||||
if (outerClass.fields().containsKey(propertyName)
|
||||
&& outerClass.fields().get(propertyName).annotations().stream()
|
||||
|
||||
TreeMap<String, JFieldVar> insensitiveFieldsMap =
|
||||
new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
|
||||
insensitiveFieldsMap.putAll(outerClass.fields());
|
||||
|
||||
if (insensitiveFieldsMap.containsKey(propertyName)
|
||||
&& insensitiveFieldsMap.get(propertyName).annotations().stream()
|
||||
.anyMatch(
|
||||
annotation ->
|
||||
MaskedField.class.getName().equals(getAnnotationClassName(annotation)))) {
|
||||
|
||||
@ -20,6 +20,7 @@ import com.sun.codemodel.JDefinedClass;
|
||||
import com.sun.codemodel.JFieldVar;
|
||||
import com.sun.codemodel.JMethod;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.TreeMap;
|
||||
import org.jsonschema2pojo.AbstractAnnotator;
|
||||
|
||||
/** Add {@link PasswordField} annotation to generated Java classes */
|
||||
@ -60,8 +61,13 @@ public class PasswordAnnotator extends AbstractAnnotator {
|
||||
Field outerClassField = JMethod.class.getDeclaredField("outer");
|
||||
outerClassField.setAccessible(true);
|
||||
JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod);
|
||||
if (outerClass.fields().containsKey(propertyName)
|
||||
&& outerClass.fields().get(propertyName).annotations().stream()
|
||||
|
||||
TreeMap<String, JFieldVar> insensitiveFieldsMap =
|
||||
new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
|
||||
insensitiveFieldsMap.putAll(outerClass.fields());
|
||||
|
||||
if (insensitiveFieldsMap.containsKey(propertyName)
|
||||
&& insensitiveFieldsMap.get(propertyName).annotations().stream()
|
||||
.anyMatch(
|
||||
annotation ->
|
||||
PasswordField.class.getName().equals(getAnnotationClassName(annotation)))) {
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import pkgutil
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
@ -46,6 +46,32 @@ class DeployDagException(Exception):
|
||||
"""
|
||||
|
||||
|
||||
def dump_with_safe_jwt(ingestion_pipeline: IngestionPipeline) -> str:
|
||||
"""
|
||||
Get the dump of the IngestionPipeline but keeping the JWT token masked.
|
||||
|
||||
Since Pydantic V2, we had to handle the serialization of secrets when dumping
|
||||
the data at model level, since we don't have anymore fine-grained control of
|
||||
it at runtime as we did with V1.
|
||||
|
||||
This means that even if the JWT token is a secret, a model_dump or model_json_dump
|
||||
will automatically show the secret value - picking it from the Secrets Manager if enabled.
|
||||
|
||||
With this workaround, we're dumping the model to JSON and then replacing the JWT token
|
||||
with the secret, so that if we are using a Secret Manager, the resulting file
|
||||
will have the secret ID `secret:/super/secret` instead of the actual value.
|
||||
|
||||
Then, the client will pick up the right secret when the workflow is triggered.
|
||||
"""
|
||||
pipeline_json = ingestion_pipeline.model_dump(mode="json", exclude_defaults=False)
|
||||
pipeline_json["openMetadataServerConnection"]["securityConfig"][
|
||||
"jwtToken"
|
||||
] = ingestion_pipeline.openMetadataServerConnection.securityConfig.jwtToken.get_secret_value(
|
||||
skip_secret_manager=True
|
||||
)
|
||||
return json.dumps(pipeline_json, ensure_ascii=True)
|
||||
|
||||
|
||||
class DagDeployer:
|
||||
"""
|
||||
Helper class to store DAG config
|
||||
@ -74,9 +100,7 @@ class DagDeployer:
|
||||
|
||||
logger.info(f"Saving file to {dag_config_file_path}")
|
||||
with open(dag_config_file_path, "w") as outfile:
|
||||
outfile.write(
|
||||
self.ingestion_pipeline.model_dump_json(exclude_defaults=False)
|
||||
)
|
||||
outfile.write(dump_with_safe_jwt(self.ingestion_pipeline))
|
||||
|
||||
return {"workflow_config_file": str(dag_config_file_path)}
|
||||
|
||||
|
||||
@ -0,0 +1,67 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Test run automations
|
||||
"""
|
||||
from metadata.generated.schema.entity.teams.user import User
|
||||
from metadata.ingestion.api.parser import parse_automation_workflow_gracefully
|
||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||
from metadata.utils.secrets.secrets_manager_factory import SecretsManagerFactory
|
||||
|
||||
JSON_REQUEST = {
|
||||
"id": "6192f18e-5f0c-42e3-8568-0c3ce6e07ac5",
|
||||
"name": "test-connection-Mysql-Prmeqo2t",
|
||||
"fullyQualifiedName": "test-connection-Mysql-Prmeqo2t",
|
||||
"workflowType": "TEST_CONNECTION",
|
||||
"request": {
|
||||
"connection": {
|
||||
"config": {
|
||||
"type": "Mysql",
|
||||
"scheme": "mysql+pymysql",
|
||||
"authType": {"password": "fernet:xyz"},
|
||||
"hostPort": "mysql:3306",
|
||||
"username": "openmetadata_user",
|
||||
"databaseSchema": "openmetadata_db",
|
||||
}
|
||||
},
|
||||
"serviceName": "mysql",
|
||||
"serviceType": "Database",
|
||||
"connectionType": "Mysql",
|
||||
},
|
||||
"openMetadataServerConnection": {
|
||||
"clusterName": "openmetadata",
|
||||
"type": "OpenMetadata",
|
||||
"hostPort": "http://localhost:8585/api",
|
||||
"authProvider": "openmetadata",
|
||||
"securityConfig": {
|
||||
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_run_automation():
|
||||
"""Follow the flow in run_automation and validate we're getting the right JWT"""
|
||||
|
||||
automation_workflow = parse_automation_workflow_gracefully(config_dict=JSON_REQUEST)
|
||||
|
||||
# we need to instantiate the secret manager in case secrets are passed
|
||||
SecretsManagerFactory(
|
||||
automation_workflow.openMetadataServerConnection.secretsManagerProvider,
|
||||
automation_workflow.openMetadataServerConnection.secretsManagerLoader,
|
||||
)
|
||||
|
||||
metadata = OpenMetadata(config=automation_workflow.openMetadataServerConnection)
|
||||
|
||||
# Check the token is valid
|
||||
admin: User = metadata.get_by_name(entity=User, fqn="admin")
|
||||
assert admin
|
||||
@ -0,0 +1,72 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Test Deploy
|
||||
"""
|
||||
import os
|
||||
import uuid
|
||||
from unittest.mock import patch
|
||||
|
||||
from openmetadata_managed_apis.operations.deploy import dump_with_safe_jwt
|
||||
|
||||
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
|
||||
AuthProvider,
|
||||
OpenMetadataConnection,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
|
||||
AirflowConfig,
|
||||
IngestionPipeline,
|
||||
PipelineType,
|
||||
)
|
||||
from metadata.generated.schema.metadataIngestion.workflow import SourceConfig
|
||||
from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
|
||||
OpenMetadataJWTClientConfig,
|
||||
)
|
||||
from metadata.generated.schema.security.secrets.secretsManagerClientLoader import (
|
||||
SecretsManagerClientLoader,
|
||||
)
|
||||
from metadata.generated.schema.security.secrets.secretsManagerProvider import (
|
||||
SecretsManagerProvider,
|
||||
)
|
||||
from metadata.generated.schema.type.basic import EntityName, Uuid
|
||||
from metadata.utils.secrets.aws_secrets_manager import AWSSecretsManager
|
||||
from metadata.utils.secrets.secrets_manager_factory import SecretsManagerFactory
|
||||
|
||||
SECRET_VALUE = "I am a secret value"
|
||||
INGESTION_PIPELINE = IngestionPipeline(
|
||||
id=Uuid(str(uuid.uuid4())),
|
||||
name=EntityName("ingestion-pipeline"),
|
||||
pipelineType=PipelineType.metadata,
|
||||
sourceConfig=SourceConfig(),
|
||||
airflowConfig=AirflowConfig(),
|
||||
openMetadataServerConnection=OpenMetadataConnection(
|
||||
hostPort="http://localhost:8585/api",
|
||||
authProvider=AuthProvider.openmetadata,
|
||||
securityConfig=OpenMetadataJWTClientConfig(jwtToken="secret:/super/secret"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"AWS_DEFAULT_REGION": "us-east-2"})
|
||||
def test_deploy_ingestion_pipeline():
|
||||
"""We can dump an ingestion pipeline to a file without exposing secrets"""
|
||||
# Instantiate the Secrets Manager
|
||||
SecretsManagerFactory.clear_all()
|
||||
with patch.object(AWSSecretsManager, "get_string_value", return_value=SECRET_VALUE):
|
||||
# Prep the singleton
|
||||
SecretsManagerFactory(
|
||||
SecretsManagerProvider.managed_aws,
|
||||
SecretsManagerClientLoader.noop,
|
||||
)
|
||||
# Now we'll try to dump the ingestion pipeline
|
||||
dumped = dump_with_safe_jwt(INGESTION_PIPELINE)
|
||||
|
||||
assert SECRET_VALUE not in dumped
|
||||
@ -113,28 +113,14 @@ We believe this update will bring greater consistency and clarity to our version
|
||||
|
||||
# Backward Incompatible Changes
|
||||
|
||||
## 1.4.0
|
||||
## 1.5.0
|
||||
|
||||
### Tooling
|
||||
|
||||
- **Metadata Backup & Restore**: The Metadata Backup/Recovery has been deprecated, and no further support will be provided. Users are advised to use database-native tools to back up data and store it in their object store for recovery.
|
||||
You can check the [docs](/deployment/backup-restore-metadata) for more information.
|
||||
- **Metadata Docker CLI**: For the past releases, we have been updating the documentation to point users to directly run the docker quickstart
|
||||
with the docker compose files in the release page ([docs](/quick-start/local-docker-deployment)). In this release, we're completely removing the support for `metadata docker`.
|
||||
- **bootstrap_storage.sh**: We have deprecated `bootstrap/bootstrap_storage.sh` and replaced it with `bootstrap/openmetadata-ops.sh`. The documentation has been updated accordingly.
|
||||
### Secrets Manager
|
||||
|
||||
- Starting with the release 1.5.0, the JWT Token for the bots will be sent to the Secrets Manager if you configured one.
|
||||
It won't appear anymore in your `dag_generated_configs` in Airflow.
|
||||
|
||||
### UI
|
||||
|
||||
- **Activity Feed**: The Activity Feed has been improved with new, updated cards that display critical information such as data quality test case updates, descriptions, tag updates, or asset removal.
|
||||
- **Lineage**: The Expand All button has been removed. A new Layers button is introduced in the bottom left corner. With the Layers button, you can add Column Level Lineage or Data Observability details to your Lineage view.
|
||||
- **View Definition**: View Definition is now renamed to Schema Definition
|
||||
- **Glossary**: Adding Glossary Term view has been improved. Now, we show glossary terms hierarchically, enabling a better understanding of how the terms are set up while labeling a table or dashboard.
|
||||
- **Classifications**: users can set it to be mutually exclusive **only** at the time of creation. Once created, you cannot change it back to mutually non-exclusive or vice versa.
|
||||
This is to prevent conflicts between adding multiple tags that belong to the same classification and later turning the mutually exclusive flag back to true.
|
||||
|
||||
### API
|
||||
|
||||
- **View Definition**: Table Schema's `ViewDefinition` is now renamed to `SchemaDefinition` to capture Tables' Create Schema.
|
||||
- **Bulk Import**: Bulk Import API now creates entities if they are not present during the import.
|
||||
- **Test Suites**: Table's `TestSuite` is migrated to an `EntityReference`. Previously it used to store entire payload of `TestSuite`.
|
||||
|
||||
@ -89,11 +89,18 @@ The default credential will look for credentials in:
|
||||
More info in [AWS SDK for Java](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) and
|
||||
[Boto3 Docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the secrets names which starts with
|
||||
`/openmetadata/...` in your AWS Secrets Manager console. The following image shows what it should look like:
|
||||
|
||||
@ -87,11 +87,18 @@ default credential will look for credentials in:
|
||||
More info in [AWS SDK for Java](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) and
|
||||
[Boto3 Docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the parameters names which starts with
|
||||
`/openmetadata/...` in your AWS Systems Manager Parameter Store console. The following image shows what it should look
|
||||
|
||||
@ -163,11 +163,18 @@ serviceAccount:
|
||||
|
||||
{% /note %}
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the parameters names which starts with
|
||||
`openmetadata-...` in your Key Vault console.
|
||||
|
||||
@ -22,4 +22,11 @@ This is our list of supported Secrets Manager implementations:
|
||||
href="/deployment/secrets-manager/supported-implementations/aws-ssm-parameter-store" %}
|
||||
AWS Systems Manager Parameter Store
|
||||
{% /inlineCallout %}
|
||||
{% inlineCallout
|
||||
color="violet-70"
|
||||
bold="Azure Key Vault"
|
||||
icon="vpn_key"
|
||||
href="/deployment/secrets-manager/supported-implementations/azure-key-vault" %}
|
||||
Azure Key Vault
|
||||
{% /inlineCallout %}
|
||||
{% /inlineCalloutContainer %}
|
||||
@ -89,11 +89,18 @@ The default credential will look for credentials in:
|
||||
More info in [AWS SDK for Java](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) and
|
||||
[Boto3 Docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the secrets names which starts with
|
||||
`/openmetadata/...` in your AWS Secrets Manager console. The following image shows what it should look like:
|
||||
|
||||
@ -87,11 +87,18 @@ default credential will look for credentials in:
|
||||
More info in [AWS SDK for Java](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) and
|
||||
[Boto3 Docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the parameters names which starts with
|
||||
`/openmetadata/...` in your AWS Systems Manager Parameter Store console. The following image shows what it should look
|
||||
|
||||
@ -163,11 +163,18 @@ serviceAccount:
|
||||
|
||||
{% /note %}
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the parameters names which starts with
|
||||
`openmetadata-...` in your Key Vault console.
|
||||
|
||||
@ -22,4 +22,11 @@ This is our list of supported Secrets Manager implementations:
|
||||
href="/deployment/secrets-manager/supported-implementations/aws-ssm-parameter-store" %}
|
||||
AWS Systems Manager Parameter Store
|
||||
{% /inlineCallout %}
|
||||
{% inlineCallout
|
||||
color="violet-70"
|
||||
bold="Azure Key Vault"
|
||||
icon="vpn_key"
|
||||
href="/deployment/secrets-manager/supported-implementations/azure-key-vault" %}
|
||||
Azure Key Vault
|
||||
{% /inlineCallout %}
|
||||
{% /inlineCalloutContainer %}
|
||||
@ -163,6 +163,56 @@ workflowConfig:
|
||||
# caCertificate: /local/path/to/certificate
|
||||
```
|
||||
|
||||
#### JWT Token with Secrets Manager
|
||||
|
||||
If you are using the [Secrets Manager](/deployment/secrets-manager), you can let the Ingestion client to pick up
|
||||
the JWT Token dynamically from the Secrets Manager at runtime. Let's show an example:
|
||||
|
||||
We have an OpenMetadata server running with the `managed-aws` Secrets Manager. Since we used the `OPENMETADATA_CLUSTER_NAME` env var
|
||||
as `test`, our `ingestion-bot` JWT Token is safely stored under the secret ID `
|
||||
/test/bot/ingestion-bot/config/jwttoken`.
|
||||
|
||||
Now, we can use the following workflow config to run the ingestion without having to pass the token, but just pointing to the secret itself:
|
||||
|
||||
```yaml
|
||||
workflowConfig:
|
||||
loggerLevel: INFO # DEBUG, INFO, WARNING or ERROR
|
||||
openMetadataServerConfig:
|
||||
hostPort: "http://localhost:8585/api"
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "secret:/test/bot/ingestion-bot/config/jwttoken"
|
||||
secretsManagerProvider: aws
|
||||
secretsManagerLoader: env
|
||||
```
|
||||
|
||||
Notice how:
|
||||
1. We specify the `secretsManagerProvider` pointing to `aws`, since that's the manager we are using.
|
||||
2. We set `secretsManagerLoader` as `env`. Since we're running this from our local, we'll let the AWS credentials to be
|
||||
loaded from the local env vars. (When running this using the UI, note that the generated workflows will have this
|
||||
value set as `airflow`!)
|
||||
3. We set the `jwtToken` value as `secret:/test/bot/ingestion-bot/config/jwttoken`, which tells the client that
|
||||
this value is a `secret` located under `/test/bot/ingestion-bot/config/jwttoken`.
|
||||
|
||||
|
||||
Those are our env vars:
|
||||
|
||||
```
|
||||
export AWS_ACCESS_KEY_ID=...
|
||||
export AWS_SECRET_ACCESS_KEY=...
|
||||
export AWS_DEFAULT_REGION=...
|
||||
```
|
||||
|
||||
And we can run this normally with `metadata ingest -c <path to yaml>`.
|
||||
|
||||
{% note %}
|
||||
|
||||
Note that **even if you are not using the Secrets Manager for the OpenMetadata Server**, you can still apply the same
|
||||
approach by storing the JWT token manually to the secrets manager, and let the Ingestion client pick it up
|
||||
from there automatically.
|
||||
|
||||
{% /note %}
|
||||
|
||||
## 3. (Optional) Ingestion Pipeline
|
||||
|
||||
Additionally, if you want to see your runs logged in the `Ingestions` tab of the connectors page in the UI as you would
|
||||
|
||||
@ -89,11 +89,18 @@ The default credential will look for credentials in:
|
||||
More info in [AWS SDK for Java](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) and
|
||||
[Boto3 Docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the secrets names which starts with
|
||||
`/openmetadata/...` in your AWS Secrets Manager console. The following image shows what it should look like:
|
||||
|
||||
@ -87,11 +87,18 @@ default credential will look for credentials in:
|
||||
More info in [AWS SDK for Java](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html) and
|
||||
[Boto3 Docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the parameters names which starts with
|
||||
`/openmetadata/...` in your AWS Systems Manager Parameter Store console. The following image shows what it should look
|
||||
|
||||
@ -163,11 +163,18 @@ serviceAccount:
|
||||
|
||||
{% /note %}
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the parameters names which starts with
|
||||
`openmetadata-...` in your Key Vault console.
|
||||
|
||||
@ -67,11 +67,18 @@ ADC will look for credentials in:
|
||||
|
||||
More info in [Set up Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc)
|
||||
|
||||
### 3. Restart both servers
|
||||
### 3. Migrate Secrets & restart both servers
|
||||
|
||||
After updating the configuration files, we are ready to restart both services. When the OM server starts, it will
|
||||
automatically detect that a Secrets Manager has been configured and will migrate all our sensitive data and remove it
|
||||
from our DB.
|
||||
After updating the configuration files, we are ready to migrate the secrets and restart both services.
|
||||
|
||||
In order to ensure that the current sensitive information is properly migrated to the Secrets Manager, you need to
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
./bootstrap/openmetadata-ops.sh migrate-secrets
|
||||
```
|
||||
|
||||
Make sure you are running it with the same environment variables required by the server.
|
||||
|
||||
If everything goes as planned, all the data would be displayed using the parameters names which starts with
|
||||
`/openmetadata/...` in your GCP Secret Manager console. The following image shows what it should look
|
||||
@ -22,4 +22,18 @@ This is our list of supported Secrets Manager implementations:
|
||||
href="/deployment/secrets-manager/supported-implementations/aws-ssm-parameter-store" %}
|
||||
AWS Systems Manager Parameter Store
|
||||
{% /inlineCallout %}
|
||||
{% inlineCallout
|
||||
color="violet-70"
|
||||
bold="Azure Key Vault"
|
||||
icon="vpn_key"
|
||||
href="/deployment/secrets-manager/supported-implementations/azure-key-vault" %}
|
||||
Azure Key Vault
|
||||
{% /inlineCallout %}
|
||||
{% inlineCallout
|
||||
color="violet-70"
|
||||
bold="GCP Secrets Manager"
|
||||
icon="vpn_key"
|
||||
href="/deployment/secrets-manager/supported-implementations/gcp-secret-manager" %}
|
||||
GCP Secrets Manager
|
||||
{% /inlineCallout %}
|
||||
{% /inlineCalloutContainer %}
|
||||
@ -25,6 +25,7 @@ import static org.openmetadata.schema.entity.teams.AuthenticationMechanism.AuthT
|
||||
import static org.openmetadata.schema.type.Include.ALL;
|
||||
import static org.openmetadata.service.exception.CatalogExceptionMessage.EMAIL_SENDING_ISSUE;
|
||||
import static org.openmetadata.service.jdbi3.UserRepository.AUTH_MECHANISM_FIELD;
|
||||
import static org.openmetadata.service.secrets.ExternalSecretsManager.NULL_SECRET_STRING;
|
||||
import static org.openmetadata.service.security.jwt.JWTTokenGenerator.getExpiryDate;
|
||||
import static org.openmetadata.service.util.UserUtil.getRoleListFromUser;
|
||||
import static org.openmetadata.service.util.UserUtil.getRolesFromAuthorizationToken;
|
||||
@ -51,6 +52,7 @@ import java.time.ZoneId;
|
||||
import java.util.Base64;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
@ -1564,7 +1566,8 @@ public class UserResource extends EntityResource<User, UserRepository> {
|
||||
switch (authType) {
|
||||
case JWT -> {
|
||||
User original = retrieveBotUser(user, uriInfo);
|
||||
if (original == null || !hasAJWTAuthMechanism(original.getAuthenticationMechanism())) {
|
||||
if (original == null
|
||||
|| !hasAJWTAuthMechanism(user, original.getAuthenticationMechanism())) {
|
||||
JWTAuthMechanism jwtAuthMechanism =
|
||||
JsonUtils.convertValue(authMechanism.getConfig(), JWTAuthMechanism.class);
|
||||
authMechanism.setConfig(
|
||||
@ -1615,12 +1618,15 @@ public class UserResource extends EntityResource<User, UserRepository> {
|
||||
new AuthenticationMechanism().withAuthType(BASIC).withConfig(newAuthForUser));
|
||||
}
|
||||
|
||||
private boolean hasAJWTAuthMechanism(AuthenticationMechanism authMechanism) {
|
||||
private boolean hasAJWTAuthMechanism(User user, AuthenticationMechanism authMechanism) {
|
||||
if (authMechanism != null && JWT.equals(authMechanism.getAuthType())) {
|
||||
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
|
||||
secretsManager.decryptAuthenticationMechanism(user.getName(), authMechanism);
|
||||
JWTAuthMechanism jwtAuthMechanism =
|
||||
JsonUtils.convertValue(authMechanism.getConfig(), JWTAuthMechanism.class);
|
||||
return jwtAuthMechanism != null
|
||||
&& jwtAuthMechanism.getJWTToken() != null
|
||||
&& !Objects.equals(jwtAuthMechanism.getJWTToken(), NULL_SECRET_STRING)
|
||||
&& !StringUtils.EMPTY.equals(jwtAuthMechanism.getJWTToken());
|
||||
}
|
||||
return false;
|
||||
|
||||
@ -71,7 +71,7 @@ public class AWSSecretsManager extends AWSBasedSecretsManager {
|
||||
UpdateSecretRequest.builder()
|
||||
.secretId(secretName)
|
||||
.description("This secret was created by OpenMetadata")
|
||||
.secretString(Objects.isNull(secretValue) ? NULL_SECRET_STRING : secretValue)
|
||||
.secretString(cleanNullOrEmpty(secretValue))
|
||||
.build();
|
||||
this.secretsClient.updateSecret(updateSecretRequest);
|
||||
}
|
||||
|
||||
@ -84,7 +84,7 @@ public class AzureKVSecretsManager extends ExternalSecretsManager {
|
||||
@Override
|
||||
void storeSecret(String secretName, String secretValue) {
|
||||
client.setSecret(
|
||||
new KeyVaultSecret(secretName, secretValue)
|
||||
new KeyVaultSecret(secretName, cleanNullOrEmpty(secretValue))
|
||||
.setProperties(
|
||||
new SecretProperties().setTags(SecretsManager.getTags(getSecretsConfig()))));
|
||||
}
|
||||
|
||||
@ -38,4 +38,9 @@ public class DBSecretsManager extends SecretsManager {
|
||||
// Nothing to delete on the Noop SM. We only delete on External SM
|
||||
@Override
|
||||
protected void deleteSecretInternal(String secretName) {}
|
||||
|
||||
@Override
|
||||
String getSecret(String secretName) {
|
||||
return secretName;
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,12 +14,12 @@
|
||||
package org.openmetadata.service.secrets;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import org.openmetadata.schema.security.secrets.SecretsManagerProvider;
|
||||
import org.openmetadata.service.exception.UnhandledServerException;
|
||||
|
||||
public abstract class ExternalSecretsManager extends SecretsManager {
|
||||
public static final String NULL_SECRET_STRING = "null";
|
||||
public static final String SECRET_FIELD_PREFIX = "secret:";
|
||||
private final long waitTimeBetweenStoreCalls;
|
||||
|
||||
protected ExternalSecretsManager(
|
||||
@ -34,7 +34,7 @@ public abstract class ExternalSecretsManager extends SecretsManager {
|
||||
protected String storeValue(String fieldName, String value, String secretId, boolean store) {
|
||||
String fieldSecretId = buildSecretId(false, secretId, fieldName.toLowerCase(Locale.ROOT));
|
||||
// check if value does not start with 'config:' only String can have password annotation
|
||||
if (!value.startsWith(SECRET_FIELD_PREFIX)) {
|
||||
if (Boolean.FALSE.equals(isSecret(value))) {
|
||||
if (store) {
|
||||
upsertSecret(fieldSecretId, value);
|
||||
}
|
||||
@ -68,8 +68,6 @@ public abstract class ExternalSecretsManager extends SecretsManager {
|
||||
|
||||
abstract void updateSecret(String secretName, String secretValue);
|
||||
|
||||
abstract String getSecret(String secretName);
|
||||
|
||||
private void sleep() {
|
||||
// delay reaching secrets manager quotas
|
||||
if (waitTimeBetweenStoreCalls > 0) {
|
||||
@ -81,4 +79,8 @@ public abstract class ExternalSecretsManager extends SecretsManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String cleanNullOrEmpty(String secretValue) {
|
||||
return Objects.isNull(secretValue) || secretValue.isEmpty() ? NULL_SECRET_STRING : secretValue;
|
||||
}
|
||||
}
|
||||
|
||||
@ -60,7 +60,7 @@ public class GCPSecretsManager extends ExternalSecretsManager {
|
||||
} catch (IOException e) {
|
||||
throw new SecretsManagerUpdateException(e.getMessage(), e);
|
||||
}
|
||||
updateSecret(secretId, secretValue);
|
||||
updateSecret(secretId, cleanNullOrEmpty(secretValue));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -30,6 +30,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.openmetadata.annotations.PasswordField;
|
||||
import org.openmetadata.common.utils.CommonUtil;
|
||||
import org.openmetadata.schema.auth.BasicAuthMechanism;
|
||||
import org.openmetadata.schema.auth.JWTAuthMechanism;
|
||||
import org.openmetadata.schema.entity.automations.Workflow;
|
||||
import org.openmetadata.schema.entity.services.ServiceType;
|
||||
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
|
||||
@ -48,6 +49,8 @@ import org.openmetadata.service.util.ReflectionUtil;
|
||||
|
||||
@Slf4j
|
||||
public abstract class SecretsManager {
|
||||
public static final String SECRET_FIELD_PREFIX = "secret:";
|
||||
|
||||
public record SecretsConfig(
|
||||
String clusterName, String prefix, List<String> tags, Parameters parameters) {}
|
||||
|
||||
@ -72,6 +75,20 @@ public abstract class SecretsManager {
|
||||
this.secretsIdConfig = builSecretsIdConfig();
|
||||
}
|
||||
|
||||
public Boolean isSecret(String string) {
|
||||
return string.startsWith(SECRET_FIELD_PREFIX);
|
||||
}
|
||||
|
||||
public String getSecretValue(String secretWithPrefix) {
|
||||
String secretName = secretWithPrefix.split(SECRET_FIELD_PREFIX, 2)[1];
|
||||
return getSecret(secretName);
|
||||
}
|
||||
|
||||
/**
|
||||
* GET a secret using the SM implementation if the string starts with `secret:/`
|
||||
*/
|
||||
abstract String getSecret(String secretName);
|
||||
|
||||
/**
|
||||
* Override this method in any Secrets Manager implementation
|
||||
* that has other requirements
|
||||
@ -124,32 +141,43 @@ public abstract class SecretsManager {
|
||||
}
|
||||
}
|
||||
|
||||
public void encryptAuthenticationMechanism(
|
||||
public Object encryptAuthenticationMechanism(
|
||||
String name, AuthenticationMechanism authenticationMechanism) {
|
||||
if (authenticationMechanism != null) {
|
||||
AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism);
|
||||
try {
|
||||
encryptPasswordFields(authenticationMechanism, buildSecretId(true, "bot", name), true);
|
||||
return encryptPasswordFields(
|
||||
authenticationMechanism, buildSecretId(true, "bot", name), true);
|
||||
} catch (Exception e) {
|
||||
throw new SecretsManagerException(
|
||||
Response.Status.BAD_REQUEST,
|
||||
String.format("Failed to encrypt user bot instance [%s]", name));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public void decryptAuthenticationMechanism(
|
||||
/**
|
||||
* This is used to handle the JWT Token internally, in the JWTFilter, when
|
||||
* calling for the auth-mechanism in the UI, etc.
|
||||
* If using SM, we need to decrypt and GET the secret to ensure we are comparing
|
||||
* the right values.
|
||||
*/
|
||||
public AuthenticationMechanism decryptAuthenticationMechanism(
|
||||
String name, AuthenticationMechanism authenticationMechanism) {
|
||||
if (authenticationMechanism != null) {
|
||||
AuthenticationMechanismBuilder.addDefinedConfig(authenticationMechanism);
|
||||
try {
|
||||
decryptPasswordFields(authenticationMechanism);
|
||||
AuthenticationMechanism fernetDecrypted =
|
||||
(AuthenticationMechanism) decryptPasswordFields(authenticationMechanism);
|
||||
return (AuthenticationMechanism) getSecretFields(fernetDecrypted);
|
||||
} catch (Exception e) {
|
||||
throw new SecretsManagerException(
|
||||
Response.Status.BAD_REQUEST,
|
||||
String.format("Failed to decrypt user bot instance [%s]", name));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public void encryptIngestionPipeline(IngestionPipeline ingestionPipeline) {
|
||||
@ -261,6 +289,22 @@ public abstract class SecretsManager {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Used only in the OM Connection Builder, which sends the credentials to Ingestion Workflows
|
||||
*/
|
||||
public JWTAuthMechanism decryptJWTAuthMechanism(JWTAuthMechanism authMechanism) {
|
||||
if (authMechanism != null) {
|
||||
try {
|
||||
decryptPasswordFields(authMechanism);
|
||||
} catch (Exception e) {
|
||||
throw new SecretsManagerException(
|
||||
Response.Status.BAD_REQUEST, "Failed to decrypt OpenMetadataConnection instance.");
|
||||
}
|
||||
return authMechanism;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Object encryptPasswordFields(Object toEncryptObject, String secretId, boolean store) {
|
||||
try {
|
||||
if (!DO_NOT_ENCRYPT_CLASSES.contains(toEncryptObject.getClass())) {
|
||||
@ -339,6 +383,45 @@ public abstract class SecretsManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the object and use the secrets manager to get the right value to show
|
||||
*/
|
||||
private Object getSecretFields(Object toDecryptObject) {
|
||||
try {
|
||||
// for each get method
|
||||
Arrays.stream(toDecryptObject.getClass().getMethods())
|
||||
.filter(ReflectionUtil::isGetMethodOfObject)
|
||||
.forEach(
|
||||
method -> {
|
||||
Object obj = ReflectionUtil.getObjectFromMethod(method, toDecryptObject);
|
||||
String fieldName = method.getName().replaceFirst("get", "");
|
||||
// if the object matches the package of openmetadata
|
||||
if (Boolean.TRUE.equals(CommonUtil.isOpenMetadataObject(obj))) {
|
||||
// encryptPasswordFields
|
||||
getSecretFields(obj);
|
||||
// check if it has annotation
|
||||
} else if (obj != null && method.getAnnotation(PasswordField.class) != null) {
|
||||
String fieldValue = (String) obj;
|
||||
// get setMethod
|
||||
Method toSet = ReflectionUtil.getToSetMethod(toDecryptObject, obj, fieldName);
|
||||
// set new value
|
||||
ReflectionUtil.setValueInMethod(
|
||||
toDecryptObject,
|
||||
Boolean.TRUE.equals(isSecret(fieldValue))
|
||||
? getSecretValue(fieldValue)
|
||||
: fieldValue,
|
||||
toSet);
|
||||
}
|
||||
});
|
||||
return toDecryptObject;
|
||||
} catch (Exception e) {
|
||||
throw new SecretsManagerException(
|
||||
String.format(
|
||||
"Error trying to GET secret [%s] due to [%s]",
|
||||
toDecryptObject.toString(), e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract String storeValue(
|
||||
String fieldName, String value, String secretId, boolean store);
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@ package org.openmetadata.service.secrets.masker;
|
||||
|
||||
import java.util.Set;
|
||||
import org.openmetadata.schema.auth.BasicAuthMechanism;
|
||||
import org.openmetadata.schema.auth.JWTAuthMechanism;
|
||||
import org.openmetadata.schema.entity.automations.Workflow;
|
||||
import org.openmetadata.schema.entity.services.ServiceType;
|
||||
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
|
||||
@ -24,7 +25,7 @@ import org.openmetadata.schema.security.client.OpenMetadataJWTClientConfig;
|
||||
public abstract class EntityMasker {
|
||||
|
||||
protected static final Set<Class<?>> DO_NOT_MASK_CLASSES =
|
||||
Set.of(OpenMetadataJWTClientConfig.class, BasicAuthMechanism.class);
|
||||
Set.of(OpenMetadataJWTClientConfig.class, JWTAuthMechanism.class, BasicAuthMechanism.class);
|
||||
|
||||
public abstract Object maskServiceConnectionConfig(
|
||||
Object connectionConfig, String connectionType, ServiceType serviceType);
|
||||
|
||||
@ -17,6 +17,8 @@ import org.openmetadata.schema.entity.teams.User;
|
||||
import org.openmetadata.service.Entity;
|
||||
import org.openmetadata.service.jdbi3.UserRepository;
|
||||
import org.openmetadata.service.resources.teams.UserResource;
|
||||
import org.openmetadata.service.secrets.SecretsManager;
|
||||
import org.openmetadata.service.secrets.SecretsManagerFactory;
|
||||
import org.openmetadata.service.util.EntityUtil.Fields;
|
||||
import org.openmetadata.service.util.JsonUtils;
|
||||
|
||||
@ -64,6 +66,8 @@ public class BotTokenCache {
|
||||
NON_DELETED,
|
||||
true);
|
||||
AuthenticationMechanism authenticationMechanism = user.getAuthenticationMechanism();
|
||||
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
|
||||
secretsManager.decryptAuthenticationMechanism(user.getName(), authenticationMechanism);
|
||||
if (authenticationMechanism != null) {
|
||||
JWTAuthMechanism jwtAuthMechanism =
|
||||
JsonUtils.convertValue(authenticationMechanism.getConfig(), JWTAuthMechanism.class);
|
||||
|
||||
@ -36,6 +36,7 @@ import org.openmetadata.service.exception.EntityNotFoundException;
|
||||
import org.openmetadata.service.jdbi3.BotRepository;
|
||||
import org.openmetadata.service.jdbi3.IngestionPipelineRepository;
|
||||
import org.openmetadata.service.jdbi3.UserRepository;
|
||||
import org.openmetadata.service.secrets.SecretsManager;
|
||||
import org.openmetadata.service.secrets.SecretsManagerFactory;
|
||||
import org.openmetadata.service.util.EntityUtil.Fields;
|
||||
|
||||
@ -52,6 +53,7 @@ public class OpenMetadataConnectionBuilder {
|
||||
private Object openMetadataSSLConfig;
|
||||
BotRepository botRepository;
|
||||
UserRepository userRepository;
|
||||
SecretsManager secretsManager;
|
||||
|
||||
public OpenMetadataConnectionBuilder(
|
||||
OpenMetadataApplicationConfig openMetadataApplicationConfig) {
|
||||
@ -128,7 +130,8 @@ public class OpenMetadataConnectionBuilder {
|
||||
|
||||
clusterName = openMetadataApplicationConfig.getClusterName();
|
||||
secretsManagerLoader = pipelineServiceClientConfiguration.getSecretsManagerLoader();
|
||||
secretsManagerProvider = SecretsManagerFactory.getSecretsManager().getSecretsManagerProvider();
|
||||
secretsManager = SecretsManagerFactory.getSecretsManager();
|
||||
secretsManagerProvider = secretsManager.getSecretsManagerProvider();
|
||||
}
|
||||
|
||||
private void initializeBotUser(String botName) {
|
||||
@ -157,6 +160,7 @@ public class OpenMetadataConnectionBuilder {
|
||||
== AuthenticationMechanism.AuthType.JWT) {
|
||||
JWTAuthMechanism jwtAuthMechanism =
|
||||
JsonUtils.convertValue(authMechanism.getConfig(), JWTAuthMechanism.class);
|
||||
secretsManager.decryptJWTAuthMechanism(jwtAuthMechanism);
|
||||
return new OpenMetadataJWTClientConfig().withJwtToken(jwtAuthMechanism.getJWTToken());
|
||||
}
|
||||
throw new IllegalArgumentException(
|
||||
|
||||
@ -108,7 +108,7 @@ public class OpenMetadataOperations implements Callable<Integer> {
|
||||
public Integer call() {
|
||||
LOG.info(
|
||||
"Subcommand needed: 'info', 'validate', 'repair', 'check-connection', "
|
||||
+ "'drop-create', 'migrate', 'reindex', 'deploy-pipelines'");
|
||||
+ "'drop-create', 'changelog', 'migrate', 'migrate-secrets', 'reindex', 'deploy-pipelines'");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -15,10 +15,13 @@ import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.openmetadata.schema.api.services.DatabaseConnection;
|
||||
import org.openmetadata.schema.auth.JWTAuthMechanism;
|
||||
import org.openmetadata.schema.auth.JWTTokenExpiry;
|
||||
import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
|
||||
import org.openmetadata.schema.entity.automations.Workflow;
|
||||
import org.openmetadata.schema.entity.automations.WorkflowType;
|
||||
import org.openmetadata.schema.entity.services.ServiceType;
|
||||
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
|
||||
import org.openmetadata.schema.services.connections.database.MysqlConnection;
|
||||
import org.openmetadata.schema.services.connections.database.common.basicAuth;
|
||||
import org.openmetadata.service.exception.SecretsManagerException;
|
||||
@ -47,6 +50,30 @@ public class SecretsManagerLifecycleTest {
|
||||
secretsManager.setFernet(fernet);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testJWTTokenEncryption() {
|
||||
AuthenticationMechanism authenticationMechanism =
|
||||
new AuthenticationMechanism()
|
||||
.withAuthType(AuthenticationMechanism.AuthType.JWT)
|
||||
.withConfig(
|
||||
new JWTAuthMechanism()
|
||||
.withJWTToken("token")
|
||||
.withJWTTokenExpiry(JWTTokenExpiry.Unlimited));
|
||||
|
||||
AuthenticationMechanism encrypted =
|
||||
(AuthenticationMechanism)
|
||||
secretsManager.encryptAuthenticationMechanism("ingestion-bot", authenticationMechanism);
|
||||
// Validate that the JWT Token gets properly encrypted
|
||||
JWTAuthMechanism encryptedAuth = (JWTAuthMechanism) encrypted.getConfig();
|
||||
assertEquals(ENCRYPTED_VALUE, encryptedAuth.getJWTToken());
|
||||
|
||||
AuthenticationMechanism decrypted =
|
||||
secretsManager.decryptAuthenticationMechanism("ingestion-bot", encrypted);
|
||||
|
||||
JWTAuthMechanism decryptedAuth = (JWTAuthMechanism) decrypted.getConfig();
|
||||
assertEquals(DECRYPTED_VALUE, decryptedAuth.getJWTToken());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDatabaseServiceConnectionConfigLifecycle() {
|
||||
String password = "openmetadata-test";
|
||||
|
||||
@ -38,8 +38,11 @@
|
||||
},
|
||||
"properties": {
|
||||
"JWTToken": {
|
||||
"title": "JWT Token",
|
||||
"description": "JWT Auth Token.",
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"format": "password",
|
||||
"expose": true
|
||||
},
|
||||
"JWTTokenExpiry": {
|
||||
"$ref": "#/definitions/JWTTokenExpiry"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user