yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-02 19:48:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Send encrypted automation workflow (#11681)

Browse Source
This commit is contained in:
Pere Miquel Brull 2023-05-19 15:04:42 +02:00 committed by GitHub
parent 2f6fe28a3e
commit d52d773707
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 47 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
ingestion/src/metadata/automations/runner.py
View File

@ -24,16 +24,26 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
def execute(automation_workflow: AutomationWorkflow) -> Any:
def execute(encrypted_automation_workflow: AutomationWorkflow) -> Any:
"""
Execute the automation workflow.
The implementation depends on the request body type
"""
return run_workflow(automation_workflow.request, automation_workflow)
# This will already instantiate the Secrets Manager
metadata = OpenMetadata(
config=encrypted_automation_workflow.openMetadataServerConnection
)
automation_workflow = metadata.get_by_name(
entity=AutomationWorkflow, fqn=encrypted_automation_workflow.name.__root__
)
return run_workflow(automation_workflow.request, automation_workflow, metadata)
@singledispatch
def run_workflow(request: Any, _: AutomationWorkflow) -> Any:
def run_workflow(request: Any, *_, **__) -> Any:
"""
Main entrypoint to execute the automation workflow
"""
@ -41,12 +51,15 @@ def run_workflow(request: Any, _: AutomationWorkflow) -> Any:
@run_workflow.register
def _(request: TestServiceConnectionRequest, automation_workflow: AutomationWorkflow):
def _(
request: TestServiceConnectionRequest,
automation_workflow: AutomationWorkflow,
metadata: OpenMetadata,
):
"""
Run the test connection
"""
# This will already instantiate the Secrets Manager
metadata = OpenMetadata(config=automation_workflow.openMetadataServerConnection)
connection = get_connection(request.connection.config)
# Find the test_connection function in each <source>/connection.py file

6
ingestion/tests/integration/ometa/test_ometa_service_api.py
View File

@ -54,11 +54,11 @@ class OMetaServiceTest(TestCase):
jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
),
)
metadata = OpenMetadata(server_config)
admin_metadata = OpenMetadata(server_config)
# we need to use ingestion bot user for this test since the admin user won't be able to see the password fields
ingestion_bot: User = metadata.get_by_name(entity=User, fqn="ingestion-bot")
ingestion_bot_auth: AuthenticationMechanism = metadata.get_by_id(
ingestion_bot: User = admin_metadata.get_by_name(entity=User, fqn="ingestion-bot")
ingestion_bot_auth: AuthenticationMechanism = admin_metadata.get_by_id(
entity=AuthenticationMechanism, entity_id=ingestion_bot.id
)
server_config.securityConfig = OpenMetadataJWTClientConfig(

22
ingestion/tests/integration/ometa/test_ometa_workflow_api.py
View File

@ -35,6 +35,7 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
)
from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
from metadata.generated.schema.entity.services.serviceType import ServiceType
from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User
from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
OpenMetadataJWTClientConfig,
)
@ -56,9 +57,19 @@ class OMetaWorkflowTest(TestCase):
jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
),
)
metadata = OpenMetadata(server_config)
admin_metadata = OpenMetadata(server_config)
assert metadata.health_check()
assert admin_metadata.health_check()
# we need to use ingestion bot user for this test since the admin user won't be able to see the password fields
ingestion_bot: User = admin_metadata.get_by_name(entity=User, fqn="ingestion-bot")
ingestion_bot_auth: AuthenticationMechanism = admin_metadata.get_by_id(
entity=AuthenticationMechanism, entity_id=ingestion_bot.id
)
server_config.securityConfig = OpenMetadataJWTClientConfig(
jwtToken=ingestion_bot_auth.config.JWTToken
)
metadata = OpenMetadata(server_config)
@classmethod
def setUpClass(cls) -> None:
@ -138,11 +149,16 @@ class OMetaWorkflowTest(TestCase):
self.metadata.create_or_update(data=self.create)
res = self.metadata.get_by_name(
res: Workflow = self.metadata.get_by_name(
entity=Workflow, fqn=self.entity.fullyQualifiedName
)
self.assertEqual(res.name, self.entity.name)
# The ingestion-bot should see the password
self.assertEqual(
res.request.connection.config.password.get_secret_value(), "password"
)
def test_get_id(self):
"""
We can fetch a Dashboard by ID and get it back as Entity

8
openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java
View File

@ -338,8 +338,12 @@ public class WorkflowResource extends EntityResource<Workflow, WorkflowRepositor
EntityUtil.Fields fields = getFields(FIELD_OWNER);
Workflow workflow = dao.get(uriInfo, id, fields);
workflow.setOpenMetadataServerConnection(new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build());
workflow = decryptOrNullify(securityContext, workflow);
workflow = unmask(workflow);
/*
We will send the encrypted Workflow to the Pipeline Service Client
It will be fetched from the API from there, since we are
decrypting on GET based on user auth. The ingestion-bot will then
be able to pick up the right data.
*/
return pipelineServiceClient.runAutomationsWorkflow(workflow);
}