diff --git a/ingestion/src/metadata/automations/runner.py b/ingestion/src/metadata/automations/runner.py index f09b10da913..e6f9880f5a6 100644 --- a/ingestion/src/metadata/automations/runner.py +++ b/ingestion/src/metadata/automations/runner.py @@ -24,16 +24,26 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.connections import get_connection, get_test_connection_fn -def execute(automation_workflow: AutomationWorkflow) -> Any: +def execute(encrypted_automation_workflow: AutomationWorkflow) -> Any: """ Execute the automation workflow. The implementation depends on the request body type """ - return run_workflow(automation_workflow.request, automation_workflow) + + # This will already instantiate the Secrets Manager + metadata = OpenMetadata( + config=encrypted_automation_workflow.openMetadataServerConnection + ) + + automation_workflow = metadata.get_by_name( + entity=AutomationWorkflow, fqn=encrypted_automation_workflow.name.__root__ + ) + + return run_workflow(automation_workflow.request, automation_workflow, metadata) @singledispatch -def run_workflow(request: Any, _: AutomationWorkflow) -> Any: +def run_workflow(request: Any, *_, **__) -> Any: """ Main entrypoint to execute the automation workflow """ @@ -41,12 +51,15 @@ def run_workflow(request: Any, _: AutomationWorkflow) -> Any: @run_workflow.register -def _(request: TestServiceConnectionRequest, automation_workflow: AutomationWorkflow): +def _( + request: TestServiceConnectionRequest, + automation_workflow: AutomationWorkflow, + metadata: OpenMetadata, +): """ Run the test connection """ - # This will already instantiate the Secrets Manager - metadata = OpenMetadata(config=automation_workflow.openMetadataServerConnection) + connection = get_connection(request.connection.config) # Find the test_connection function in each /connection.py file diff --git a/ingestion/tests/integration/ometa/test_ometa_service_api.py b/ingestion/tests/integration/ometa/test_ometa_service_api.py index fbd244933f8..14e4a5dd074 100644 --- a/ingestion/tests/integration/ometa/test_ometa_service_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_service_api.py @@ -54,11 +54,11 @@ class OMetaServiceTest(TestCase): jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" ), ) - metadata = OpenMetadata(server_config) + admin_metadata = OpenMetadata(server_config) # we need to use ingestion bot user for this test since the admin user won't be able to see the password fields - ingestion_bot: User = metadata.get_by_name(entity=User, fqn="ingestion-bot") - ingestion_bot_auth: AuthenticationMechanism = metadata.get_by_id( + ingestion_bot: User = admin_metadata.get_by_name(entity=User, fqn="ingestion-bot") + ingestion_bot_auth: AuthenticationMechanism = admin_metadata.get_by_id( entity=AuthenticationMechanism, entity_id=ingestion_bot.id ) server_config.securityConfig = OpenMetadataJWTClientConfig( diff --git a/ingestion/tests/integration/ometa/test_ometa_workflow_api.py b/ingestion/tests/integration/ometa/test_ometa_workflow_api.py index 525a2334880..c3efea294dc 100644 --- a/ingestion/tests/integration/ometa/test_ometa_workflow_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_workflow_api.py @@ -35,6 +35,7 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata ) from metadata.generated.schema.entity.services.databaseService import DatabaseConnection from metadata.generated.schema.entity.services.serviceType import ServiceType +from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( OpenMetadataJWTClientConfig, ) @@ -56,9 +57,19 @@ class OMetaWorkflowTest(TestCase): jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" ), ) - metadata = OpenMetadata(server_config) + admin_metadata = OpenMetadata(server_config) - assert metadata.health_check() + assert admin_metadata.health_check() + + # we need to use ingestion bot user for this test since the admin user won't be able to see the password fields + ingestion_bot: User = admin_metadata.get_by_name(entity=User, fqn="ingestion-bot") + ingestion_bot_auth: AuthenticationMechanism = admin_metadata.get_by_id( + entity=AuthenticationMechanism, entity_id=ingestion_bot.id + ) + server_config.securityConfig = OpenMetadataJWTClientConfig( + jwtToken=ingestion_bot_auth.config.JWTToken + ) + metadata = OpenMetadata(server_config) @classmethod def setUpClass(cls) -> None: @@ -138,11 +149,16 @@ class OMetaWorkflowTest(TestCase): self.metadata.create_or_update(data=self.create) - res = self.metadata.get_by_name( + res: Workflow = self.metadata.get_by_name( entity=Workflow, fqn=self.entity.fullyQualifiedName ) self.assertEqual(res.name, self.entity.name) + # The ingestion-bot should see the password + self.assertEqual( + res.request.connection.config.password.get_secret_value(), "password" + ) + def test_get_id(self): """ We can fetch a Dashboard by ID and get it back as Entity diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java index 46086c64579..184ed0756b6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java @@ -338,8 +338,12 @@ public class WorkflowResource extends EntityResource