From 215a274bd0ea812a4ad6d5b5bc7d4fd1b7551ea3 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Fri, 3 May 2024 15:17:25 +0530 Subject: [PATCH] Fix #14130: Add support for azure auth in trino (#16117) --- .../source/database/trino/connection.py | 13 +++++ .../connectors/database/trino/index.md | 25 ++++++++++ .../connectors/database/trino/yaml.md | 31 ++++++++++++ .../connections/database/trinoConnection.json | 3 ++ .../ui/public/locales/en-US/Database/Trino.md | 50 +++++++++++++++++++ 5 files changed, 122 insertions(+) diff --git a/ingestion/src/metadata/ingestion/source/database/trino/connection.py b/ingestion/src/metadata/ingestion/source/database/trino/connection.py index b9252311ce3..245e0a4f9b3 100644 --- a/ingestion/src/metadata/ingestion/source/database/trino/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/trino/connection.py @@ -18,6 +18,7 @@ from urllib.parse import quote_plus from requests import Session from sqlalchemy.engine import Engine +from metadata.clients.azure_client import AzureClient from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) @@ -98,6 +99,18 @@ def get_connection(connection: TrinoConnection) -> Engine: connection.connectionArguments.__root__["verify"] = { "verify": connection.verify } + if hasattr(connection.authType, "azureConfig"): + azure_client = AzureClient(connection.authType.azureConfig).create_client() + if not connection.authType.azureConfig.scopes: + raise ValueError( + "Azure Scopes are missing, please refer https://learn.microsoft.com/en-gb/azure/mysql/flexible-server/how-to-azure-ad#2---retrieve-microsoft-entra-access-token and fetch the resource associated with it, for e.g. https://ossrdbms-aad.database.windows.net/.default" + ) + access_token_obj = azure_client.get_token( + *connection.authType.azureConfig.scopes.split(",") + ) + if not connection.connectionOptions: + connection.connectionOptions = init_empty_connection_options() + connection.connectionOptions.__root__["access_token"] = access_token_obj.token return create_generic_db_connection( connection=connection, get_connection_url_fn=get_connection_url, diff --git a/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/trino/index.md b/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/trino/index.md index 56b53e668b1..620d61bc3da 100644 --- a/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/trino/index.md +++ b/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/trino/index.md @@ -57,6 +57,31 @@ Executing the profiler workflow or data quality tests, will require the user to - **Password**: Password to connect to Trino. - **JWT Auth Config**: - **JWT**: JWT can be used to authenticate with trino. Follow the steps in the [official trino](https://trino.io/docs/current/security/jwt.html) documentation to setup trino with jwt. + - **Azure**: + - **Client ID**: To get the Client ID (also known as application ID), follow these steps: + 1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). + 2. Search for `App registrations` and select the `App registrations link`. + 3. Select the `Azure AD` app you're using for Trino. + 4. From the Overview section, copy the `Application (client) ID`. + - **Client Secret**: To get the client secret, follow these steps: + 1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). + 2. Search for `App registrations` and select the `App registrations link`. + 3. Select the `Azure AD` app you're using for Trino. + 4. Under `Manage`, select `Certificates & secrets`. + 5. Under `Client secrets`, select `New client secret`. + 6. In the `Add a client secret` pop-up window, provide a description for your application secret. Choose when the application should expire, and select `Add`. + 7. From the `Client secrets` section, copy the string in the `Value` column of the newly created application secret. + - **Tenant ID**: To get the tenant ID, follow these steps: + 1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). + 2. Search for `App registrations` and select the `App registrations link`. + 3. Select the `Azure AD` app you're using for Trino. + 4. From the `Overview` section, copy the `Directory (tenant) ID`. + - **Scopes**: To let OM use the Trino Auth APIs using your Azure AD app, you'll need to add the scope + 1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). + 2. Search for `App registrations` and select the `App registrations link`. + 3. Select the `Azure AD` app you're using for Trino. + 4. From the `Expose an API` section, copy the `Application ID URI` + 5. Make sure the URI ends with `/.default` in case it does not, you can append the same manually - **Host and Port**: Enter the fully qualified hostname and port number for your Trino deployment in the Host and Port field. - **Catalog**: Trino offers a catalog feature where all the databases are stored. - **DatabaseSchema**: DatabaseSchema of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata Ingestion attempts to scan all the databaseSchema. diff --git a/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/trino/yaml.md b/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/trino/yaml.md index c90d4ddc337..c6ce7dccb72 100644 --- a/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/trino/yaml.md +++ b/openmetadata-docs/content/v1.4.x-SNAPSHOT/connectors/database/trino/yaml.md @@ -74,6 +74,32 @@ This is a sample config for Trino: - **password**: Password to connect to Trino. - **JWT Auth Config**: - **jwt**: JWT can be used to authenticate with trino. Follow the steps in the [official trino](https://trino.io/docs/current/security/jwt.html) documentation to setup trino with jwt. + - **Azure**: + - **clientId**: To get the Client ID (also known as application ID), follow these steps: + 1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). + 2. Search for `App registrations` and select the `App registrations link`. + 3. Select the `Azure AD` app you're using for Trino. + 4. From the Overview section, copy the `Application (client) ID`. + - **clientSecret**: To get the client secret, follow these steps: + 1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). + 2. Search for `App registrations` and select the `App registrations link`. + 3. Select the `Azure AD` app you're using for Trino. + 4. Under `Manage`, select `Certificates & secrets`. + 5. Under `Client secrets`, select `New client secret`. + 6. In the `Add a client secret` pop-up window, provide a description for your application secret. Choose when the application should expire, and select `Add`. + 7. From the `Client secrets` section, copy the string in the `Value` column of the newly created application secret. + - **tenantId**: To get the tenant ID, follow these steps: + 1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). + 2. Search for `App registrations` and select the `App registrations link`. + 3. Select the `Azure AD` app you're using for Trino. + 4. From the `Overview` section, copy the `Directory (tenant) ID`. + - **scopes**: To let OM use the Trino Auth APIs using your Azure AD app, you'll need to add the scope + 1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). + 2. Search for `App registrations` and select the `App registrations link`. + 3. Select the `Azure AD` app you're using for Trino. + 4. From the `Expose an API` section, copy the `Application ID URI` + 5. Make sure the URI ends with `/.default` in case it does not, you can append the same manually + {% /codeInfo %} @@ -150,6 +176,11 @@ source: password: password # # For JWT auth # jwt: jwt_token + # azureConfig: + # clientId: your-client-id + # clientSecret: your-client-secret + # tenantId: your-tenant-id + # scopes: https://your-scope/.default ``` ```yaml {% srNumber=4 %} catalog: diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/trinoConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/trinoConnection.json index 36f56c6e5fe..08b3ab4c64d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/trinoConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/trinoConnection.json @@ -46,6 +46,9 @@ }, { "$ref": "./common/jwtAuth.json" + }, + { + "$ref": "./common/azureConfig.json" } ] }, diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Trino.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Trino.md index e6369593415..4e86cd674d9 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Trino.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Trino.md @@ -53,6 +53,56 @@ Follow the steps in the [official trino](https://trino.io/docs/current/security/ $$ +## Azure + +$$section +### Client ID $(id="clientId") + +To get the Client ID (also known as application ID), follow these steps: + +1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). +2. Search for `App registrations` and select the `App registrations link`. +3. Select the `Azure AD` app you're using for Trino. +4. From the Overview section, copy the `Application (client) ID`. + +$$ + +$$section +### Client Secret $(id="clientSecret") +To get the client secret, follow these steps: + +1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). +2. Search for `App registrations` and select the `App registrations link`. +3. Select the `Azure AD` app you're using for Trino. +4. Under `Manage`, select `Certificates & secrets`. +5. Under `Client secrets`, select `New client secret`. +6. In the `Add a client secret` pop-up window, provide a description for your application secret. Choose when the application should expire, and select `Add`. +7. From the `Client secrets` section, copy the string in the `Value` column of the newly created application secret. + +$$ + +$$section +### Tenant ID $(id="tenantId") + +To get the tenant ID, follow these steps: + +1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). +2. Search for `App registrations` and select the `App registrations link`. +3. Select the `Azure AD` app you're using for Trino. +4. From the `Overview` section, copy the `Directory (tenant) ID`. +$$ + +$$section +### Scopes $(id="Scopes") + +To let OM use the Trino Auth APIs using your Azure AD app, you'll need to add the scope +1. Log into [Microsoft Azure](https://ms.portal.azure.com/#allservices). +2. Search for `App registrations` and select the `App registrations link`. +3. Select the `Azure AD` app you're using for Trino. +4. From the `Expose an API` section, copy the `Application ID URI` +5. Make sure the URI ends with `/.default` in case it does not, you can append the same manually +$$ + $$section ### Host Port $(id="hostPort") This parameter specifies the host and port of the Trino instance. This should be specified as a string in the format `hostname:port`. For example, you might set the hostPort parameter to `localhost:8080`.