From 5f07c6281a782f1abb9e7a18dcaba9ca56d6cf08 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Mon, 31 Jul 2023 19:26:03 +0530 Subject: [PATCH] Clickhouse: Make https protocol as permanent field (#12303) --- .../v014__create_db_connection_info.sql | 5 ++++ .../v014__create_db_connection_info.sql | 5 ++++ .../source/database/clickhouse/connection.py | 9 +++++- .../connectors/database/clickhouse/index.md | 12 ++------ .../connectors/database/clickhouse/yaml.md | 28 +++++++++++++++++++ .../database/clickhouseConnection.json | 7 ++++- .../locales/en-US/Database/Clickhouse.md | 8 +++++- 7 files changed, 62 insertions(+), 12 deletions(-) diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql index 2c6bf5877eb..3f67a62b67c 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql @@ -64,3 +64,8 @@ where de2.serviceType = 'Mssql' -- column deleted not needed for entities that don't support soft delete ALTER TABLE query_entity DROP COLUMN deleted; ALTER TABLE event_subscription_entity DROP COLUMN deleted; + +-- remove keyfile from clickhouse +UPDATE dbservice_entity +SET json = JSON_REMOVE(json, '$.connection.config.keyfile') +WHERE serviceType = 'Clickhouse'; diff --git a/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql index 7077980cec9..431fe5e163b 100644 --- a/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql @@ -60,3 +60,8 @@ AND json->>'{connection,config,database}' IS NULL; -- column deleted not needed for entities that don't support soft delete ALTER TABLE query_entity DROP COLUMN deleted; ALTER TABLE event_subscription_entity DROP COLUMN deleted; + +-- remove keyfile from clickhouse +UPDATE dbservice_entity +SET json = json #-'{connection,config,keyfile}' +WHERE serviceType = 'Clickhouse'; diff --git a/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py b/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py index 45b280144ff..20cd6c2cb87 100644 --- a/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py @@ -28,6 +28,7 @@ from metadata.ingestion.connections.builders import ( get_connection_args_common, get_connection_url_common, init_empty_connection_arguments, + init_empty_connection_options, ) from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -35,18 +36,24 @@ from metadata.ingestion.source.database.clickhouse.queries import ( CLICKHOUSE_SQL_STATEMENT_TEST, ) +HTTPS_PROTOCOL = "https" + def get_connection(connection: ClickhouseConnection) -> Engine: """ Create Clickhouse connection """ if connection.secure or connection.keyfile: - if connection.connectionArguments: + if not connection.connectionArguments: connection.connectionArguments = init_empty_connection_arguments() if connection.secure: connection.connectionArguments.__root__["secure"] = connection.secure if connection.keyfile: connection.connectionArguments.__root__["keyfile"] = connection.keyfile + if connection.https: + if not connection.connectionOptions: + connection.connectionOptions = init_empty_connection_options() + connection.connectionOptions.__root__["protocol"] = HTTPS_PROTOCOL return create_generic_db_connection( connection=connection, diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/index.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/index.md index 8f50a92d458..2fbd4071607 100644 --- a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/index.md +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/index.md @@ -91,18 +91,12 @@ For the usage and lineage workflow, the user will need `SELECT` privilege. You c - **Username**: Specify the User to connect to Clickhouse. It should have enough privileges to read all the metadata. - **Password**: Password to connect to Clickhouse. - **Host and Port**: Enter the fully qualified hostname and port number for your Clickhouse deployment in the Host and Port field. +- **Use HTTPS Protocol**: Enable this flag when the when the Clickhouse instance is hosted via HTTPS protocol. This flag is useful when you are using `clickhouse+http` connection scheme. +- **Secure Connection**: Establish secure connection with ClickHouse. ClickHouse supports secure communication over SSL/TLS to protect data in transit, by checking this option, it establishes secure connection with ClickHouse. This flag is useful when you are using `clickhouse+native` connection scheme. +- **Key File**: The key file path is the location when ClickHouse looks for a file containing the private key needed for secure communication over SSL/TLS. By default, ClickHouse will look for the key file in the `/etc/clickhouse-server directory`, with the file name `server.key`. However, this can be customized in the ClickHouse configuration file (`config.xml`). This flag is useful when you are using `clickhouse+native` connection scheme and the secure connection flag is enabled. {% partial file="/v1.1.1/connectors/database/advanced-configuration.md" /%} -You can find the full list of accepted options [here](https://clickhouse-driver.readthedocs.io/en/latest/api.html#clickhouse_driver.connection.Connection). - -- **Connecting to Clickhouse with SSL Certificate**: You will need to use the `clickhouse+native` connection scheme. Then in the `Connection Options` reference the following key with their value: - - `verify`: `true` - - `secure`: `true` - - `keyfile`: `/path/to/key/file` - -The `keyfile` needs to be accessible by the service running the ingestion. For example if you are running the ingestion in a docker container, your `keyfile` needs to be present in the container at the location specify as a value in the `Connection Options`. Additionally, your `keyfile` needs to be in the `.cert` or `.pem` format. - {% /extraContent %} {% partial file="/v1.1.1/connectors/test-connection.md" /%} diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/yaml.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/yaml.md index 3d18a5b722b..0bdf14d6900 100644 --- a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/yaml.md +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/yaml.md @@ -148,6 +148,25 @@ This is a sample config for Clickhouse: - **clickhouse+native**: Uses the native ClickHouse TCP protocol for communication. Faster than http, but may require additional server-side configuration. Recommended for performance-critical applications. +{% /codeInfo %} + +{% codeInfo srNumber=35 %} + +**https**: Enable this flag when the when the Clickhouse instance is hosted via HTTPS protocol. This flag is useful when you are using `clickhouse+http` connection scheme. + +{% /codeInfo %} + + +{% codeInfo srNumber=36 %} + +**secure**: Establish secure connection with ClickHouse. ClickHouse supports secure communication over SSL/TLS to protect data in transit, by checking this option, it establishes secure connection with ClickHouse. This flag is useful when you are using `clickhouse+native` connection scheme. + +{% /codeInfo %} + +{% codeInfo srNumber=37 %} + +**keyfile**: The key file path is the location when ClickHouse looks for a file containing the private key needed for secure communication over SSL/TLS. By default, ClickHouse will look for the key file in the `/etc/clickhouse-server directory`, with the file name `server.key`. However, this can be customized in the ClickHouse configuration file (`config.xml`). This flag is useful when you are using `clickhouse+native` connection scheme and the secure connection flag is enabled. + {% /codeInfo %} @@ -223,6 +242,15 @@ source: ```yaml {% srNumber=6 %} # scheme: clickhouse+http (default), or clickhouse+native ``` +```yaml {% srNumber=35 %} + # https: false +``` +```yaml {% srNumber=36 %} + # secure: true +``` +```yaml {% srNumber=37 %} + # keyfile: /etc/clickhouse-server/server.key +``` ```yaml {% srNumber=7 %} # connectionOptions: # key: value diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json index 81288b78725..b3cfd418856 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json @@ -63,6 +63,11 @@ "description": "Clickhouse SQL connection duration.", "type": "integer" }, + "https": { + "title": "Use HTTPS Protocol", + "description": "Use HTTPS Protocol for connection with clickhouse", + "type": "boolean" + }, "secure": { "title": "Secure Connection", "description": "Establish secure connection with clickhouse", @@ -71,7 +76,7 @@ "keyfile": { "title": "Key File Path", "description": "Path to key file for establishing secure connection", - "type": "boolean" + "type": "string" }, "connectionOptions": { "title": "Connection Options", diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Clickhouse.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Clickhouse.md index 91fe18997dd..d34a0e26e3a 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Clickhouse.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Clickhouse.md @@ -85,12 +85,18 @@ The duration of an SQL connection in ClickHouse depends on the configuration of Connections are kept open for as long as needed to complete a query, but they can also be closed based on duration set. $$ +$$section +### Use HTTPS Protocol $(id="https") + +Enable this flag when the when the Clickhouse instance is hosted via HTTPS protocol. This flag is useful when you are using `clickhouse+http` connection scheme. +$$ + $$section ### Secure $(id="secure") Establish secure connection with ClickHouse. -ClickHouse supports secure communication over SSL/TLS to protect data in transit, by checking this option, it establishes secure connection with ClickHouse +ClickHouse supports secure communication over SSL/TLS to protect data in transit, by checking this option, it establishes secure connection with ClickHouse. This flag is useful when you are using `clickhouse+native` connection scheme. $$ $$section