Clickhouse: Make https protocol as permanent field (#12303)

bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql

@@ -64,3 +64,8 @@ where de2.serviceType = 'Mssql'
 -- column deleted not needed for entities that don't support soft delete
 ALTER TABLE query_entity DROP COLUMN deleted;
 ALTER TABLE event_subscription_entity DROP COLUMN deleted;
+
+-- remove keyfile from clickhouse
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.keyfile')
+WHERE serviceType = 'Clickhouse';

bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql

@@ -60,3 +60,8 @@ AND json->>'{connection,config,database}' IS NULL;
 -- column deleted not needed for entities that don't support soft delete
 ALTER TABLE query_entity DROP COLUMN deleted;
 ALTER TABLE event_subscription_entity DROP COLUMN deleted;
+
+-- remove keyfile from clickhouse
+UPDATE dbservice_entity
+SET json = json #-'{connection,config,keyfile}'
+WHERE serviceType = 'Clickhouse';

ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py

@@ -28,6 +28,7 @@ from metadata.ingestion.connections.builders import (
     get_connection_args_common,
     get_connection_url_common,
     init_empty_connection_arguments,
+    init_empty_connection_options,
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -35,18 +36,24 @@ from metadata.ingestion.source.database.clickhouse.queries import (
     CLICKHOUSE_SQL_STATEMENT_TEST,
 )
 
+HTTPS_PROTOCOL = "https"
+
 
 def get_connection(connection: ClickhouseConnection) -> Engine:
     """
     Create Clickhouse connection
     """
     if connection.secure or connection.keyfile:
-        if connection.connectionArguments:
+        if not connection.connectionArguments:
             connection.connectionArguments = init_empty_connection_arguments()
         if connection.secure:
             connection.connectionArguments.__root__["secure"] = connection.secure
         if connection.keyfile:
             connection.connectionArguments.__root__["keyfile"] = connection.keyfile
+    if connection.https:
+        if not connection.connectionOptions:
+            connection.connectionOptions = init_empty_connection_options()
+        connection.connectionOptions.__root__["protocol"] = HTTPS_PROTOCOL
 
     return create_generic_db_connection(
         connection=connection,

openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/index.md

@@ -91,18 +91,12 @@ For the usage and lineage workflow, the user will need `SELECT` privilege. You c
 - **Username**: Specify the User to connect to Clickhouse. It should have enough privileges to read all the metadata.
 - **Password**: Password to connect to Clickhouse.
 - **Host and Port**: Enter the fully qualified hostname and port number for your Clickhouse deployment in the Host and Port field.
+- **Use HTTPS Protocol**: Enable this flag when the when the Clickhouse instance is hosted via HTTPS protocol. This flag is useful when you are using `clickhouse+http` connection scheme.
+- **Secure Connection**: Establish secure connection with ClickHouse. ClickHouse supports secure communication over SSL/TLS to protect data in transit, by checking this option, it establishes secure connection with ClickHouse. This flag is useful when you are using `clickhouse+native` connection scheme.
+- **Key File**: The key file path is the location when ClickHouse looks for a file containing the private key needed for secure communication over SSL/TLS. By default, ClickHouse will look for the key file in the `/etc/clickhouse-server directory`, with the file name `server.key`. However, this can be customized in the ClickHouse configuration file (`config.xml`). This flag is useful when you are using `clickhouse+native` connection scheme and the secure connection flag is enabled.
 
 {% partial file="/v1.1.1/connectors/database/advanced-configuration.md" /%}
 
-You can find the full list of accepted options [here](https://clickhouse-driver.readthedocs.io/en/latest/api.html#clickhouse_driver.connection.Connection).
-
-- **Connecting to Clickhouse with SSL Certificate**: You will need to use the `clickhouse+native` connection scheme. Then in the `Connection Options` reference the following key with their value:
-  - `verify`: `true`
-  - `secure`: `true`
-  - `keyfile`: `/path/to/key/file`
-
-The `keyfile` needs to be accessible by the service running the ingestion. For example if you are running the ingestion in a docker container, your `keyfile` needs to be present in the container at the location specify as a value in the `Connection Options`. Additionally, your `keyfile` needs to be in the `.cert` or `.pem` format.
-
 {% /extraContent %}
 
 {% partial file="/v1.1.1/connectors/test-connection.md" /%}

openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/database/clickhouse/yaml.md

@@ -148,6 +148,25 @@ This is a sample config for Clickhouse:
 - **clickhouse+native**: Uses the native ClickHouse TCP protocol for communication. Faster than http, but may require additional server-side configuration. Recommended for performance-critical applications.
 
 
+{% /codeInfo %}
+
+{% codeInfo srNumber=35 %}
+
+**https**: Enable this flag when the when the Clickhouse instance is hosted via HTTPS protocol. This flag is useful when you are using `clickhouse+http` connection scheme.
+
+{% /codeInfo %}
+
+
+{% codeInfo srNumber=36 %}
+
+**secure**: Establish secure connection with ClickHouse. ClickHouse supports secure communication over SSL/TLS to protect data in transit, by checking this option, it establishes secure connection with ClickHouse. This flag is useful when you are using `clickhouse+native` connection scheme.
+
+{% /codeInfo %}
+
+{% codeInfo srNumber=37 %}
+
+**keyfile**: The key file path is the location when ClickHouse looks for a file containing the private key needed for secure communication over SSL/TLS. By default, ClickHouse will look for the key file in the `/etc/clickhouse-server directory`, with the file name `server.key`. However, this can be customized in the ClickHouse configuration file (`config.xml`). This flag is useful when you are using `clickhouse+native` connection scheme and the secure connection flag is enabled.
+
 {% /codeInfo %}
 
 
@@ -223,6 +242,15 @@ source:
 ```yaml {% srNumber=6 %}
       # scheme: clickhouse+http (default), or clickhouse+native
 ```
+```yaml {% srNumber=35 %}
+      # https: false
+```
+```yaml {% srNumber=36 %}
+      # secure: true
+```
+```yaml {% srNumber=37 %}
+      # keyfile: /etc/clickhouse-server/server.key
+```
 ```yaml {% srNumber=7 %}
       # connectionOptions:
       #   key: value

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json

@@ -63,6 +63,11 @@
       "description": "Clickhouse SQL connection duration.",
       "type": "integer"
     },
+    "https": {
+      "title": "Use HTTPS Protocol",
+      "description": "Use HTTPS Protocol for connection with clickhouse",
+      "type": "boolean"
+    },
     "secure": {
       "title": "Secure Connection",
       "description": "Establish secure connection with clickhouse",
@@ -71,7 +76,7 @@
     "keyfile": {
       "title": "Key File Path",
       "description": "Path to key file for establishing secure connection",
-      "type": "boolean"
+      "type": "string"
     },
     "connectionOptions": {
       "title": "Connection Options",

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Clickhouse.md

@@ -85,12 +85,18 @@ The duration of an SQL connection in ClickHouse depends on the configuration of
 Connections are kept open for as long as needed to complete a query, but they can also be closed based on duration set.
 $$
 
+$$section
+### Use HTTPS Protocol $(id="https")
+
+Enable this flag when the when the Clickhouse instance is hosted via HTTPS protocol. This flag is useful when you are using `clickhouse+http` connection scheme.
+$$
+
 $$section
 ### Secure $(id="secure")
 
 Establish secure connection with ClickHouse.
 
-ClickHouse supports secure communication over SSL/TLS to protect data in transit, by checking this option, it establishes secure connection with ClickHouse
+ClickHouse supports secure communication over SSL/TLS to protect data in transit, by checking this option, it establishes secure connection with ClickHouse. This flag is useful when you are using `clickhouse+native` connection scheme.
 $$
 
 $$section