From b22192575d53362f2576ef67ecf89eecc5558825 Mon Sep 17 00:00:00 2001 From: Nahuel Date: Mon, 8 May 2023 09:35:49 +0200 Subject: [PATCH] Fix#11216: Use Enum for SASL Mechanism for Kafka and Redpanda connections (#11385) * Fix#11216: Use ENUM for SASL Mechanism for Kafka and Redpanda connections * Minor change * Address PR comments * Fix postgres migration --- .../v010__create_db_connection_info.sql | 6 ++++++ .../v010__create_db_connection_info.sql | 6 ++++++ .../ingestion/source/messaging/kafka/connection.py | 2 +- .../connections/messaging/kafkaConnection.json | 3 ++- .../connections/messaging/redpandaConnection.json | 3 ++- .../connections/messaging/saslMechanismType.json | 12 ++++++++++++ 6 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 bootstrap/sql/com.mysql.cj.jdbc.Driver/v010__create_db_connection_info.sql create mode 100644 bootstrap/sql/org.postgresql.Driver/v010__create_db_connection_info.sql create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/saslMechanismType.json diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v010__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v010__create_db_connection_info.sql new file mode 100644 index 00000000000..9f996135ed8 --- /dev/null +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v010__create_db_connection_info.sql @@ -0,0 +1,6 @@ +-- Updating the value of SASL Mechanism for Kafka and Redpanda connections +UPDATE messaging_service_entity +SET json = JSON_REPLACE(json, '$.connection.config.saslMechanism', 'PLAIN') +WHERE (serviceType = 'Kafka' OR serviceType = 'Redpanda') + AND JSON_EXTRACT(json, '$.connection.config.saslMechanism') IS NOT NULL + AND JSON_EXTRACT(json, '$.connection.config.saslMechanism') NOT IN ('GSSAPI', 'PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512', 'OAUTHBEARER'); \ No newline at end of file diff --git a/bootstrap/sql/org.postgresql.Driver/v010__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v010__create_db_connection_info.sql new file mode 100644 index 00000000000..142982291d1 --- /dev/null +++ b/bootstrap/sql/org.postgresql.Driver/v010__create_db_connection_info.sql @@ -0,0 +1,6 @@ +-- Updating the value of SASL Mechanism for Kafka and Redpanda connections +UPDATE messaging_service_entity +SET json = JSONB_SET(json::jsonb, '{connection,config,saslMechanism}', '"PLAIN"') +WHERE (servicetype = 'Kafka' OR serviceType = 'Redpanda') + AND json#>'{connection,config,saslMechanism}' IS NOT NULL + AND json#>'{connection,config,saslMechanism}' NOT IN ('"GSSAPI"', '"PLAIN"', '"SCRAM-SHA-256"', '"SCRAM-SHA-512"', '"OAUTHBEARER"'); \ No newline at end of file diff --git a/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py b/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py index e19360fc948..0e4b252275d 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py +++ b/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py @@ -58,7 +58,7 @@ def get_connection( "sasl.password" ] = connection.saslPassword.get_secret_value() if connection.saslMechanism: - connection.consumerConfig["sasl.mechanism"] = connection.saslMechanism + connection.consumerConfig["sasl.mechanism"] = connection.saslMechanism.value if connection.basicAuthUserInfo: connection.schemaRegistryConfig = connection.schemaRegistryConfig or {} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json index c1941dd47a3..2ea66d1b8f5 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json @@ -45,7 +45,8 @@ "saslMechanism": { "title": "SASL Mechanism", "description": "sasl.mechanism Consumer Config property", - "type": "string" + "$ref": "saslMechanismType.json", + "default": "PLAIN" }, "basicAuthUserInfo": { "title": "Basic Auth User Info", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/redpandaConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/redpandaConnection.json index bc24135cfbd..1f6cf10c2c8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/redpandaConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/redpandaConnection.json @@ -45,7 +45,8 @@ "saslMechanism": { "title": "SASL Mechanism", "description": "sasl.mechanism Consumer Config property", - "type": "string" + "$ref": "saslMechanismType.json", + "default": "PLAIN" }, "basicAuthUserInfo": { "title": "Basic Auth User Info", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/saslMechanismType.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/saslMechanismType.json new file mode 100644 index 00000000000..d17bdca4dbd --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/saslMechanismType.json @@ -0,0 +1,12 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/messaging/saslMechanismType.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SaslMechanismType", + "javaType": "org.openmetadata.schema.services.connections.messaging.SaslMechanismType", + "description": "SASL Mechanism consumer config property", + "type": "string", + "enum": ["PLAIN", "GSSAPI", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"], + "default": "PLAIN" +} + +