diff --git a/ingestion/src/metadata/utils/ssl_manager.py b/ingestion/src/metadata/utils/ssl_manager.py index 8d50c354cfc..a9220ae71ad 100644 --- a/ingestion/src/metadata/utils/ssl_manager.py +++ b/ingestion/src/metadata/utils/ssl_manager.py @@ -43,6 +43,9 @@ from metadata.generated.schema.entity.services.connections.database.hiveConnecti from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import ( MongoDBConnection, ) +from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( + MssqlConnection, +) from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( MysqlConnection, ) @@ -275,6 +278,29 @@ class SSLManager: return connection + @setup_ssl.register(MssqlConnection) + def _(self, connection): + connection = cast(MssqlConnection, connection) + + if not connection.connectionArguments: + connection.connectionArguments = init_empty_connection_arguments() + + # Handle driver-specific SSL configuration + if connection.scheme.value == "mssql+pyodbc": + # ODBC Driver SSL parameters + if connection.encrypt: + connection.connectionArguments.root["Encrypt"] = "yes" + + if connection.trustServerCertificate: + connection.connectionArguments.root["TrustServerCertificate"] = "yes" + + elif connection.scheme.value == "mssql+pytds": + # pytds driver SSL parameters + if self.ca_file_path: + connection.connectionArguments.root["cafile"] = self.ca_file_path + + return connection + @singledispatch def check_ssl_and_init( @@ -327,6 +353,21 @@ def _(connection): return None +@check_ssl_and_init.register(MssqlConnection) +def _(connection): + service_connection = cast(MssqlConnection, connection) + ssl: Optional[ + verifySSLConfig.SslConfig + ] = service_connection.sslConfig or verifySSLConfig.SslConfig( + **{"caCertificate": None} + ) + return SSLManager( + ca=ssl.root.caCertificate, + cert=ssl.root.sslCertificate, + key=ssl.root.sslKey, + ) + + @check_ssl_and_init.register(MongoDBConnection) def _(connection): service_connection = cast(Union[MysqlConnection, DorisConnection], connection) diff --git a/ingestion/tests/unit/test_ssl_manager.py b/ingestion/tests/unit/test_ssl_manager.py index a48eac5000c..8ceda29f46a 100644 --- a/ingestion/tests/unit/test_ssl_manager.py +++ b/ingestion/tests/unit/test_ssl_manager.py @@ -210,3 +210,166 @@ class CassandraSourceSSLTest(TestCase): cassandra_source_with_ssl.service_connection.sslConfig.root.sslCertificate.get_secret_value(), "sslCertificateData", ) + + +class MssqlSSLManagerTest(TestCase): + """ + Tests for MSSQL SSL Manager functionality + """ + + def test_check_ssl_and_init_with_ssl_config(self): + """Test SSL manager initialization with sslConfig""" + from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( + MssqlConnection, + ) + from metadata.utils.ssl_manager import check_ssl_and_init + + connection_with_ssl = MssqlConnection( + hostPort="localhost:1433", + database="testdb", + username="sa", + password="password", + encrypt=False, + trustServerCertificate=False, + sslConfig={"caCertificate": "caCertificateData"}, + ) + + ssl_manager = check_ssl_and_init(connection_with_ssl) + + self.assertIsNotNone(ssl_manager) + self.assertIsNotNone(ssl_manager.ca_file_path) + + ssl_manager.cleanup_temp_files() + + def test_check_ssl_and_init_without_ssl_config(self): + """Test SSL manager initialization without sslConfig""" + from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( + MssqlConnection, + ) + from metadata.utils.ssl_manager import check_ssl_and_init + + connection_without_ssl = MssqlConnection( + hostPort="localhost:1433", + database="testdb", + username="sa", + password="password", + encrypt=True, + trustServerCertificate=True, + ) + + ssl_manager = check_ssl_and_init(connection_without_ssl) + + self.assertIsNone(ssl_manager.ca_file_path) + + def test_setup_ssl_pyodbc_driver(self): + """Test SSL setup for pyodbc driver""" + from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( + MssqlConnection, + MssqlScheme, + ) + + connection = MssqlConnection( + hostPort="localhost:1433", + database="testdb", + username="sa", + password="password", + scheme=MssqlScheme.mssql_pyodbc, + encrypt=True, + trustServerCertificate=False, + ) + + ssl_manager = SSLManager( + ca=SecretStr("CA cert"), cert=SecretStr("Cert"), key=SecretStr("Key") + ) + updated_connection = ssl_manager.setup_ssl(connection) + + self.assertIsNotNone(updated_connection.connectionArguments) + self.assertEqual( + updated_connection.connectionArguments.root.get("Encrypt"), "yes" + ) + self.assertIsNone( + updated_connection.connectionArguments.root.get("TrustServerCertificate") + ) + + ssl_manager.cleanup_temp_files() + + def test_setup_ssl_pyodbc_with_trust_certificate(self): + """Test SSL setup for pyodbc driver with trustServerCertificate""" + from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( + MssqlConnection, + MssqlScheme, + ) + + connection = MssqlConnection( + hostPort="localhost:1433", + database="testdb", + username="sa", + password="password", + scheme=MssqlScheme.mssql_pyodbc, + encrypt=True, + trustServerCertificate=True, + ) + + ssl_manager = SSLManager( + ca=SecretStr("CA cert"), cert=SecretStr("Cert"), key=SecretStr("Key") + ) + updated_connection = ssl_manager.setup_ssl(connection) + + self.assertIsNotNone(updated_connection.connectionArguments) + self.assertEqual( + updated_connection.connectionArguments.root.get("Encrypt"), "yes" + ) + self.assertEqual( + updated_connection.connectionArguments.root.get("TrustServerCertificate"), + "yes", + ) + + ssl_manager.cleanup_temp_files() + + def test_setup_ssl_pytds_driver(self): + """Test SSL setup for pytds driver""" + from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( + MssqlConnection, + MssqlScheme, + ) + from metadata.utils.ssl_manager import check_ssl_and_init + + connection = MssqlConnection( + hostPort="localhost:1433", + database="testdb", + username="sa", + password="password", + scheme=MssqlScheme.mssql_pytds, + sslConfig={"caCertificate": "caCertificateData"}, + ) + + ssl_manager = check_ssl_and_init(connection) + updated_connection = ssl_manager.setup_ssl(connection) + + self.assertIsNotNone(updated_connection.connectionArguments.root["cafile"]) + + ssl_manager.cleanup_temp_files() + + def test_setup_ssl_pymssql_driver(self): + """Test SSL setup for pymssql driver""" + from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( + MssqlConnection, + MssqlScheme, + ) + + connection = MssqlConnection( + hostPort="localhost:1433", + database="testdb", + username="sa", + password="password", + scheme=MssqlScheme.mssql_pymssql, + ) + + ssl_manager = SSLManager( + ca=SecretStr("CA cert"), cert=SecretStr("Cert"), key=SecretStr("Key") + ) + updated_connection = ssl_manager.setup_ssl(connection) + + self.assertDictEqual(updated_connection.connectionArguments.root, {}) + + ssl_manager.cleanup_temp_files() diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/DirectoryService.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/DirectoryService.java new file mode 100644 index 00000000000..4374ad437cb --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/DirectoryService.java @@ -0,0 +1,23 @@ +package org.openmetadata.sdk.services.drives; + +import org.openmetadata.schema.api.data.CreateDirectory; +import org.openmetadata.schema.entity.data.Directory; +import org.openmetadata.sdk.exceptions.OpenMetadataException; +import org.openmetadata.sdk.network.HttpClient; +import org.openmetadata.sdk.network.HttpMethod; +import org.openmetadata.sdk.services.EntityServiceBase; + +public class DirectoryService extends EntityServiceBase { + public DirectoryService(HttpClient httpClient) { + super(httpClient, "/v1/drives/directories"); + } + + @Override + protected Class getEntityClass() { + return Directory.class; + } + + public Directory create(CreateDirectory request) throws OpenMetadataException { + return httpClient.execute(HttpMethod.POST, basePath, request, Directory.class); + } +} diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/FileService.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/FileService.java new file mode 100644 index 00000000000..be1a110c793 --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/FileService.java @@ -0,0 +1,23 @@ +package org.openmetadata.sdk.services.drives; + +import org.openmetadata.schema.api.data.CreateFile; +import org.openmetadata.schema.entity.data.File; +import org.openmetadata.sdk.exceptions.OpenMetadataException; +import org.openmetadata.sdk.network.HttpClient; +import org.openmetadata.sdk.network.HttpMethod; +import org.openmetadata.sdk.services.EntityServiceBase; + +public class FileService extends EntityServiceBase { + public FileService(HttpClient httpClient) { + super(httpClient, "/v1/drives/files"); + } + + @Override + protected Class getEntityClass() { + return File.class; + } + + public File create(CreateFile request) throws OpenMetadataException { + return httpClient.execute(HttpMethod.POST, basePath, request, File.class); + } +} diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/SpreadsheetService.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/SpreadsheetService.java new file mode 100644 index 00000000000..7c8a8de5687 --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/SpreadsheetService.java @@ -0,0 +1,23 @@ +package org.openmetadata.sdk.services.drives; + +import org.openmetadata.schema.api.data.CreateSpreadsheet; +import org.openmetadata.schema.entity.data.Spreadsheet; +import org.openmetadata.sdk.exceptions.OpenMetadataException; +import org.openmetadata.sdk.network.HttpClient; +import org.openmetadata.sdk.network.HttpMethod; +import org.openmetadata.sdk.services.EntityServiceBase; + +public class SpreadsheetService extends EntityServiceBase { + public SpreadsheetService(HttpClient httpClient) { + super(httpClient, "/v1/drives/spreadsheets"); + } + + @Override + protected Class getEntityClass() { + return Spreadsheet.class; + } + + public Spreadsheet create(CreateSpreadsheet request) throws OpenMetadataException { + return httpClient.execute(HttpMethod.POST, basePath, request, Spreadsheet.class); + } +} diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/WorksheetService.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/WorksheetService.java new file mode 100644 index 00000000000..285820c4101 --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/services/drives/WorksheetService.java @@ -0,0 +1,23 @@ +package org.openmetadata.sdk.services.drives; + +import org.openmetadata.schema.api.data.CreateWorksheet; +import org.openmetadata.schema.entity.data.Worksheet; +import org.openmetadata.sdk.exceptions.OpenMetadataException; +import org.openmetadata.sdk.network.HttpClient; +import org.openmetadata.sdk.network.HttpMethod; +import org.openmetadata.sdk.services.EntityServiceBase; + +public class WorksheetService extends EntityServiceBase { + public WorksheetService(HttpClient httpClient) { + super(httpClient, "/v1/drives/worksheets"); + } + + @Override + protected Class getEntityClass() { + return Worksheet.class; + } + + public Worksheet create(CreateWorksheet request) throws OpenMetadataException { + return httpClient.execute(HttpMethod.POST, basePath, request, Worksheet.class); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index c85cd77897e..169fffd13f2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -36,6 +36,7 @@ import org.openmetadata.schema.services.connections.database.DeltaLakeConnection import org.openmetadata.schema.services.connections.database.GreenplumConnection; import org.openmetadata.schema.services.connections.database.HiveConnection; import org.openmetadata.schema.services.connections.database.IcebergConnection; +import org.openmetadata.schema.services.connections.database.MssqlConnection; import org.openmetadata.schema.services.connections.database.MysqlConnection; import org.openmetadata.schema.services.connections.database.PostgresConnection; import org.openmetadata.schema.services.connections.database.RedshiftConnection; @@ -87,6 +88,7 @@ public final class ClassConverterFactory { Map.entry(IcebergConnection.class, new IcebergConnectionClassConverter()), Map.entry(IcebergFileSystem.class, new IcebergFileSystemClassConverter()), Map.entry(LookerConnection.class, new LookerConnectionClassConverter()), + Map.entry(MssqlConnection.class, new MssqlConnectionClassConverter()), Map.entry(MysqlConnection.class, new MysqlConnectionClassConverter()), Map.entry(RedshiftConnection.class, new RedshiftConnectionClassConverter()), Map.entry(GreenplumConnection.class, new GreenplumConnectionClassConverter()), diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MssqlConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MssqlConnectionClassConverter.java new file mode 100644 index 00000000000..dc1e64fb933 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MssqlConnectionClassConverter.java @@ -0,0 +1,38 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.security.ssl.ValidateSSLClientConfig; +import org.openmetadata.schema.services.connections.database.MssqlConnection; +import org.openmetadata.schema.utils.JsonUtils; + +public class MssqlConnectionClassConverter extends ClassConverter { + + private static final List> SSL_SOURCE_CLASS = List.of(ValidateSSLClientConfig.class); + + public MssqlConnectionClassConverter() { + super(MssqlConnection.class); + } + + @Override + public Object convert(Object object) { + MssqlConnection mssqlConnection = (MssqlConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvert(mssqlConnection.getSslConfig(), SSL_SOURCE_CLASS) + .ifPresent(mssqlConnection::setSslConfig); + + return mssqlConnection; + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json index 5e756d2f251..c84e9690bc2 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json @@ -59,6 +59,23 @@ "type": "string", "default": "ODBC Driver 18 for SQL Server" }, + "encrypt": { + "title": "Encrypt Connection", + "description": "Enable SSL/TLS encryption for the MSSQL connection. When enabled, all data transmitted between the client and server will be encrypted.", + "type": "boolean", + "default": false + }, + "trustServerCertificate": { + "title": "Trust Server Certificate", + "description": "Trust the server certificate without validation. Set to false in production to validate server certificates against the certificate authority.", + "type": "boolean", + "default": false + }, + "sslConfig": { + "title": "SSL Configuration", + "description": "SSL/TLS certificate configuration for client authentication. Provide CA certificate, client certificate, and private key for mutual TLS authentication.", + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" + }, "ingestAllDatabases": { "title": "Ingest All Databases", "description": "Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this.", diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Mssql.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Mssql.md index 844f3ef09fd..66521365367 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Mssql.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Mssql.md @@ -97,6 +97,35 @@ You can download the ODBC driver from