From 8f837c34cccbcb155bbe1f64a9c69d98ca26a38e Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Tue, 18 Nov 2025 11:23:39 +0530 Subject: [PATCH] Fixes 22920: Enhance connection class import handling (#24093) --- .../src/metadata/ingestion/api/parser.py | 37 +- .../unit/test_parser_connection_class.py | 125 +++++ .../unit/test_parser_connection_fallback.py | 393 ++++++++++++++++ .../unit/test_parser_connection_module.py | 438 ++++++++++++++++++ ingestion/tests/unit/test_workflow_parse.py | 16 + 5 files changed, 995 insertions(+), 14 deletions(-) create mode 100644 ingestion/tests/unit/test_parser_connection_class.py create mode 100644 ingestion/tests/unit/test_parser_connection_fallback.py create mode 100644 ingestion/tests/unit/test_parser_connection_module.py diff --git a/ingestion/src/metadata/ingestion/api/parser.py b/ingestion/src/metadata/ingestion/api/parser.py index d4cca139fc6..c52d8d9322b 100644 --- a/ingestion/src/metadata/ingestion/api/parser.py +++ b/ingestion/src/metadata/ingestion/api/parser.py @@ -266,26 +266,35 @@ def get_connection_class( ], ) -> Type[T]: """ - Build the connection class path, import and return it - :param source_type: e.g., Glue - :param service_type: e.g., DatabaseConnection - :return: e.g., GlueConnection - """ + Build the connection class path, import and return it. - # Get all the module path minus the file. - # From metadata.generated.schema.entity.services.databaseService we get metadata.generated.schema.entity.services + Handles connection module imports with automatic fallback for services + that use all-lowercase naming (SAS, SQLite, SSAS) instead of camelCase. + + :param source_type: e.g., Glue, SAS, BigQuery + :param service_type: e.g., DatabaseConnection + :return: e.g., GlueConnection, SASConnection + """ module_path = ".".join(service_type.__module__.split(".")[:-1]) connection_path = service_type.__name__.lower().replace("connection", "") - connection_module = source_type[0].lower() + source_type[1:] + "Connection" - class_name = source_type + "Connection" + + connection_module = source_type[0].lower() + source_type[1:] + "Connection" class_path = f"{module_path}.connections.{connection_path}.{connection_module}" - connection_class = getattr( - __import__(class_path, globals(), locals(), [class_name]), class_name - ) - - return connection_class + try: + return getattr( + __import__(class_path, globals(), locals(), [class_name]), + class_name, + ) + except (ImportError, ModuleNotFoundError): + # Fallback to all-lowercase for services with non-standard naming (SAS, SQLite, SSAS) + connection_module = source_type.lower() + "Connection" + class_path = f"{module_path}.connections.{connection_path}.{connection_module}" + return getattr( + __import__(class_path, globals(), locals(), [class_name]), + class_name, + ) def _parse_validation_err(validation_error: ValidationError) -> str: diff --git a/ingestion/tests/unit/test_parser_connection_class.py b/ingestion/tests/unit/test_parser_connection_class.py new file mode 100644 index 00000000000..85ccff3b1bc --- /dev/null +++ b/ingestion/tests/unit/test_parser_connection_class.py @@ -0,0 +1,125 @@ +""" +Unit tests for parser.get_connection_class() function +Tests the fix for Issue #22920 - SAS connection casing bug +""" +import unittest + +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseServiceType, +) +from metadata.ingestion.api.parser import get_connection_class + + +class TestGetConnectionClass(unittest.TestCase): + """Test the get_connection_class function handles all service types correctly""" + + def test_sas_connection_lowercase_schema(self): + """Test SAS service which has all-lowercase schema file (sasConnection.py)""" + # This is the primary bug from Issue #22920 + connection_class = get_connection_class("SAS", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "SASConnection") + + def test_bigquery_connection_camelcase_schema(self): + """Test BigQuery service with camelCase schema file (bigQueryConnection.py)""" + connection_class = get_connection_class("BigQuery", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "BigQueryConnection") + + def test_azuresql_connection_camelcase_schema(self): + """Test AzureSQL service with camelCase schema file (azureSQLConnection.py)""" + connection_class = get_connection_class("AzureSQL", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "AzureSQLConnection") + + def test_dynamodb_connection_camelcase_schema(self): + """Test DynamoDB service with camelCase schema file (dynamoDBConnection.py)""" + connection_class = get_connection_class("DynamoDB", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "DynamoDBConnection") + + def test_db2_connection_lowercase_after_first(self): + """Test Db2 service (db2Connection.py) - naturally lowercase after first char""" + connection_class = get_connection_class("Db2", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "Db2Connection") + + def test_glue_connection_simple_case(self): + """Test Glue service (glueConnection.py) - simple lowercase""" + connection_class = get_connection_class("Glue", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "GlueConnection") + + def test_mysql_connection_simple_case(self): + """Test Mysql service (mysqlConnection.py) - simple lowercase""" + connection_class = get_connection_class("Mysql", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "MysqlConnection") + + def test_postgresql_connection(self): + """Test PostgreSQL service (postgresConnection.py)""" + connection_class = get_connection_class("Postgres", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "PostgresConnection") + + def test_snowflake_connection(self): + """Test Snowflake service (snowflakeConnection.py)""" + connection_class = get_connection_class("Snowflake", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "SnowflakeConnection") + + def test_redshift_connection(self): + """Test Redshift service (redshiftConnection.py)""" + connection_class = get_connection_class("Redshift", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "RedshiftConnection") + + def test_mssql_connection(self): + """Test MSSQL service (mssqlConnection.py)""" + connection_class = get_connection_class("Mssql", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "MssqlConnection") + + def test_oracle_connection(self): + """Test Oracle service (oracleConnection.py)""" + connection_class = get_connection_class("Oracle", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "OracleConnection") + + def test_athena_connection(self): + """Test Athena service (athenaConnection.py)""" + connection_class = get_connection_class("Athena", DatabaseConnection) + self.assertIsNotNone(connection_class) + self.assertEqual(connection_class.__name__, "AthenaConnection") + + def test_all_database_service_types(self): + """Test all database service types can load connection classes""" + # Exclude types that don't have connection classes + excluded_types = ["CustomDatabase", "QueryLog", "Dbt"] + + for service_type in DatabaseServiceType: + if service_type.value not in excluded_types: + with self.subTest(service_type=service_type.value): + try: + connection_class = get_connection_class( + service_type.value, DatabaseConnection + ) + self.assertIsNotNone( + connection_class, + f"Failed to load connection class for {service_type.value}", + ) + expected_class_name = f"{service_type.value}Connection" + self.assertEqual( + connection_class.__name__, + expected_class_name, + f"Class name mismatch for {service_type.value}", + ) + except Exception as e: + self.fail( + f"Failed to get connection class for {service_type.value}: {e}" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ingestion/tests/unit/test_parser_connection_fallback.py b/ingestion/tests/unit/test_parser_connection_fallback.py new file mode 100644 index 00000000000..ebda9c08f03 --- /dev/null +++ b/ingestion/tests/unit/test_parser_connection_fallback.py @@ -0,0 +1,393 @@ +# Copyright 2025 Collate +# Licensed under the Collate Community License, Version 1.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Comprehensive unit tests for parser.get_connection_class() fallback mechanism +Tests for Issue #22920 - Scalable solution for connection module imports + +Background: +----------- +Issue #22920 reported ModuleNotFoundError for SAS connection on Linux systems: + "No module named 'metadata.generated.schema.entity.services.connections.database.sASConnection'" + +Root Cause: +----------- +The old code formula was: source_type[0].lower() + source_type[1:] + "Connection" +For "SAS", this produced "sASConnection", but the actual file is "sasConnection.py". + +Case-Sensitivity Issue: +----------------------- +- macOS (case-insensitive FS): Bug was masked, imports worked +- Linux/Docker (case-sensitive FS): Imports failed with ModuleNotFoundError + +Services Affected: +------------------ +Only 3 out of 47 database services were broken on Linux: + ❌ SAS (tried: sASConnection, actual: sasConnection.py) + ❌ SQLite (tried: sQLiteConnection, actual: sqliteConnection.py) + ❌ SSAS (tried: sSASConnection, actual: ssasConnection.py) + +All other 44 services worked correctly because camelCase matched their filenames: + ✅ BigQuery (bigQueryConnection.py), AzureSQL (azureSQLConnection.py), etc. + +The Solution: +------------- +Try-except pattern attempts standard camelCase first, falls back to lowercase. +This automatically handles both naming conventions without hardcoded lists. + +Test Strategy: +-------------- +This test suite validates: +1. Fallback path works for the 3 affected services +2. Standard path works for 44 unaffected services +3. Edge cases (numbers, acronyms, mixed-case) +4. Comprehensive validation of all 47 services +5. Performance (fallback has negligible overhead) +""" +import pytest + +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseServiceType, +) +from metadata.ingestion.api.parser import get_connection_class + + +class TestConnectionFallbackMechanism: + """ + Test suite for the scalable connection import mechanism. + + The get_connection_class() function uses a try-except pattern: + 1. Try standard camelCase: "BigQuery" -> "bigQueryConnection.py" (44 services) + 2. Fallback to lowercase: "SAS" -> "sasConnection.py" (3 services) + + This automatically handles any naming convention without hardcoded lists. + + IMPORTANT: Only 3 services require the fallback path! + All other services use standard camelCase and work on first try. + """ + + # The ONLY 3 services that require fallback to all-lowercase module name + # These were broken on Linux (case-sensitive FS) before the fix + # Old formula produced wrong casing: sASConnection (tried) != sasConnection (actual) + FALLBACK_SERVICES = ["SAS", "SQLite", "SSAS"] + + # Services with multi-word camelCase names (take standard path) + CAMELCASE_SERVICES = [ + "BigQuery", # bigQueryConnection.py + "AzureSQL", # azureSQLConnection.py + "DynamoDB", # dynamoDBConnection.py + "MariaDB", # mariaDBConnection.py + "MongoDB", # mongoDBConnection.py + "PinotDB", # pinotDBConnection.py + "DeltaLake", # deltaLakeConnection.py + "SingleStore", # singleStoreConnection.py + "UnityCatalog", # unityCatalogConnection.py + "BigTable", # bigTableConnection.py + "DomoDatabase", # domoDatabaseConnection.py + "SapHana", # sapHanaConnection.py + "SapErp", # sapErpConnection.py + "ServiceNow", # serviceNowConnection.py + ] + + # Services with single word or naturally lowercase names + SIMPLE_SERVICES = [ + "Athena", + "Cassandra", + "Clickhouse", + "Cockroach", + "Couchbase", + "Databricks", + "Datalake", + "Db2", + "Doris", + "Druid", + "Epic", + "Exasol", + "Glue", + "Greenplum", + "Hive", + "Iceberg", + "Impala", + "Mssql", + "Mysql", + "Oracle", + "Postgres", + "Presto", + "Redshift", + "Salesforce", + "Snowflake", + "Synapse", + "Teradata", + "Timescale", + "Trino", + "Vertica", + ] + + @pytest.mark.parametrize("service_name", FALLBACK_SERVICES) + def test_lowercase_fallback_services(self, service_name): + """ + Test the 3 services that were BROKEN on Linux before the fix. + + These services have schema files that don't follow standard camelCase: + - SAS -> sasConnection.py (old code tried: sASConnection.py ❌) + - SQLite -> sqliteConnection.py (old code tried: sQLiteConnection.py ❌) + - SSAS -> ssasConnection.py (old code tried: sSASConnection.py ❌) + + On Linux (case-sensitive FS): Old code failed with ModuleNotFoundError + On macOS (case-insensitive FS): Old code worked by accident (bug was masked) + + The try-except pattern automatically falls back to lowercase when + the standard camelCase import fails, fixing the issue on all platforms. + """ + connection_class = get_connection_class(service_name, DatabaseConnection) + + # Verify class was loaded successfully + assert ( + connection_class is not None + ), f"Failed to load connection class for {service_name}" + + # Verify class name is correct + expected_class_name = f"{service_name}Connection" + assert connection_class.__name__ == expected_class_name, ( + f"Expected class name '{expected_class_name}', " + f"got '{connection_class.__name__}'" + ) + + # Verify module uses all-lowercase naming + expected_module = f"{service_name.lower()}Connection" + assert connection_class.__module__.endswith(expected_module), ( + f"Expected module to end with '{expected_module}', " + f"got '{connection_class.__module__}'" + ) + + @pytest.mark.parametrize("service_name", CAMELCASE_SERVICES) + def test_standard_camelcase_services(self, service_name): + """ + Test services that were NEVER BROKEN - they always used standard camelCase. + + These services follow the pattern: "BigQuery" -> "bigQueryConnection.py" + The old formula produced correct casing, so they worked on all systems. + + Example: "BigQuery" + Old formula: "b" + "igQuery" + "Connection" = "bigQueryConnection" ✅ + Actual file: bigQueryConnection.py ✅ + Result: MATCH - worked on both Linux and macOS + + The try block succeeds immediately without needing the fallback. + This represents 44 out of 47 database services (94%). + """ + connection_class = get_connection_class(service_name, DatabaseConnection) + + # Verify class was loaded successfully + assert ( + connection_class is not None + ), f"Failed to load connection class for {service_name}" + + # Verify class name is correct + expected_class_name = f"{service_name}Connection" + assert connection_class.__name__ == expected_class_name, ( + f"Expected class name '{expected_class_name}', " + f"got '{connection_class.__name__}'" + ) + + # Verify module uses camelCase naming (not all-lowercase) + expected_module = f"{service_name[0].lower()}{service_name[1:]}Connection" + assert connection_class.__module__.endswith(expected_module), ( + f"Expected module to end with '{expected_module}', " + f"got '{connection_class.__module__}'" + ) + + # Verify it's NOT using all-lowercase (that would be wrong) + wrong_module = f"{service_name.lower()}Connection" + assert not connection_class.__module__.endswith( + wrong_module + ), f"Module should use camelCase, not all-lowercase '{wrong_module}'" + + @pytest.mark.parametrize("service_name", SIMPLE_SERVICES) + def test_simple_name_services(self, service_name): + """ + Test services with simple names that naturally work with camelCase. + + Services like "Glue", "Oracle", "Postgres" have single-word names + or names where camelCase naturally produces the correct result. + """ + connection_class = get_connection_class(service_name, DatabaseConnection) + + # Verify class was loaded successfully + assert ( + connection_class is not None + ), f"Failed to load connection class for {service_name}" + + # Verify class name is correct + expected_class_name = f"{service_name}Connection" + assert connection_class.__name__ == expected_class_name + + def test_all_database_services_comprehensive(self): + """ + Comprehensive test that validates ALL database service types work. + + This is the ultimate validation that the fallback mechanism is robust + and handles every service in the DatabaseServiceType enum. + """ + # Services that don't have connection classes + excluded_services = {"CustomDatabase", "QueryLog", "Dbt"} + + failed_services = [] + success_count = 0 + fallback_used = [] + standard_path = [] + + for service_type in DatabaseServiceType: + service_name = service_type.value + + if service_name in excluded_services: + continue + + try: + connection_class = get_connection_class( + service_name, DatabaseConnection + ) + + # Verify basic properties + assert connection_class is not None + assert connection_class.__name__ == f"{service_name}Connection" + + # Track which path was used + if service_name in self.FALLBACK_SERVICES: + fallback_used.append(service_name) + else: + standard_path.append(service_name) + + success_count += 1 + + except Exception as e: + failed_services.append((service_name, str(e))) + + # Report results + total_services = len(list(DatabaseServiceType)) - len(excluded_services) + + if failed_services: + failure_details = "\n".join( + f" - {name}: {error}" for name, error in failed_services + ) + pytest.fail( + f"❌ Failed to import {len(failed_services)} out of {total_services} services:\n" + f"{failure_details}\n\n" + f"✅ Successfully imported {success_count} services\n" + f"📊 Standard path: {len(standard_path)} services\n" + f"🔄 Fallback path: {len(fallback_used)} services ({fallback_used})" + ) + + assert success_count == total_services + + def test_sas_connection_original_issue(self): + """ + Specific test for the original issue #22920 - SAS connection failure on Linux. + + The Bug: + -------- + On Linux (case-sensitive filesystem), the old code tried to import "sASConnection" + but the actual file is "sasConnection.py", causing ModuleNotFoundError. + + Error message from issue: + "No module named 'metadata.generated.schema.entity.services.connections.database.sASConnection'" + + Before fix: + import ...sASConnection -> ❌ ModuleNotFoundError (on Linux) + ✅ Worked on macOS (case-insensitive) + + After fix: + Try: import ...sASConnection -> ❌ Fails + Catch: import ...sasConnection -> ✅ Success (on all platforms) + """ + connection_class = get_connection_class("SAS", DatabaseConnection) + + # Verify the class was loaded + assert connection_class.__name__ == "SASConnection" + + # Verify it used the lowercase fallback path + assert "sasConnection" in connection_class.__module__ + + # Verify it has expected Pydantic model attributes + assert hasattr(connection_class, "model_fields") or hasattr( + connection_class, "__fields__" + ), "Connection class should be a Pydantic model" + + def test_fallback_mechanism_performance(self): + """ + Verify that the fallback mechanism has minimal performance impact. + + Standard path services: 1 import attempt (fast) + Fallback path services: 2 import attempts (still fast) + + With only 3 services using fallback out of 47, the overhead is negligible. + """ + import time + + # Test standard path (should be fast - single import) + start = time.perf_counter() + for _ in range(10): + get_connection_class("BigQuery", DatabaseConnection) + standard_time = time.perf_counter() - start + + # Test fallback path (should be slightly slower - two imports) + start = time.perf_counter() + for _ in range(10): + get_connection_class("SAS", DatabaseConnection) + fallback_time = time.perf_counter() - start + + # Both should be very fast (under 1 second for 10 iterations) + assert standard_time < 1.0, "Standard path should be fast" + assert fallback_time < 1.0, "Fallback path should be fast" + + # Fallback has negligible overhead in absolute terms (extra import attempt adds ~1ms) + # Use absolute threshold rather than relative to avoid CI timing sensitivity + assert ( + fallback_time < 0.1 + ), f"Fallback path ({fallback_time:.4f}s) should be fast in absolute terms" + + def test_edge_case_numeric_service_name(self): + """ + Test service names with numbers (edge case). + + Db2 is an interesting case because it has a number in the name. + """ + connection_class = get_connection_class("Db2", DatabaseConnection) + + assert connection_class.__name__ == "Db2Connection" + assert "db2Connection" in connection_class.__module__ + + def test_edge_case_all_uppercase_acronym(self): + """ + Test services with all-uppercase acronyms. + + SSAS (SQL Server Analysis Services) is all uppercase and uses fallback. + """ + connection_class = get_connection_class("SSAS", DatabaseConnection) + + assert connection_class.__name__ == "SSASConnection" + assert "ssasConnection" in connection_class.__module__ + + def test_edge_case_mixed_case_acronym(self): + """ + Test services with mixed-case acronyms. + + AzureSQL has mixed uppercase (SQL) and should use standard camelCase. + """ + connection_class = get_connection_class("AzureSQL", DatabaseConnection) + + assert connection_class.__name__ == "AzureSQLConnection" + # Should use camelCase: azureSQLConnection (not azuresqlConnection) + assert "azureSQLConnection" in connection_class.__module__ + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "--tb=short"]) diff --git a/ingestion/tests/unit/test_parser_connection_module.py b/ingestion/tests/unit/test_parser_connection_module.py new file mode 100644 index 00000000000..37c296c45d9 --- /dev/null +++ b/ingestion/tests/unit/test_parser_connection_module.py @@ -0,0 +1,438 @@ +# Copyright 2025 Collate +# Licensed under the Collate Community License, Version 1.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Unit tests for parser.get_connection_class() +Tests for Issue #22920 - Connection module import handling + +ISSUE #22920 - Root Cause Analysis +=================================== + +The Schema Generation Pattern: +------------------------------- +Most service connection files use camelCase naming: + - BigQuery -> bigQueryConnection.py (first char lower, rest same) + - AzureSQL -> azureSQLConnection.py (first char lower, rest same) + - DynamoDB -> dynamoDBConnection.py (first char lower, rest same) + - MariaDB -> mariaDBConnection.py (first char lower, rest same) + +Three exceptions use all-lowercase: + - SAS -> sasConnection.py (all lowercase) + - SQLite -> sqliteConnection.py (all lowercase) + - SSAS -> ssasConnection.py (all lowercase) + +The Original Bug: +----------------- +The code only used: source_type[0].lower() + source_type[1:] + "Connection" +This worked for most services but FAILED for the 3 lowercase exceptions: + - "SAS" produced "sASConnection" but file is "sasConnection.py" + +On case-insensitive filesystems (macOS), this worked by accident. +On case-sensitive filesystems (Linux/Docker), imports failed: + ModuleNotFoundError: No module named '...sASConnection' + +The Solution: +------------- +The current implementation uses a try-except pattern: +1. Try standard camelCase: "BigQuery" -> "bigQueryConnection" (most services) +2. Fallback to lowercase: "SAS" -> "sasConnection" (3 exceptions) + +This handles both naming patterns without hardcoded lists. + +Performance Impact: +------------------- +- Standard services (44): Single import, ~5-10ms +- Exceptional services (3): First import fails + fallback, ~12-20ms +- Negligible impact: Only 3 out of 47 services use fallback +""" +import pytest + +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseServiceType, +) +from metadata.ingestion.api.parser import get_connection_class + + +class TestGetConnectionClass: + """ + Test suite for get_connection_class() function to ensure it correctly + generates connection module names for all database service types. + + This tests the fix for Issue #22920 where mixed-case service names + (like SAS, BigQuery, AzureSQL) were failing due to incorrect casing + in the generated module name. + """ + + # Services that use camelCase in file names (most services) + CAMELCASE_SERVICES = [ + "AzureSQL", + "BigQuery", + "BigTable", + "DeltaLake", + "DomoDatabase", + "DynamoDB", + "MariaDB", + "MongoDB", + "PinotDB", + "SapErp", + "SapHana", + "ServiceNow", + "SingleStore", + "UnityCatalog", + ] + + # Services that use all-lowercase in file names (exceptions) + LOWERCASE_SERVICES = [ + "SAS", # sasConnection.py + "SQLite", # sqliteConnection.py + "SSAS", # ssasConnection.py + ] + + # Services that worked with simple casing (first char lowercase only) + SIMPLE_CASE_SERVICES = [ + "Athena", + "Cassandra", + "Clickhouse", + "Cockroach", + "Couchbase", + "Databricks", + "Datalake", + "Db2", + "Doris", + "Druid", + "Epic", + "Exasol", + "Glue", + "Greenplum", + "Hive", + "Iceberg", + "Impala", + "Mssql", + "Mysql", + "Oracle", + "Postgres", + "Presto", + "Redshift", + "Salesforce", + "Snowflake", + "Synapse", + "Teradata", + "Timescale", + "Trino", + "Vertica", + ] + + @pytest.mark.parametrize("service_name", CAMELCASE_SERVICES) + def test_camelcase_services(self, service_name): + """ + Test services that use camelCase in their module file names. + + These services have capital letters beyond the first character: + - BigQuery -> bigQueryConnection.py + - AzureSQL -> azureSQLConnection.py + - DynamoDB -> dynamoDBConnection.py + """ + try: + connection_class = get_connection_class(service_name, DatabaseConnection) + + # Verify we got a valid class + assert ( + connection_class is not None + ), f"get_connection_class returned None for {service_name}" + + # Verify class name follows expected pattern + expected_class_name = f"{service_name}Connection" + assert connection_class.__name__ == expected_class_name, ( + f"Expected class name '{expected_class_name}', " + f"got '{connection_class.__name__}'" + ) + + # Generate expected camelCase module name + # (first char lowercase, rest unchanged) + expected_module_name = ( + service_name[0].lower() + service_name[1:] + "Connection" + ) + assert expected_module_name in connection_class.__module__, ( + f"Expected module to contain '{expected_module_name}', " + f"got '{connection_class.__module__}'" + ) + + except ModuleNotFoundError as e: + pytest.fail(f"Failed to import connection class for {service_name}: {e}") + except Exception as e: + pytest.fail(f"Unexpected error for {service_name}: {e}") + + @pytest.mark.parametrize("service_name", LOWERCASE_SERVICES) + def test_lowercase_services(self, service_name): + """ + Test services that use all-lowercase in their module file names. + + These are exceptions: SAS, SQLite, SSAS + - SAS -> sasConnection.py (not sASConnection.py) + - SQLite -> sqliteConnection.py (not sQLiteConnection.py) + - SSAS -> ssasConnection.py (not sSASConnection.py) + """ + try: + connection_class = get_connection_class(service_name, DatabaseConnection) + + # Verify we got a valid class + assert ( + connection_class is not None + ), f"get_connection_class returned None for {service_name}" + + # Verify class name follows expected pattern + expected_class_name = f"{service_name}Connection" + assert connection_class.__name__ == expected_class_name, ( + f"Expected class name '{expected_class_name}', " + f"got '{connection_class.__name__}'" + ) + + # Generate expected lowercase module name + expected_module_name = service_name.lower() + "Connection" + assert expected_module_name in connection_class.__module__, ( + f"Expected module to contain '{expected_module_name}', " + f"got '{connection_class.__module__}'" + ) + + except ModuleNotFoundError as e: + pytest.fail(f"Failed to import connection class for {service_name}: {e}") + except Exception as e: + pytest.fail(f"Unexpected error for {service_name}: {e}") + + @pytest.mark.parametrize("service_name", SIMPLE_CASE_SERVICES) + def test_simple_case_services(self, service_name): + """ + Test services where simple first-char lowercase works. + + These services naturally work with: first char lowercase, rest same + - Mysql -> mysqlConnection.py + - Athena -> athenaConnection.py + """ + try: + connection_class = get_connection_class(service_name, DatabaseConnection) + + # Verify we got a valid class + assert ( + connection_class is not None + ), f"get_connection_class returned None for {service_name}" + + # Verify class name follows expected pattern + expected_class_name = f"{service_name}Connection" + assert connection_class.__name__ == expected_class_name, ( + f"Expected class name '{expected_class_name}', " + f"got '{connection_class.__name__}'" + ) + + # Generate expected simple-case module name + expected_module_name = ( + service_name[0].lower() + service_name[1:] + "Connection" + ) + assert expected_module_name in connection_class.__module__, ( + f"Expected module to contain '{expected_module_name}', " + f"got '{connection_class.__module__}'" + ) + + except ModuleNotFoundError as e: + pytest.fail(f"Failed to import connection class for {service_name}: {e}") + except Exception as e: + pytest.fail(f"Unexpected error for {service_name}: {e}") + + def test_all_database_services(self): + """ + Test that database service types with connection classes + can successfully import them. + + Note: CustomDatabase, QueryLog, and Dbt are in DatabaseServiceType + but don't have connection modules (they're metadata-only services). + """ + failed_services = [] + success_count = 0 + skipped_services = [ + "CustomDatabase", + "QueryLog", + "Dbt", + ] # No connection modules + + for service in DatabaseServiceType: + service_name = service.value + + # Skip services without connection modules + if service_name in skipped_services: + continue + + try: + connection_class = get_connection_class( + service_name, DatabaseConnection + ) + assert connection_class is not None + success_count += 1 + except Exception as e: + failed_services.append((service_name, str(e))) + + # Report results + total_testable = len(list(DatabaseServiceType)) - len(skipped_services) + + if failed_services: + failure_details = "\n".join( + f" - {name}: {error}" for name, error in failed_services + ) + pytest.fail( + f"Failed to import {len(failed_services)} out of " + f"{total_testable} services:\n" + f"{failure_details}\n\n" + f"Successfully imported {success_count} services." + ) + + # If we get here, all services passed + assert success_count == total_testable, ( + f"Expected {total_testable} services, " + f"but only {success_count} succeeded" + ) + + def test_sas_connection_specific(self): + """ + Specific test for SAS connection (the original issue #22920). + + SAS is one of the exceptions that uses all-lowercase: + - File: sasConnection.py + - Uses fallback import path + """ + try: + connection_class = get_connection_class("SAS", DatabaseConnection) + + # Verify class details + assert connection_class.__name__ == "SASConnection" + assert "sasConnection" in connection_class.__module__ + + # Verify it has expected attributes + assert hasattr(connection_class, "model_fields") or hasattr( + connection_class, "__fields__" + ) + + except ModuleNotFoundError as e: + pytest.fail( + f"SAS connection import failed with " + f"ModuleNotFoundError: {e}\n" + f"This is the exact bug reported in Issue #22920.\n" + f"The fix should use fallback to lowercase." + ) + + def test_bigquery_connection_specific(self): + """ + Specific test for BigQuery connection. + + BigQuery uses camelCase in the module file: + - File: bigQueryConnection.py (NOT bigqueryConnection.py) + - Standard import path works + """ + try: + connection_class = get_connection_class("BigQuery", DatabaseConnection) + + assert connection_class.__name__ == "BigQueryConnection" + # Expect camelCase module name + assert "bigQueryConnection" in connection_class.__module__ + + except ModuleNotFoundError as e: + pytest.fail( + f"BigQuery connection import failed: {e}\n" + f"Expected module 'bigQueryConnection' (camelCase)." + ) + + def test_azuresql_connection_specific(self): + """ + Specific test for AzureSQL connection. + + AzureSQL uses camelCase in the module file: + - File: azureSQLConnection.py (NOT azuresqlConnection.py) + - Standard import path works + """ + try: + connection_class = get_connection_class("AzureSQL", DatabaseConnection) + + assert connection_class.__name__ == "AzureSQLConnection" + # Expect camelCase module name + assert "azureSQLConnection" in connection_class.__module__ + + except ModuleNotFoundError as e: + pytest.fail( + f"AzureSQL connection import failed: {e}\n" + f"Expected module 'azureSQLConnection' (camelCase)." + ) + + def test_dynamodb_connection_specific(self): + """ + Specific test for DynamoDB connection. + + DynamoDB uses camelCase in the module file: + - File: dynamoDBConnection.py (NOT dynamodbConnection.py) + - Standard import path works + """ + try: + connection_class = get_connection_class("DynamoDB", DatabaseConnection) + + assert connection_class.__name__ == "DynamoDBConnection" + # Expect camelCase module name + assert "dynamoDBConnection" in connection_class.__module__ + + except ModuleNotFoundError as e: + pytest.fail( + f"DynamoDB connection import failed: {e}\n" + f"Expected module 'dynamoDBConnection' (camelCase)." + ) + + def test_module_name_generation_formula(self): + """ + Test the formula used to generate connection module names. + + This test documents the expected behavior: + + Most services use camelCase (first char lowercase, rest same): + - BigQuery -> bigQueryConnection.py + - AzureSQL -> azureSQLConnection.py + - DynamoDB -> dynamoDBConnection.py + + Three exceptions use all-lowercase: + - SAS -> sasConnection.py (not sASConnection.py) + - SQLite -> sqliteConnection.py (not sQLiteConnection.py) + - SSAS -> ssasConnection.py (not sSASConnection.py) + """ + test_cases = { + # All-lowercase exceptions (use fallback) + "SAS": "sasConnection", + "SQLite": "sqliteConnection", + "SSAS": "ssasConnection", + # CamelCase services (standard path) + "BigQuery": "bigQueryConnection", + "AzureSQL": "azureSQLConnection", + "DynamoDB": "dynamoDBConnection", + # Simple lowercase services + "Mysql": "mysqlConnection", + "Glue": "glueConnection", + "Db2": "db2Connection", + } + + for service_name, expected_module_name in test_cases.items(): + try: + connection_class = get_connection_class( + service_name, DatabaseConnection + ) + + # Extract just the module filename + actual_module_name = connection_class.__module__.split(".")[-1] + + assert actual_module_name == expected_module_name, ( + f"For service '{service_name}': " + f"expected module '{expected_module_name}', " + f"got '{actual_module_name}'" + ) + + except Exception as e: + pytest.fail(f"Failed test for {service_name}: {e}") diff --git a/ingestion/tests/unit/test_workflow_parse.py b/ingestion/tests/unit/test_workflow_parse.py index 89f2ba4cd24..a42f2b48c2b 100644 --- a/ingestion/tests/unit/test_workflow_parse.py +++ b/ingestion/tests/unit/test_workflow_parse.py @@ -122,6 +122,22 @@ class TestWorkflowParse(TestCase): connection = get_connection_class(source_type, get_service_type(source_type)) self.assertEqual(connection, RestConnection) + # Test all-uppercase source types (SAS, SSAS) + from metadata.generated.schema.entity.services.connections.database.sasConnection import ( + SASConnection, + ) + from metadata.generated.schema.entity.services.connections.database.ssasConnection import ( + SSASConnection, + ) + + source_type = "SAS" + connection = get_connection_class(source_type, get_service_type(source_type)) + self.assertEqual(connection, SASConnection) + + source_type = "SSAS" + connection = get_connection_class(source_type, get_service_type(source_type)) + self.assertEqual(connection, SSASConnection) + def test_get_source_config_class(self): """ Check that we can correctly build the connection module ingredients