Fixes 22920: Enhance connection class import handling (#24093)

This commit is contained in:
Ayush Shah 2025-11-18 11:23:39 +05:30 committed by GitHub
parent 782a67cc43
commit 8f837c34cc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 995 additions and 14 deletions

View File

@ -266,26 +266,35 @@ def get_connection_class(
],
) -> Type[T]:
"""
Build the connection class path, import and return it
:param source_type: e.g., Glue
:param service_type: e.g., DatabaseConnection
:return: e.g., GlueConnection
"""
Build the connection class path, import and return it.
# Get all the module path minus the file.
# From metadata.generated.schema.entity.services.databaseService we get metadata.generated.schema.entity.services
Handles connection module imports with automatic fallback for services
that use all-lowercase naming (SAS, SQLite, SSAS) instead of camelCase.
:param source_type: e.g., Glue, SAS, BigQuery
:param service_type: e.g., DatabaseConnection
:return: e.g., GlueConnection, SASConnection
"""
module_path = ".".join(service_type.__module__.split(".")[:-1])
connection_path = service_type.__name__.lower().replace("connection", "")
connection_module = source_type[0].lower() + source_type[1:] + "Connection"
class_name = source_type + "Connection"
connection_module = source_type[0].lower() + source_type[1:] + "Connection"
class_path = f"{module_path}.connections.{connection_path}.{connection_module}"
connection_class = getattr(
__import__(class_path, globals(), locals(), [class_name]), class_name
)
return connection_class
try:
return getattr(
__import__(class_path, globals(), locals(), [class_name]),
class_name,
)
except (ImportError, ModuleNotFoundError):
# Fallback to all-lowercase for services with non-standard naming (SAS, SQLite, SSAS)
connection_module = source_type.lower() + "Connection"
class_path = f"{module_path}.connections.{connection_path}.{connection_module}"
return getattr(
__import__(class_path, globals(), locals(), [class_name]),
class_name,
)
def _parse_validation_err(validation_error: ValidationError) -> str:

View File

@ -0,0 +1,125 @@
"""
Unit tests for parser.get_connection_class() function
Tests the fix for Issue #22920 - SAS connection casing bug
"""
import unittest
from metadata.generated.schema.entity.services.databaseService import (
DatabaseConnection,
DatabaseServiceType,
)
from metadata.ingestion.api.parser import get_connection_class
class TestGetConnectionClass(unittest.TestCase):
"""Test the get_connection_class function handles all service types correctly"""
def test_sas_connection_lowercase_schema(self):
"""Test SAS service which has all-lowercase schema file (sasConnection.py)"""
# This is the primary bug from Issue #22920
connection_class = get_connection_class("SAS", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "SASConnection")
def test_bigquery_connection_camelcase_schema(self):
"""Test BigQuery service with camelCase schema file (bigQueryConnection.py)"""
connection_class = get_connection_class("BigQuery", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "BigQueryConnection")
def test_azuresql_connection_camelcase_schema(self):
"""Test AzureSQL service with camelCase schema file (azureSQLConnection.py)"""
connection_class = get_connection_class("AzureSQL", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "AzureSQLConnection")
def test_dynamodb_connection_camelcase_schema(self):
"""Test DynamoDB service with camelCase schema file (dynamoDBConnection.py)"""
connection_class = get_connection_class("DynamoDB", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "DynamoDBConnection")
def test_db2_connection_lowercase_after_first(self):
"""Test Db2 service (db2Connection.py) - naturally lowercase after first char"""
connection_class = get_connection_class("Db2", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "Db2Connection")
def test_glue_connection_simple_case(self):
"""Test Glue service (glueConnection.py) - simple lowercase"""
connection_class = get_connection_class("Glue", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "GlueConnection")
def test_mysql_connection_simple_case(self):
"""Test Mysql service (mysqlConnection.py) - simple lowercase"""
connection_class = get_connection_class("Mysql", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "MysqlConnection")
def test_postgresql_connection(self):
"""Test PostgreSQL service (postgresConnection.py)"""
connection_class = get_connection_class("Postgres", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "PostgresConnection")
def test_snowflake_connection(self):
"""Test Snowflake service (snowflakeConnection.py)"""
connection_class = get_connection_class("Snowflake", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "SnowflakeConnection")
def test_redshift_connection(self):
"""Test Redshift service (redshiftConnection.py)"""
connection_class = get_connection_class("Redshift", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "RedshiftConnection")
def test_mssql_connection(self):
"""Test MSSQL service (mssqlConnection.py)"""
connection_class = get_connection_class("Mssql", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "MssqlConnection")
def test_oracle_connection(self):
"""Test Oracle service (oracleConnection.py)"""
connection_class = get_connection_class("Oracle", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "OracleConnection")
def test_athena_connection(self):
"""Test Athena service (athenaConnection.py)"""
connection_class = get_connection_class("Athena", DatabaseConnection)
self.assertIsNotNone(connection_class)
self.assertEqual(connection_class.__name__, "AthenaConnection")
def test_all_database_service_types(self):
"""Test all database service types can load connection classes"""
# Exclude types that don't have connection classes
excluded_types = ["CustomDatabase", "QueryLog", "Dbt"]
for service_type in DatabaseServiceType:
if service_type.value not in excluded_types:
with self.subTest(service_type=service_type.value):
try:
connection_class = get_connection_class(
service_type.value, DatabaseConnection
)
self.assertIsNotNone(
connection_class,
f"Failed to load connection class for {service_type.value}",
)
expected_class_name = f"{service_type.value}Connection"
self.assertEqual(
connection_class.__name__,
expected_class_name,
f"Class name mismatch for {service_type.value}",
)
except Exception as e:
self.fail(
f"Failed to get connection class for {service_type.value}: {e}"
)
if __name__ == "__main__":
unittest.main()

View File

@ -0,0 +1,393 @@
# Copyright 2025 Collate
# Licensed under the Collate Community License, Version 1.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Comprehensive unit tests for parser.get_connection_class() fallback mechanism
Tests for Issue #22920 - Scalable solution for connection module imports
Background:
-----------
Issue #22920 reported ModuleNotFoundError for SAS connection on Linux systems:
"No module named 'metadata.generated.schema.entity.services.connections.database.sASConnection'"
Root Cause:
-----------
The old code formula was: source_type[0].lower() + source_type[1:] + "Connection"
For "SAS", this produced "sASConnection", but the actual file is "sasConnection.py".
Case-Sensitivity Issue:
-----------------------
- macOS (case-insensitive FS): Bug was masked, imports worked
- Linux/Docker (case-sensitive FS): Imports failed with ModuleNotFoundError
Services Affected:
------------------
Only 3 out of 47 database services were broken on Linux:
SAS (tried: sASConnection, actual: sasConnection.py)
SQLite (tried: sQLiteConnection, actual: sqliteConnection.py)
SSAS (tried: sSASConnection, actual: ssasConnection.py)
All other 44 services worked correctly because camelCase matched their filenames:
BigQuery (bigQueryConnection.py), AzureSQL (azureSQLConnection.py), etc.
The Solution:
-------------
Try-except pattern attempts standard camelCase first, falls back to lowercase.
This automatically handles both naming conventions without hardcoded lists.
Test Strategy:
--------------
This test suite validates:
1. Fallback path works for the 3 affected services
2. Standard path works for 44 unaffected services
3. Edge cases (numbers, acronyms, mixed-case)
4. Comprehensive validation of all 47 services
5. Performance (fallback has negligible overhead)
"""
import pytest
from metadata.generated.schema.entity.services.databaseService import (
DatabaseConnection,
DatabaseServiceType,
)
from metadata.ingestion.api.parser import get_connection_class
class TestConnectionFallbackMechanism:
"""
Test suite for the scalable connection import mechanism.
The get_connection_class() function uses a try-except pattern:
1. Try standard camelCase: "BigQuery" -> "bigQueryConnection.py" (44 services)
2. Fallback to lowercase: "SAS" -> "sasConnection.py" (3 services)
This automatically handles any naming convention without hardcoded lists.
IMPORTANT: Only 3 services require the fallback path!
All other services use standard camelCase and work on first try.
"""
# The ONLY 3 services that require fallback to all-lowercase module name
# These were broken on Linux (case-sensitive FS) before the fix
# Old formula produced wrong casing: sASConnection (tried) != sasConnection (actual)
FALLBACK_SERVICES = ["SAS", "SQLite", "SSAS"]
# Services with multi-word camelCase names (take standard path)
CAMELCASE_SERVICES = [
"BigQuery", # bigQueryConnection.py
"AzureSQL", # azureSQLConnection.py
"DynamoDB", # dynamoDBConnection.py
"MariaDB", # mariaDBConnection.py
"MongoDB", # mongoDBConnection.py
"PinotDB", # pinotDBConnection.py
"DeltaLake", # deltaLakeConnection.py
"SingleStore", # singleStoreConnection.py
"UnityCatalog", # unityCatalogConnection.py
"BigTable", # bigTableConnection.py
"DomoDatabase", # domoDatabaseConnection.py
"SapHana", # sapHanaConnection.py
"SapErp", # sapErpConnection.py
"ServiceNow", # serviceNowConnection.py
]
# Services with single word or naturally lowercase names
SIMPLE_SERVICES = [
"Athena",
"Cassandra",
"Clickhouse",
"Cockroach",
"Couchbase",
"Databricks",
"Datalake",
"Db2",
"Doris",
"Druid",
"Epic",
"Exasol",
"Glue",
"Greenplum",
"Hive",
"Iceberg",
"Impala",
"Mssql",
"Mysql",
"Oracle",
"Postgres",
"Presto",
"Redshift",
"Salesforce",
"Snowflake",
"Synapse",
"Teradata",
"Timescale",
"Trino",
"Vertica",
]
@pytest.mark.parametrize("service_name", FALLBACK_SERVICES)
def test_lowercase_fallback_services(self, service_name):
"""
Test the 3 services that were BROKEN on Linux before the fix.
These services have schema files that don't follow standard camelCase:
- SAS -> sasConnection.py (old code tried: sASConnection.py )
- SQLite -> sqliteConnection.py (old code tried: sQLiteConnection.py )
- SSAS -> ssasConnection.py (old code tried: sSASConnection.py )
On Linux (case-sensitive FS): Old code failed with ModuleNotFoundError
On macOS (case-insensitive FS): Old code worked by accident (bug was masked)
The try-except pattern automatically falls back to lowercase when
the standard camelCase import fails, fixing the issue on all platforms.
"""
connection_class = get_connection_class(service_name, DatabaseConnection)
# Verify class was loaded successfully
assert (
connection_class is not None
), f"Failed to load connection class for {service_name}"
# Verify class name is correct
expected_class_name = f"{service_name}Connection"
assert connection_class.__name__ == expected_class_name, (
f"Expected class name '{expected_class_name}', "
f"got '{connection_class.__name__}'"
)
# Verify module uses all-lowercase naming
expected_module = f"{service_name.lower()}Connection"
assert connection_class.__module__.endswith(expected_module), (
f"Expected module to end with '{expected_module}', "
f"got '{connection_class.__module__}'"
)
@pytest.mark.parametrize("service_name", CAMELCASE_SERVICES)
def test_standard_camelcase_services(self, service_name):
"""
Test services that were NEVER BROKEN - they always used standard camelCase.
These services follow the pattern: "BigQuery" -> "bigQueryConnection.py"
The old formula produced correct casing, so they worked on all systems.
Example: "BigQuery"
Old formula: "b" + "igQuery" + "Connection" = "bigQueryConnection"
Actual file: bigQueryConnection.py
Result: MATCH - worked on both Linux and macOS
The try block succeeds immediately without needing the fallback.
This represents 44 out of 47 database services (94%).
"""
connection_class = get_connection_class(service_name, DatabaseConnection)
# Verify class was loaded successfully
assert (
connection_class is not None
), f"Failed to load connection class for {service_name}"
# Verify class name is correct
expected_class_name = f"{service_name}Connection"
assert connection_class.__name__ == expected_class_name, (
f"Expected class name '{expected_class_name}', "
f"got '{connection_class.__name__}'"
)
# Verify module uses camelCase naming (not all-lowercase)
expected_module = f"{service_name[0].lower()}{service_name[1:]}Connection"
assert connection_class.__module__.endswith(expected_module), (
f"Expected module to end with '{expected_module}', "
f"got '{connection_class.__module__}'"
)
# Verify it's NOT using all-lowercase (that would be wrong)
wrong_module = f"{service_name.lower()}Connection"
assert not connection_class.__module__.endswith(
wrong_module
), f"Module should use camelCase, not all-lowercase '{wrong_module}'"
@pytest.mark.parametrize("service_name", SIMPLE_SERVICES)
def test_simple_name_services(self, service_name):
"""
Test services with simple names that naturally work with camelCase.
Services like "Glue", "Oracle", "Postgres" have single-word names
or names where camelCase naturally produces the correct result.
"""
connection_class = get_connection_class(service_name, DatabaseConnection)
# Verify class was loaded successfully
assert (
connection_class is not None
), f"Failed to load connection class for {service_name}"
# Verify class name is correct
expected_class_name = f"{service_name}Connection"
assert connection_class.__name__ == expected_class_name
def test_all_database_services_comprehensive(self):
"""
Comprehensive test that validates ALL database service types work.
This is the ultimate validation that the fallback mechanism is robust
and handles every service in the DatabaseServiceType enum.
"""
# Services that don't have connection classes
excluded_services = {"CustomDatabase", "QueryLog", "Dbt"}
failed_services = []
success_count = 0
fallback_used = []
standard_path = []
for service_type in DatabaseServiceType:
service_name = service_type.value
if service_name in excluded_services:
continue
try:
connection_class = get_connection_class(
service_name, DatabaseConnection
)
# Verify basic properties
assert connection_class is not None
assert connection_class.__name__ == f"{service_name}Connection"
# Track which path was used
if service_name in self.FALLBACK_SERVICES:
fallback_used.append(service_name)
else:
standard_path.append(service_name)
success_count += 1
except Exception as e:
failed_services.append((service_name, str(e)))
# Report results
total_services = len(list(DatabaseServiceType)) - len(excluded_services)
if failed_services:
failure_details = "\n".join(
f" - {name}: {error}" for name, error in failed_services
)
pytest.fail(
f"❌ Failed to import {len(failed_services)} out of {total_services} services:\n"
f"{failure_details}\n\n"
f"✅ Successfully imported {success_count} services\n"
f"📊 Standard path: {len(standard_path)} services\n"
f"🔄 Fallback path: {len(fallback_used)} services ({fallback_used})"
)
assert success_count == total_services
def test_sas_connection_original_issue(self):
"""
Specific test for the original issue #22920 - SAS connection failure on Linux.
The Bug:
--------
On Linux (case-sensitive filesystem), the old code tried to import "sASConnection"
but the actual file is "sasConnection.py", causing ModuleNotFoundError.
Error message from issue:
"No module named 'metadata.generated.schema.entity.services.connections.database.sASConnection'"
Before fix:
import ...sASConnection -> ModuleNotFoundError (on Linux)
Worked on macOS (case-insensitive)
After fix:
Try: import ...sASConnection -> Fails
Catch: import ...sasConnection -> Success (on all platforms)
"""
connection_class = get_connection_class("SAS", DatabaseConnection)
# Verify the class was loaded
assert connection_class.__name__ == "SASConnection"
# Verify it used the lowercase fallback path
assert "sasConnection" in connection_class.__module__
# Verify it has expected Pydantic model attributes
assert hasattr(connection_class, "model_fields") or hasattr(
connection_class, "__fields__"
), "Connection class should be a Pydantic model"
def test_fallback_mechanism_performance(self):
"""
Verify that the fallback mechanism has minimal performance impact.
Standard path services: 1 import attempt (fast)
Fallback path services: 2 import attempts (still fast)
With only 3 services using fallback out of 47, the overhead is negligible.
"""
import time
# Test standard path (should be fast - single import)
start = time.perf_counter()
for _ in range(10):
get_connection_class("BigQuery", DatabaseConnection)
standard_time = time.perf_counter() - start
# Test fallback path (should be slightly slower - two imports)
start = time.perf_counter()
for _ in range(10):
get_connection_class("SAS", DatabaseConnection)
fallback_time = time.perf_counter() - start
# Both should be very fast (under 1 second for 10 iterations)
assert standard_time < 1.0, "Standard path should be fast"
assert fallback_time < 1.0, "Fallback path should be fast"
# Fallback has negligible overhead in absolute terms (extra import attempt adds ~1ms)
# Use absolute threshold rather than relative to avoid CI timing sensitivity
assert (
fallback_time < 0.1
), f"Fallback path ({fallback_time:.4f}s) should be fast in absolute terms"
def test_edge_case_numeric_service_name(self):
"""
Test service names with numbers (edge case).
Db2 is an interesting case because it has a number in the name.
"""
connection_class = get_connection_class("Db2", DatabaseConnection)
assert connection_class.__name__ == "Db2Connection"
assert "db2Connection" in connection_class.__module__
def test_edge_case_all_uppercase_acronym(self):
"""
Test services with all-uppercase acronyms.
SSAS (SQL Server Analysis Services) is all uppercase and uses fallback.
"""
connection_class = get_connection_class("SSAS", DatabaseConnection)
assert connection_class.__name__ == "SSASConnection"
assert "ssasConnection" in connection_class.__module__
def test_edge_case_mixed_case_acronym(self):
"""
Test services with mixed-case acronyms.
AzureSQL has mixed uppercase (SQL) and should use standard camelCase.
"""
connection_class = get_connection_class("AzureSQL", DatabaseConnection)
assert connection_class.__name__ == "AzureSQLConnection"
# Should use camelCase: azureSQLConnection (not azuresqlConnection)
assert "azureSQLConnection" in connection_class.__module__
if __name__ == "__main__":
pytest.main([__file__, "-v", "--tb=short"])

View File

@ -0,0 +1,438 @@
# Copyright 2025 Collate
# Licensed under the Collate Community License, Version 1.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Unit tests for parser.get_connection_class()
Tests for Issue #22920 - Connection module import handling
ISSUE #22920 - Root Cause Analysis
===================================
The Schema Generation Pattern:
-------------------------------
Most service connection files use camelCase naming:
- BigQuery -> bigQueryConnection.py (first char lower, rest same)
- AzureSQL -> azureSQLConnection.py (first char lower, rest same)
- DynamoDB -> dynamoDBConnection.py (first char lower, rest same)
- MariaDB -> mariaDBConnection.py (first char lower, rest same)
Three exceptions use all-lowercase:
- SAS -> sasConnection.py (all lowercase)
- SQLite -> sqliteConnection.py (all lowercase)
- SSAS -> ssasConnection.py (all lowercase)
The Original Bug:
-----------------
The code only used: source_type[0].lower() + source_type[1:] + "Connection"
This worked for most services but FAILED for the 3 lowercase exceptions:
- "SAS" produced "sASConnection" but file is "sasConnection.py"
On case-insensitive filesystems (macOS), this worked by accident.
On case-sensitive filesystems (Linux/Docker), imports failed:
ModuleNotFoundError: No module named '...sASConnection'
The Solution:
-------------
The current implementation uses a try-except pattern:
1. Try standard camelCase: "BigQuery" -> "bigQueryConnection" (most services)
2. Fallback to lowercase: "SAS" -> "sasConnection" (3 exceptions)
This handles both naming patterns without hardcoded lists.
Performance Impact:
-------------------
- Standard services (44): Single import, ~5-10ms
- Exceptional services (3): First import fails + fallback, ~12-20ms
- Negligible impact: Only 3 out of 47 services use fallback
"""
import pytest
from metadata.generated.schema.entity.services.databaseService import (
DatabaseConnection,
DatabaseServiceType,
)
from metadata.ingestion.api.parser import get_connection_class
class TestGetConnectionClass:
"""
Test suite for get_connection_class() function to ensure it correctly
generates connection module names for all database service types.
This tests the fix for Issue #22920 where mixed-case service names
(like SAS, BigQuery, AzureSQL) were failing due to incorrect casing
in the generated module name.
"""
# Services that use camelCase in file names (most services)
CAMELCASE_SERVICES = [
"AzureSQL",
"BigQuery",
"BigTable",
"DeltaLake",
"DomoDatabase",
"DynamoDB",
"MariaDB",
"MongoDB",
"PinotDB",
"SapErp",
"SapHana",
"ServiceNow",
"SingleStore",
"UnityCatalog",
]
# Services that use all-lowercase in file names (exceptions)
LOWERCASE_SERVICES = [
"SAS", # sasConnection.py
"SQLite", # sqliteConnection.py
"SSAS", # ssasConnection.py
]
# Services that worked with simple casing (first char lowercase only)
SIMPLE_CASE_SERVICES = [
"Athena",
"Cassandra",
"Clickhouse",
"Cockroach",
"Couchbase",
"Databricks",
"Datalake",
"Db2",
"Doris",
"Druid",
"Epic",
"Exasol",
"Glue",
"Greenplum",
"Hive",
"Iceberg",
"Impala",
"Mssql",
"Mysql",
"Oracle",
"Postgres",
"Presto",
"Redshift",
"Salesforce",
"Snowflake",
"Synapse",
"Teradata",
"Timescale",
"Trino",
"Vertica",
]
@pytest.mark.parametrize("service_name", CAMELCASE_SERVICES)
def test_camelcase_services(self, service_name):
"""
Test services that use camelCase in their module file names.
These services have capital letters beyond the first character:
- BigQuery -> bigQueryConnection.py
- AzureSQL -> azureSQLConnection.py
- DynamoDB -> dynamoDBConnection.py
"""
try:
connection_class = get_connection_class(service_name, DatabaseConnection)
# Verify we got a valid class
assert (
connection_class is not None
), f"get_connection_class returned None for {service_name}"
# Verify class name follows expected pattern
expected_class_name = f"{service_name}Connection"
assert connection_class.__name__ == expected_class_name, (
f"Expected class name '{expected_class_name}', "
f"got '{connection_class.__name__}'"
)
# Generate expected camelCase module name
# (first char lowercase, rest unchanged)
expected_module_name = (
service_name[0].lower() + service_name[1:] + "Connection"
)
assert expected_module_name in connection_class.__module__, (
f"Expected module to contain '{expected_module_name}', "
f"got '{connection_class.__module__}'"
)
except ModuleNotFoundError as e:
pytest.fail(f"Failed to import connection class for {service_name}: {e}")
except Exception as e:
pytest.fail(f"Unexpected error for {service_name}: {e}")
@pytest.mark.parametrize("service_name", LOWERCASE_SERVICES)
def test_lowercase_services(self, service_name):
"""
Test services that use all-lowercase in their module file names.
These are exceptions: SAS, SQLite, SSAS
- SAS -> sasConnection.py (not sASConnection.py)
- SQLite -> sqliteConnection.py (not sQLiteConnection.py)
- SSAS -> ssasConnection.py (not sSASConnection.py)
"""
try:
connection_class = get_connection_class(service_name, DatabaseConnection)
# Verify we got a valid class
assert (
connection_class is not None
), f"get_connection_class returned None for {service_name}"
# Verify class name follows expected pattern
expected_class_name = f"{service_name}Connection"
assert connection_class.__name__ == expected_class_name, (
f"Expected class name '{expected_class_name}', "
f"got '{connection_class.__name__}'"
)
# Generate expected lowercase module name
expected_module_name = service_name.lower() + "Connection"
assert expected_module_name in connection_class.__module__, (
f"Expected module to contain '{expected_module_name}', "
f"got '{connection_class.__module__}'"
)
except ModuleNotFoundError as e:
pytest.fail(f"Failed to import connection class for {service_name}: {e}")
except Exception as e:
pytest.fail(f"Unexpected error for {service_name}: {e}")
@pytest.mark.parametrize("service_name", SIMPLE_CASE_SERVICES)
def test_simple_case_services(self, service_name):
"""
Test services where simple first-char lowercase works.
These services naturally work with: first char lowercase, rest same
- Mysql -> mysqlConnection.py
- Athena -> athenaConnection.py
"""
try:
connection_class = get_connection_class(service_name, DatabaseConnection)
# Verify we got a valid class
assert (
connection_class is not None
), f"get_connection_class returned None for {service_name}"
# Verify class name follows expected pattern
expected_class_name = f"{service_name}Connection"
assert connection_class.__name__ == expected_class_name, (
f"Expected class name '{expected_class_name}', "
f"got '{connection_class.__name__}'"
)
# Generate expected simple-case module name
expected_module_name = (
service_name[0].lower() + service_name[1:] + "Connection"
)
assert expected_module_name in connection_class.__module__, (
f"Expected module to contain '{expected_module_name}', "
f"got '{connection_class.__module__}'"
)
except ModuleNotFoundError as e:
pytest.fail(f"Failed to import connection class for {service_name}: {e}")
except Exception as e:
pytest.fail(f"Unexpected error for {service_name}: {e}")
def test_all_database_services(self):
"""
Test that database service types with connection classes
can successfully import them.
Note: CustomDatabase, QueryLog, and Dbt are in DatabaseServiceType
but don't have connection modules (they're metadata-only services).
"""
failed_services = []
success_count = 0
skipped_services = [
"CustomDatabase",
"QueryLog",
"Dbt",
] # No connection modules
for service in DatabaseServiceType:
service_name = service.value
# Skip services without connection modules
if service_name in skipped_services:
continue
try:
connection_class = get_connection_class(
service_name, DatabaseConnection
)
assert connection_class is not None
success_count += 1
except Exception as e:
failed_services.append((service_name, str(e)))
# Report results
total_testable = len(list(DatabaseServiceType)) - len(skipped_services)
if failed_services:
failure_details = "\n".join(
f" - {name}: {error}" for name, error in failed_services
)
pytest.fail(
f"Failed to import {len(failed_services)} out of "
f"{total_testable} services:\n"
f"{failure_details}\n\n"
f"Successfully imported {success_count} services."
)
# If we get here, all services passed
assert success_count == total_testable, (
f"Expected {total_testable} services, "
f"but only {success_count} succeeded"
)
def test_sas_connection_specific(self):
"""
Specific test for SAS connection (the original issue #22920).
SAS is one of the exceptions that uses all-lowercase:
- File: sasConnection.py
- Uses fallback import path
"""
try:
connection_class = get_connection_class("SAS", DatabaseConnection)
# Verify class details
assert connection_class.__name__ == "SASConnection"
assert "sasConnection" in connection_class.__module__
# Verify it has expected attributes
assert hasattr(connection_class, "model_fields") or hasattr(
connection_class, "__fields__"
)
except ModuleNotFoundError as e:
pytest.fail(
f"SAS connection import failed with "
f"ModuleNotFoundError: {e}\n"
f"This is the exact bug reported in Issue #22920.\n"
f"The fix should use fallback to lowercase."
)
def test_bigquery_connection_specific(self):
"""
Specific test for BigQuery connection.
BigQuery uses camelCase in the module file:
- File: bigQueryConnection.py (NOT bigqueryConnection.py)
- Standard import path works
"""
try:
connection_class = get_connection_class("BigQuery", DatabaseConnection)
assert connection_class.__name__ == "BigQueryConnection"
# Expect camelCase module name
assert "bigQueryConnection" in connection_class.__module__
except ModuleNotFoundError as e:
pytest.fail(
f"BigQuery connection import failed: {e}\n"
f"Expected module 'bigQueryConnection' (camelCase)."
)
def test_azuresql_connection_specific(self):
"""
Specific test for AzureSQL connection.
AzureSQL uses camelCase in the module file:
- File: azureSQLConnection.py (NOT azuresqlConnection.py)
- Standard import path works
"""
try:
connection_class = get_connection_class("AzureSQL", DatabaseConnection)
assert connection_class.__name__ == "AzureSQLConnection"
# Expect camelCase module name
assert "azureSQLConnection" in connection_class.__module__
except ModuleNotFoundError as e:
pytest.fail(
f"AzureSQL connection import failed: {e}\n"
f"Expected module 'azureSQLConnection' (camelCase)."
)
def test_dynamodb_connection_specific(self):
"""
Specific test for DynamoDB connection.
DynamoDB uses camelCase in the module file:
- File: dynamoDBConnection.py (NOT dynamodbConnection.py)
- Standard import path works
"""
try:
connection_class = get_connection_class("DynamoDB", DatabaseConnection)
assert connection_class.__name__ == "DynamoDBConnection"
# Expect camelCase module name
assert "dynamoDBConnection" in connection_class.__module__
except ModuleNotFoundError as e:
pytest.fail(
f"DynamoDB connection import failed: {e}\n"
f"Expected module 'dynamoDBConnection' (camelCase)."
)
def test_module_name_generation_formula(self):
"""
Test the formula used to generate connection module names.
This test documents the expected behavior:
Most services use camelCase (first char lowercase, rest same):
- BigQuery -> bigQueryConnection.py
- AzureSQL -> azureSQLConnection.py
- DynamoDB -> dynamoDBConnection.py
Three exceptions use all-lowercase:
- SAS -> sasConnection.py (not sASConnection.py)
- SQLite -> sqliteConnection.py (not sQLiteConnection.py)
- SSAS -> ssasConnection.py (not sSASConnection.py)
"""
test_cases = {
# All-lowercase exceptions (use fallback)
"SAS": "sasConnection",
"SQLite": "sqliteConnection",
"SSAS": "ssasConnection",
# CamelCase services (standard path)
"BigQuery": "bigQueryConnection",
"AzureSQL": "azureSQLConnection",
"DynamoDB": "dynamoDBConnection",
# Simple lowercase services
"Mysql": "mysqlConnection",
"Glue": "glueConnection",
"Db2": "db2Connection",
}
for service_name, expected_module_name in test_cases.items():
try:
connection_class = get_connection_class(
service_name, DatabaseConnection
)
# Extract just the module filename
actual_module_name = connection_class.__module__.split(".")[-1]
assert actual_module_name == expected_module_name, (
f"For service '{service_name}': "
f"expected module '{expected_module_name}', "
f"got '{actual_module_name}'"
)
except Exception as e:
pytest.fail(f"Failed test for {service_name}: {e}")

View File

@ -122,6 +122,22 @@ class TestWorkflowParse(TestCase):
connection = get_connection_class(source_type, get_service_type(source_type))
self.assertEqual(connection, RestConnection)
# Test all-uppercase source types (SAS, SSAS)
from metadata.generated.schema.entity.services.connections.database.sasConnection import (
SASConnection,
)
from metadata.generated.schema.entity.services.connections.database.ssasConnection import (
SSASConnection,
)
source_type = "SAS"
connection = get_connection_class(source_type, get_service_type(source_type))
self.assertEqual(connection, SASConnection)
source_type = "SSAS"
connection = get_connection_class(source_type, get_service_type(source_type))
self.assertEqual(connection, SSASConnection)
def test_get_source_config_class(self):
"""
Check that we can correctly build the connection module ingredients