diff --git a/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py b/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py index 2c6f43e3989..f27a3c475ea 100644 --- a/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py @@ -59,13 +59,13 @@ def test_connection( holder = SchemaHolder() - def test_get_databases(client: MongoClient, holder: SchemaHolder): - for database in client.list_database_names(): - holder.database = database + def test_get_databases(client_: MongoClient, holder_: SchemaHolder): + for database in client_.list_database_names(): + holder_.database = database break - def test_get_collections(client: MongoClient, holder: SchemaHolder): - database = client.get_database(holder.database) + def test_get_collections(client_: MongoClient, holder_: SchemaHolder): + database = client_.get_database(holder_.database) database.list_collection_names() test_fn = { diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py index d11aa82d7bd..48e08300fd2 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py @@ -44,6 +44,7 @@ from metadata.ingestion.source.database.snowflake.queries import ( SNOWFLAKE_GET_DATABASES, SNOWFLAKE_TEST_FETCH_TAG, SNOWFLAKE_TEST_GET_QUERIES, + SNOWFLAKE_TEST_GET_TABLES, ) from metadata.utils.logger import ingestion_logger @@ -145,7 +146,18 @@ def test_connection( ) -> None: """ Test connection. This can be executed either as part - of a metadata workflow or during an Automation Workflow + of a metadata workflow or during an Automation Workflow. + + Note how we run a custom GetTables query: + + The default inspector `get_table_names` runs a SHOW which + has a limit on 10000 rows in the result set: + https://github.com/open-metadata/OpenMetadata/issues/12798 + + This can cause errors if we are running tests against schemas + with more tables than that. There is no issues during the metadata + ingestion since in metadata.py we are overriding the default + `get_table_names` function with our custom queries. """ engine_wrapper = SnowflakeEngineWrapper( service_connection=service_connection, engine=engine @@ -158,7 +170,9 @@ def test_connection( "GetSchemas": partial( execute_inspector_func, engine_wrapper, "get_schema_names" ), - "GetTables": partial(execute_inspector_func, engine_wrapper, "get_table_names"), + "GetTables": partial( + test_query, statement=SNOWFLAKE_TEST_GET_TABLES, engine=engine + ), "GetViews": partial(execute_inspector_func, engine_wrapper, "get_view_names"), "GetQueries": partial( test_query, statement=SNOWFLAKE_TEST_GET_QUERIES, engine=engine diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py b/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py index 41cc3ec50af..2e995e4ab3e 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py @@ -117,6 +117,10 @@ SNOWFLAKE_TEST_GET_QUERIES = """ SELECT query_text from snowflake.account_usage.query_history limit 1 """ +SNOWFLAKE_TEST_GET_TABLES = """ +SELECT TABLE_NAME FROM information_schema.tables LIMIT 1 +""" + SNOWFLAKE_GET_DATABASES = "SHOW DATABASES"