yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-30 04:39:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #12798 - Update Snowflake GetTables test query (#12804)

Browse Source
This commit is contained in:
Pere Miquel Brull 2023-08-10 07:30:34 +02:00 committed by GitHub
parent 5da63c5aed
commit 97c982b12a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ingestion/src/metadata/ingestion/source/database/mongodb/connection.py
View File

@ -59,13 +59,13 @@ def test_connection(
holder = SchemaHolder() holder = SchemaHolder()
def test_get_databases(client: MongoClient, holder: SchemaHolder): def test_get_databases(client_: MongoClient, holder_: SchemaHolder):
for database in client.list_database_names(): for database in client_.list_database_names():
holder.database = database holder_.database = database
break break
def test_get_collections(client: MongoClient, holder: SchemaHolder): def test_get_collections(client_: MongoClient, holder_: SchemaHolder):
database = client.get_database(holder.database) database = client_.get_database(holder_.database)
database.list_collection_names() database.list_collection_names()
test_fn = { test_fn = {

18
ingestion/src/metadata/ingestion/source/database/snowflake/connection.py
View File

@ -44,6 +44,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
SNOWFLAKE_GET_DATABASES, SNOWFLAKE_GET_DATABASES,
SNOWFLAKE_TEST_FETCH_TAG, SNOWFLAKE_TEST_FETCH_TAG,
SNOWFLAKE_TEST_GET_QUERIES, SNOWFLAKE_TEST_GET_QUERIES,
SNOWFLAKE_TEST_GET_TABLES,
) )
from metadata.utils.logger import ingestion_logger from metadata.utils.logger import ingestion_logger
@ -145,7 +146,18 @@ def test_connection(
) -> None: ) -> None:
""" """
Test connection. This can be executed either as part Test connection. This can be executed either as part
of a metadata workflow or during an Automation Workflow of a metadata workflow or during an Automation Workflow.
Note how we run a custom GetTables query:
The default inspector `get_table_names` runs a SHOW which
has a limit on 10000 rows in the result set:
https://github.com/open-metadata/OpenMetadata/issues/12798
This can cause errors if we are running tests against schemas
with more tables than that. There is no issues during the metadata
ingestion since in metadata.py we are overriding the default
`get_table_names` function with our custom queries.
""" """
engine_wrapper = SnowflakeEngineWrapper( engine_wrapper = SnowflakeEngineWrapper(
service_connection=service_connection, engine=engine service_connection=service_connection, engine=engine
@ -158,7 +170,9 @@ def test_connection(
"GetSchemas": partial( "GetSchemas": partial(
execute_inspector_func, engine_wrapper, "get_schema_names" execute_inspector_func, engine_wrapper, "get_schema_names"
), ),
"GetTables": partial(execute_inspector_func, engine_wrapper, "get_table_names"), "GetTables": partial(
test_query, statement=SNOWFLAKE_TEST_GET_TABLES, engine=engine
),
"GetViews": partial(execute_inspector_func, engine_wrapper, "get_view_names"), "GetViews": partial(execute_inspector_func, engine_wrapper, "get_view_names"),
"GetQueries": partial( "GetQueries": partial(
test_query, statement=SNOWFLAKE_TEST_GET_QUERIES, engine=engine test_query, statement=SNOWFLAKE_TEST_GET_QUERIES, engine=engine

4
ingestion/src/metadata/ingestion/source/database/snowflake/queries.py
View File

@ -117,6 +117,10 @@ SNOWFLAKE_TEST_GET_QUERIES = """
SELECT query_text from snowflake.account_usage.query_history limit 1 SELECT query_text from snowflake.account_usage.query_history limit 1
""" """
SNOWFLAKE_TEST_GET_TABLES = """
SELECT TABLE_NAME FROM information_schema.tables LIMIT 1
"""
SNOWFLAKE_GET_DATABASES = "SHOW DATABASES" SNOWFLAKE_GET_DATABASES = "SHOW DATABASES"