Refractor test connection for glue & dynamodb (#4373)

* Refractor test connection for glue & dynamodb

* Renamed test_connection_client

ingestion/examples/workflows/dynamodb.json

@@ -8,8 +8,8 @@
         "awsConfig": {
           "awsAccessKeyId": "aws_access_key_id",
           "awsSecretAccessKey": "aws_secret_access_key",
-          "awsRegion": "us-east-2",
-          "hostPort": "https://dynamodb.us-east-2.amazonaws.com"
+          "awsRegion": "aws region",
+          "endPointURL": "https://dynamodb.<region_name>.amazonaws.com"
         },
         "database": "custom_database_name"
       }

ingestion/src/metadata/ingestion/source/dynamodb.py

@@ -22,7 +22,6 @@ from metadata.ingestion.api.source import InvalidSourceException, Source, Source
 from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.sql_source import SQLSourceStatus
-from metadata.utils.aws_client import AWSClient
 from metadata.utils.column_type_parser import ColumnTypeParser
 from metadata.utils.connections import get_connection, test_connection
 from metadata.utils.filters import filter_by_table

ingestion/src/metadata/ingestion/source/glue.py

@@ -36,7 +36,6 @@ from metadata.ingestion.api.source import InvalidSourceException, Source, Source
 from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.sql_source import SQLSourceStatus
-from metadata.utils.aws_client import AWSClient
 from metadata.utils.column_type_parser import ColumnTypeParser
 from metadata.utils.connections import get_connection, test_connection
 from metadata.utils.filters import filter_by_schema, filter_by_table

ingestion/src/metadata/utils/aws_client.py

@@ -9,26 +9,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from dataclasses import dataclass
 from typing import Any
 
 from boto3 import Session
-from pydantic import SecretStr
 
-from metadata.config.common import ConfigModel
 from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials
-
-
-@dataclass
-class GlueClient:
-    def __init__(self, client) -> None:
-        self.client = client
-
-
-@dataclass
-class DynamoClient:
-    def __init__(self, client) -> None:
-        self.client = client
+from metadata.utils.connection_clients import DynamoClient, GlueClient
 
 
 class AWSClient:

ingestion/src/metadata/utils/connection_clients.py

@@ -0,0 +1,24 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from dataclasses import dataclass
+
+
+@dataclass
+class GlueClient:
+    def __init__(self, client) -> None:
+        self.client = client
+
+
+@dataclass
+class DynamoClient:
+    def __init__(self, client) -> None:
+        self.client = client

ingestion/src/metadata/utils/connections.py

@@ -16,7 +16,6 @@ import logging
 from functools import singledispatch
 from typing import Union
 
-from botocore.client import ClientError
 from sqlalchemy import create_engine
 from sqlalchemy.engine.base import Engine
 from sqlalchemy.exc import OperationalError
@@ -41,7 +40,7 @@ from metadata.generated.schema.entity.services.connections.database.glueConnecti
 from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
     SnowflakeConnection,
 )
-from metadata.utils.aws_client import AWSClient, DynamoClient, GlueClient
+from metadata.utils.connection_clients import DynamoClient, GlueClient
 from metadata.utils.credentials import set_google_credentials
 from metadata.utils.source_connections import get_connection_args, get_connection_url
 from metadata.utils.timeout import timeout
@@ -140,12 +139,16 @@ def _(connection: BigQueryConnection, verbose: bool = False):
 
 @get_connection.register
 def _(connection: DynamoDBConnection, verbose: bool = False):
+    from metadata.utils.aws_client import AWSClient
+
     dynomo_connection = AWSClient(connection.awsConfig).get_dynomo_client()
     return dynomo_connection
 
 
 @get_connection.register
 def _(connection: GlueConnection, verbose: bool = False):
+    from metadata.utils.aws_client import AWSClient
+
     glue_connection = AWSClient(connection.awsConfig).get_glue_client()
     return glue_connection
 
@@ -188,6 +191,8 @@ def _(connection: DynamoClient) -> None:
     :param engine: boto service resource to test
     :return: None or raise an exception if we cannot connect
     """
+    from botocore.client import ClientError
+
     try:
         connection.client.tables.all()
     except ClientError as err:
@@ -207,6 +212,8 @@ def _(connection: GlueClient) -> None:
     :param engine: boto cliet to test
     :return: None or raise an exception if we cannot connect
     """
+    from botocore.client import ClientError
+
     try:
         connection.client.list_workflows()
     except ClientError as err: