diff --git a/ingestion/examples/workflows/dynamodb.json b/ingestion/examples/workflows/dynamodb.json index 8212b50b756..bcba603b746 100644 --- a/ingestion/examples/workflows/dynamodb.json +++ b/ingestion/examples/workflows/dynamodb.json @@ -8,8 +8,8 @@ "awsConfig": { "awsAccessKeyId": "aws_access_key_id", "awsSecretAccessKey": "aws_secret_access_key", - "awsRegion": "us-east-2", - "hostPort": "https://dynamodb.us-east-2.amazonaws.com" + "awsRegion": "aws region", + "endPointURL": "https://dynamodb..amazonaws.com" }, "database": "custom_database_name" } diff --git a/ingestion/src/metadata/ingestion/source/dynamodb.py b/ingestion/src/metadata/ingestion/source/dynamodb.py index 47b4c5537c0..3a69f8aa615 100644 --- a/ingestion/src/metadata/ingestion/source/dynamodb.py +++ b/ingestion/src/metadata/ingestion/source/dynamodb.py @@ -22,7 +22,6 @@ from metadata.ingestion.api.source import InvalidSourceException, Source, Source from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.sql_source import SQLSourceStatus -from metadata.utils.aws_client import AWSClient from metadata.utils.column_type_parser import ColumnTypeParser from metadata.utils.connections import get_connection, test_connection from metadata.utils.filters import filter_by_table diff --git a/ingestion/src/metadata/ingestion/source/glue.py b/ingestion/src/metadata/ingestion/source/glue.py index 5170a66c88a..6fbe6b70980 100644 --- a/ingestion/src/metadata/ingestion/source/glue.py +++ b/ingestion/src/metadata/ingestion/source/glue.py @@ -36,7 +36,6 @@ from metadata.ingestion.api.source import InvalidSourceException, Source, Source from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.sql_source import SQLSourceStatus -from metadata.utils.aws_client import AWSClient from metadata.utils.column_type_parser import ColumnTypeParser from metadata.utils.connections import get_connection, test_connection from metadata.utils.filters import filter_by_schema, filter_by_table diff --git a/ingestion/src/metadata/utils/aws_client.py b/ingestion/src/metadata/utils/aws_client.py index e642b3b6eb4..0280a8bb1d2 100644 --- a/ingestion/src/metadata/utils/aws_client.py +++ b/ingestion/src/metadata/utils/aws_client.py @@ -9,26 +9,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass from typing import Any from boto3 import Session -from pydantic import SecretStr -from metadata.config.common import ConfigModel from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials - - -@dataclass -class GlueClient: - def __init__(self, client) -> None: - self.client = client - - -@dataclass -class DynamoClient: - def __init__(self, client) -> None: - self.client = client +from metadata.utils.connection_clients import DynamoClient, GlueClient class AWSClient: diff --git a/ingestion/src/metadata/utils/connection_clients.py b/ingestion/src/metadata/utils/connection_clients.py new file mode 100644 index 00000000000..f54d54ca52b --- /dev/null +++ b/ingestion/src/metadata/utils/connection_clients.py @@ -0,0 +1,24 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + + +@dataclass +class GlueClient: + def __init__(self, client) -> None: + self.client = client + + +@dataclass +class DynamoClient: + def __init__(self, client) -> None: + self.client = client diff --git a/ingestion/src/metadata/utils/connections.py b/ingestion/src/metadata/utils/connections.py index 79d9b31bfef..6f920c1152d 100644 --- a/ingestion/src/metadata/utils/connections.py +++ b/ingestion/src/metadata/utils/connections.py @@ -16,7 +16,6 @@ import logging from functools import singledispatch from typing import Union -from botocore.client import ClientError from sqlalchemy import create_engine from sqlalchemy.engine.base import Engine from sqlalchemy.exc import OperationalError @@ -41,7 +40,7 @@ from metadata.generated.schema.entity.services.connections.database.glueConnecti from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import ( SnowflakeConnection, ) -from metadata.utils.aws_client import AWSClient, DynamoClient, GlueClient +from metadata.utils.connection_clients import DynamoClient, GlueClient from metadata.utils.credentials import set_google_credentials from metadata.utils.source_connections import get_connection_args, get_connection_url from metadata.utils.timeout import timeout @@ -140,12 +139,16 @@ def _(connection: BigQueryConnection, verbose: bool = False): @get_connection.register def _(connection: DynamoDBConnection, verbose: bool = False): + from metadata.utils.aws_client import AWSClient + dynomo_connection = AWSClient(connection.awsConfig).get_dynomo_client() return dynomo_connection @get_connection.register def _(connection: GlueConnection, verbose: bool = False): + from metadata.utils.aws_client import AWSClient + glue_connection = AWSClient(connection.awsConfig).get_glue_client() return glue_connection @@ -188,6 +191,8 @@ def _(connection: DynamoClient) -> None: :param engine: boto service resource to test :return: None or raise an exception if we cannot connect """ + from botocore.client import ClientError + try: connection.client.tables.all() except ClientError as err: @@ -207,6 +212,8 @@ def _(connection: GlueClient) -> None: :param engine: boto cliet to test :return: None or raise an exception if we cannot connect """ + from botocore.client import ClientError + try: connection.client.list_workflows() except ClientError as err: