Add unit test for dynamodb

This commit is contained in:
Milan Bariya 2022-04-11 19:33:27 +05:30
parent 2732906059
commit b64384e84a
2 changed files with 31 additions and 28 deletions

View File

@ -19,7 +19,7 @@ from metadata.ingestion.api.common import Entity
from metadata.ingestion.api.source import InvalidSourceException, Source, SourceStatus
from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.sql_source_common import SQLSourceStatus
from metadata.ingestion.source.sql_source import SQLSourceStatus
from metadata.utils.aws_client import AWSClient
from metadata.utils.column_type_parser import ColumnTypeParser
from metadata.utils.filters import filter_by_table
@ -27,8 +27,6 @@ from metadata.utils.helpers import get_database_service_or_create
logger: logging.Logger = logging.getLogger(__name__)
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
class DynamodbSource(Source[Entity]):
def __init__(self, config, metadata_config: OpenMetadataServerConfig):

View File

@ -10,7 +10,7 @@
# limitations under the License.
"""
Query parser utils tests
Dynamodb unit test
"""
import json
from unittest import TestCase
@ -24,35 +24,40 @@ from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
CONFIG = """
{
"source": {
"type": "dynamodb",
"source": {
"type": "dynamodb",
"serviceName": "local_dynamodb",
"serviceConnection": {
"config": {
"aws_access_key_id": "aws_access_key_id",
"aws_secret_access_key": "aws_secret_access_key",
"service_name": "DynamoDBTest",
"region_name": "us-east-2",
"endpoint_url": "https://dynamodb.us-east-2.amazonaws.com",
"db_name":"custom_database_name",
"table_filter_pattern":{
"excludes": ["Music.*"]
"awsAccessKeyId": "aws_access_key_id",
"awsSecretAccessKey": "aws_secret_access_key",
"awsRegion": "us-east-2",
"endPointURL": "https://dynamodb.us-east-2.amazonaws.com",
"database": "custom_database_name"
}
},
"sourceConfig": {
"config": {
"enableDataProfiler": false,
"tableFilterPattern": {
"includes": [""]
}
}
},
"sink": {
"type": "file",
"config": {
"filename": "./datasets.json"
}
},
"metadata_server": {
"type": "metadata-server",
"config": {
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
}
},
"sink": {
"type": "file",
"config": {
"filename": "/var/tmp/datasets.json"
}
},
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "no-auth"
}
}
}
"""