diff --git a/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json b/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json index 2b24005dd24..41865f41a3e 100644 --- a/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json +++ b/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json @@ -7,13 +7,11 @@ "databaseMetadataConfigType": { "description": "Database Source Config Metadata Pipeline type", "type": "string", - "enum": [ - "DatabaseMetadata" - ], + "enum": ["DatabaseMetadata"], "default": "DatabaseMetadata" }, - "localHttpDBTConfig": { - "description": "Local and HTTP DBT configs.", + "dbtConfig": { + "description": "DBT Catalog and Manifest file path config.", "type": "object", "properties": { "dbtCatalogFilePath": { @@ -79,11 +77,11 @@ "enum": ["local", "http", "gcs", "gcs-path", "s3"] }, "dbtConfig": { + "$ref": "#/definitions/dbtConfig" + }, + "dbtSecurityConfig": { "description": "DBT configuration.", "oneOf": [ - { - "$ref": "#/definitions/localHttpDBTConfig" - }, { "$ref": "../security/credentials/gcsCredentials.json" }, diff --git a/ingestion/src/metadata/ingestion/source/sql_source.py b/ingestion/src/metadata/ingestion/source/sql_source.py index 9b3011ac7d0..ac260740c5d 100644 --- a/ingestion/src/metadata/ingestion/source/sql_source.py +++ b/ingestion/src/metadata/ingestion/source/sql_source.py @@ -142,7 +142,7 @@ def get_dbt_http(config) -> Optional[Tuple[str, str]]: def get_dbt_gcs(config) -> Optional[Tuple[str, str]]: try: - dbt_options = config.dbtConfig.gcsConfig + dbt_options = config.dbtSecurityConfig.gcsConfig if config.dbtProvider.value == "gcs": options = { "credentials": { @@ -183,7 +183,7 @@ def get_dbt_s3(config) -> Optional[Tuple[str, str]]: try: from metadata.utils.aws_client import AWSClient - aws_client = AWSClient(config.dbtConfig).get_resource("s3") + aws_client = AWSClient(config.dbtSecurityConfig).get_resource("s3") buckets = aws_client.buckets.all() for bucket in buckets: for bucket_object in bucket.objects.all():