yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 08:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

schema changes for dbt (#4462)

Browse Source 
schema changes for dbt (#4462)
This commit is contained in:
Onkar Ravgan 2022-04-25 14:14:05 +05:30 committed by GitHub
parent b7e1f4db5a
commit 12b5bda2fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json
View File

@ -7,13 +7,11 @@
"databaseMetadataConfigType": {
"description": "Database Source Config Metadata Pipeline type",
"type": "string",
"enum": [
"DatabaseMetadata"
],
"enum": ["DatabaseMetadata"],
"default": "DatabaseMetadata"
},
"localHttpDBTConfig": {
"description": "Local and HTTP DBT configs.",
"dbtConfig": {
"description": "DBT Catalog and Manifest file path config.",
"type": "object",
"properties": {
"dbtCatalogFilePath": {
@ -79,11 +77,11 @@
"enum": ["local", "http", "gcs", "gcs-path", "s3"]
},
"dbtConfig": {
"$ref": "#/definitions/dbtConfig"
},
"dbtSecurityConfig": {
"description": "DBT configuration.",
"oneOf": [
{
"$ref": "#/definitions/localHttpDBTConfig"
},
{
"$ref": "../security/credentials/gcsCredentials.json"
},

4
ingestion/src/metadata/ingestion/source/sql_source.py
View File

@ -142,7 +142,7 @@ def get_dbt_http(config) -> Optional[Tuple[str, str]]:
def get_dbt_gcs(config) -> Optional[Tuple[str, str]]:
try:
dbt_options = config.dbtConfig.gcsConfig
dbt_options = config.dbtSecurityConfig.gcsConfig
if config.dbtProvider.value == "gcs":
options = {
"credentials": {
@ -183,7 +183,7 @@ def get_dbt_s3(config) -> Optional[Tuple[str, str]]:
try:
from metadata.utils.aws_client import AWSClient
aws_client = AWSClient(config.dbtConfig).get_resource("s3")
aws_client = AWSClient(config.dbtSecurityConfig).get_resource("s3")
buckets = aws_client.buckets.all()
for bucket in buckets:
for bucket_object in bucket.objects.all():