yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 17:04:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

schema changes for dbt (#4462)

Browse Source 
schema changes for dbt (#4462)
This commit is contained in:
Onkar Ravgan 2022-04-25 14:14:05 +05:30 committed by GitHub
parent b7e1f4db5a
commit 12b5bda2fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json
View File

@ -7,13 +7,11 @@
"databaseMetadataConfigType": { "databaseMetadataConfigType": {
"description": "Database Source Config Metadata Pipeline type", "description": "Database Source Config Metadata Pipeline type",
"type": "string", "type": "string",
"enum": [ "enum": ["DatabaseMetadata"],
"DatabaseMetadata"
],
"default": "DatabaseMetadata" "default": "DatabaseMetadata"
}, },
"localHttpDBTConfig": { "dbtConfig": {
"description": "Local and HTTP DBT configs.", "description": "DBT Catalog and Manifest file path config.",
"type": "object", "type": "object",
"properties": { "properties": {
"dbtCatalogFilePath": { "dbtCatalogFilePath": {
@ -79,11 +77,11 @@
"enum": ["local", "http", "gcs", "gcs-path", "s3"] "enum": ["local", "http", "gcs", "gcs-path", "s3"]
}, },
"dbtConfig": { "dbtConfig": {
"$ref": "#/definitions/dbtConfig"
},
"dbtSecurityConfig": {
"description": "DBT configuration.", "description": "DBT configuration.",
"oneOf": [ "oneOf": [
{
"$ref": "#/definitions/localHttpDBTConfig"
},
{ {
"$ref": "../security/credentials/gcsCredentials.json" "$ref": "../security/credentials/gcsCredentials.json"
}, },

4
ingestion/src/metadata/ingestion/source/sql_source.py
View File

@ -142,7 +142,7 @@ def get_dbt_http(config) -> Optional[Tuple[str, str]]:
def get_dbt_gcs(config) -> Optional[Tuple[str, str]]: def get_dbt_gcs(config) -> Optional[Tuple[str, str]]:
try: try:
dbt_options = config.dbtConfig.gcsConfig dbt_options = config.dbtSecurityConfig.gcsConfig
if config.dbtProvider.value == "gcs": if config.dbtProvider.value == "gcs":
options = { options = {
"credentials": { "credentials": {
@ -183,7 +183,7 @@ def get_dbt_s3(config) -> Optional[Tuple[str, str]]:
try: try:
from metadata.utils.aws_client import AWSClient from metadata.utils.aws_client import AWSClient
aws_client = AWSClient(config.dbtConfig).get_resource("s3") aws_client = AWSClient(config.dbtSecurityConfig).get_resource("s3")
buckets = aws_client.buckets.all() buckets = aws_client.buckets.all()
for bucket in buckets: for bucket in buckets:
for bucket_object in bucket.objects.all(): for bucket_object in bucket.objects.all():