yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-31 10:39:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed dbt optional files for local config (#13242)

Browse Source
This commit is contained in:
Onkar Ravgan 2023-10-03 12:44:06 +05:30 committed by GitHub
parent 462b2f9445
commit bc491be5ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py
View File

@ -232,7 +232,10 @@ def download_dbt_files(
""" """
Method to download the files from sources Method to download the files from sources
""" """
for key, blobs in blob_grouped_by_directory.items(): for ( # pylint: disable=too-many-nested-blocks
key,
blobs,
) in blob_grouped_by_directory.items():
dbt_catalog = None dbt_catalog = None
dbt_manifest = None dbt_manifest = None
dbt_run_results = None dbt_run_results = None
@ -241,16 +244,27 @@ def download_dbt_files(
kwargs = {"bucket_name": bucket_name} kwargs = {"bucket_name": bucket_name}
try: try:
for blob in blobs: for blob in blobs:
reader = get_reader(config_source=config, client=client) if blob:
if DBT_MANIFEST_FILE_NAME in blob: reader = get_reader(config_source=config, client=client)
logger.debug(f"{DBT_MANIFEST_FILE_NAME} found in {key}") if DBT_MANIFEST_FILE_NAME in blob:
dbt_manifest = reader.read(path=blob, **kwargs) logger.debug(f"{DBT_MANIFEST_FILE_NAME} found in {key}")
if DBT_CATALOG_FILE_NAME in blob: dbt_manifest = reader.read(path=blob, **kwargs)
logger.debug(f"{DBT_CATALOG_FILE_NAME} found in {key}") if DBT_CATALOG_FILE_NAME in blob:
dbt_catalog = reader.read(path=blob, **kwargs) try:
if DBT_RUN_RESULTS_FILE_NAME in blob: logger.debug(f"{DBT_CATALOG_FILE_NAME} found in {key}")
logger.debug(f"{DBT_RUN_RESULTS_FILE_NAME} found in {key}") dbt_catalog = reader.read(path=blob, **kwargs)
dbt_run_results = reader.read(path=blob, **kwargs) except Exception as exc:
logger.warning(
f"{DBT_CATALOG_FILE_NAME} not found in {key}: {exc}"
)
if DBT_RUN_RESULTS_FILE_NAME in blob:
try:
logger.debug(f"{DBT_RUN_RESULTS_FILE_NAME} found in {key}")
dbt_run_results = reader.read(path=blob, **kwargs)
except Exception as exc:
logger.warning(
f"{DBT_RUN_RESULTS_FILE_NAME} not found in {key}: {exc}"
)
if not dbt_manifest: if not dbt_manifest:
raise DBTConfigException(f"Manifest file not found at: {key}") raise DBTConfigException(f"Manifest file not found at: {key}")
yield DbtFiles( yield DbtFiles(