From bc491be5ad7bb8a22c9c4c1a8e087f2a63b94af0 Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Tue, 3 Oct 2023 12:44:06 +0530 Subject: [PATCH] Fixed dbt optional files for local config (#13242) --- .../source/database/dbt/dbt_config.py | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py index 2343af705ab..e85e29d6312 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py @@ -232,7 +232,10 @@ def download_dbt_files( """ Method to download the files from sources """ - for key, blobs in blob_grouped_by_directory.items(): + for ( # pylint: disable=too-many-nested-blocks + key, + blobs, + ) in blob_grouped_by_directory.items(): dbt_catalog = None dbt_manifest = None dbt_run_results = None @@ -241,16 +244,27 @@ def download_dbt_files( kwargs = {"bucket_name": bucket_name} try: for blob in blobs: - reader = get_reader(config_source=config, client=client) - if DBT_MANIFEST_FILE_NAME in blob: - logger.debug(f"{DBT_MANIFEST_FILE_NAME} found in {key}") - dbt_manifest = reader.read(path=blob, **kwargs) - if DBT_CATALOG_FILE_NAME in blob: - logger.debug(f"{DBT_CATALOG_FILE_NAME} found in {key}") - dbt_catalog = reader.read(path=blob, **kwargs) - if DBT_RUN_RESULTS_FILE_NAME in blob: - logger.debug(f"{DBT_RUN_RESULTS_FILE_NAME} found in {key}") - dbt_run_results = reader.read(path=blob, **kwargs) + if blob: + reader = get_reader(config_source=config, client=client) + if DBT_MANIFEST_FILE_NAME in blob: + logger.debug(f"{DBT_MANIFEST_FILE_NAME} found in {key}") + dbt_manifest = reader.read(path=blob, **kwargs) + if DBT_CATALOG_FILE_NAME in blob: + try: + logger.debug(f"{DBT_CATALOG_FILE_NAME} found in {key}") + dbt_catalog = reader.read(path=blob, **kwargs) + except Exception as exc: + logger.warning( + f"{DBT_CATALOG_FILE_NAME} not found in {key}: {exc}" + ) + if DBT_RUN_RESULTS_FILE_NAME in blob: + try: + logger.debug(f"{DBT_RUN_RESULTS_FILE_NAME} found in {key}") + dbt_run_results = reader.read(path=blob, **kwargs) + except Exception as exc: + logger.warning( + f"{DBT_RUN_RESULTS_FILE_NAME} not found in {key}: {exc}" + ) if not dbt_manifest: raise DBTConfigException(f"Manifest file not found at: {key}") yield DbtFiles(