diff --git a/ingestion/src/metadata/ingestion/source/database/dbt.py b/ingestion/src/metadata/ingestion/source/database/dbt.py index 030ff389009..75ac2e60301 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt.py @@ -286,28 +286,30 @@ class DbtSource(DbtServiceSource): # pylint: disable=too-many-public-methods self.context.dbt_tests = {} for key, manifest_node in manifest_entities.items(): try: - # Skip the analysis node since it does not contain relevant metatada - if manifest_node["resource_type"] in ["analysis"]: - continue # If the run_results file is passed then only DBT tests will be processed - if dbt_files.dbt_run_results: + if ( + dbt_files.dbt_run_results + and manifest_node["resource_type"] == "test" + ): # Test nodes will be processed further in the topology - if manifest_node["resource_type"] == "test": - self.context.dbt_tests[key] = manifest_node - self.context.dbt_tests[key][ - "upstream" - ] = self.parse_upstream_nodes( - manifest_entities, manifest_node - ) - self.context.dbt_tests[key][ - "results" - ] = next( # pylint: disable=stop-iteration-return - item - for item in dbt_files.dbt_run_results.get("results") - if item["unique_id"] == key - ) - continue + self.context.dbt_tests[key] = manifest_node + self.context.dbt_tests[key][ + "upstream" + ] = self.parse_upstream_nodes(manifest_entities, manifest_node) + self.context.dbt_tests[key][ + "results" + ] = next( # pylint: disable=stop-iteration-return + item + for item in dbt_files.dbt_run_results.get("results") + if item["unique_id"] == key + ) + continue + + # Skip the analysis and test nodes + if manifest_node["resource_type"] in ("analysis", "test"): + logger.info(f"Skipping DBT node: {key}.") + continue model_name = ( manifest_node["alias"] diff --git a/ingestion/src/metadata/utils/dbt_config.py b/ingestion/src/metadata/utils/dbt_config.py index d184a96c03e..8b45935d3b8 100644 --- a/ingestion/src/metadata/utils/dbt_config.py +++ b/ingestion/src/metadata/utils/dbt_config.py @@ -79,7 +79,7 @@ def _(config: DbtLocalConfig): ) with open(config.dbtManifestFilePath, "r", encoding="utf-8") as manifest: dbt_manifest = manifest.read() - if config.dbtRunResultsFilePath is not None: + if config.dbtRunResultsFilePath: logger.debug( f"Reading [dbtRunResultsFilePath] from: {config.dbtRunResultsFilePath}" ) @@ -87,7 +87,7 @@ def _(config: DbtLocalConfig): config.dbtRunResultsFilePath, "r", encoding="utf-8" ) as run_results: dbt_run_results = run_results.read() - if config.dbtCatalogFilePath is not None: + if config.dbtCatalogFilePath: logger.debug( f"Reading [dbtCatalogFilePath] from: {config.dbtCatalogFilePath}" ) @@ -113,7 +113,7 @@ def _(config: DbtHttpConfig): config.dbtManifestHttpPath ) dbt_run_results = None - if config.dbtRunResultsHttpPath is not None: + if config.dbtRunResultsHttpPath: logger.debug( f"Requesting [dbtRunResultsHttpPath] to: {config.dbtRunResultsHttpPath}" ) @@ -122,7 +122,7 @@ def _(config: DbtHttpConfig): ) dbt_catalog = None - if config.dbtCatalogHttpPath is not None: + if config.dbtCatalogHttpPath: logger.debug( f"Requesting [dbtCatalogHttpPath] to: {config.dbtCatalogHttpPath}" )