yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-24 22:18:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: ingestion for dbt > 1.8.0 resource_type is not an enum (#16415)

Browse Source 
* fix: resource_type is not an enum

* feat: add log to display finis

* improve readability

* use getattr to be compatible

* format
This commit is contained in:
Antoine Balliet 2024-05-28 17:00:04 +02:00 committed by GitHub
parent 7dad49640d
commit 0abd3ca5fe
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py
View File

@ -180,7 +180,11 @@ def _(config: DbtCloudConfig): # pylint: disable=too-many-locals
)
runs_data = response.get("data")
if runs_data:
run_id = runs_data[0]["id"]
last_run = runs_data[0]
run_id = last_run["id"]
logger.info(
f"Retrieved last successful run [{run_id}] finished {last_run['finished_at_humanized']} (duration: {last_run['duration_humanized']})"
)
try:
logger.debug("Requesting [dbt_catalog]")
dbt_catalog = client.get(

14
ingestion/src/metadata/ingestion/source/database/dbt/metadata.py
View File

@ -339,11 +339,15 @@ class DbtSource(DbtServiceSource):
)
for key, manifest_node in manifest_entities.items():
try:
resource_type = getattr(
manifest_node.resource_type,
"value",
manifest_node.resource_type,
)
# If the run_results file is passed then only DBT tests will be processed
if (
dbt_objects.dbt_run_results
and manifest_node.resource_type.value
== SkipResourceTypeEnum.TEST.value
and resource_type == SkipResourceTypeEnum.TEST.value
):
# Test nodes will be processed further in the topology
self.add_dbt_tests(
@ -360,9 +364,7 @@ class DbtSource(DbtServiceSource):
continue
# Skip the analysis and test nodes
if manifest_node.resource_type.value in [
item.value for item in SkipResourceTypeEnum
]:
if resource_type in [item.value for item in SkipResourceTypeEnum]:
logger.debug(f"Skipping DBT node: {key}.")
continue
@ -428,7 +430,7 @@ class DbtSource(DbtServiceSource):
table_entity=table_entity,
datamodel=DataModel(
modelType=ModelType.DBT,
resourceType=manifest_node.resource_type.value,
resourceType=resource_type,
description=manifest_node.description
if manifest_node.description
else None,