yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-20 23:18:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: dbt manifest parsing issue (#18930)

Browse Source
This commit is contained in:
Suman Maharana 2024-12-05 13:06:25 +05:30 committed by SumanMaharana
parent 122a2937ec
commit 04685f685b
2 changed files with 23 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ingestion/src/metadata/ingestion/source/database/dbt/constants.py
View File

@ -22,6 +22,14 @@ REQUIRED_MANIFEST_KEYS = ["name", "schema", "resource_type"]
# Based on https://schemas.getdbt.com/dbt/catalog/v1.json # Based on https://schemas.getdbt.com/dbt/catalog/v1.json
REQUIRED_CATALOG_KEYS = ["name", "type", "index"] REQUIRED_CATALOG_KEYS = ["name", "type", "index"]
REQUIRED_CONSTRAINT_KEYS = [
"type",
"name",
"expression",
"warn_unenforced",
"warn_unsupported",
]
REQUIRED_RESULTS_KEYS = { REQUIRED_RESULTS_KEYS = {
"status", "status",
"timing", "timing",

15
ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py
View File

@ -38,6 +38,7 @@ from metadata.ingestion.models.topology import (
) )
from metadata.ingestion.source.database.database_service import DataModelLink from metadata.ingestion.source.database.database_service import DataModelLink
from metadata.ingestion.source.database.dbt.constants import ( from metadata.ingestion.source.database.dbt.constants import (
REQUIRED_CONSTRAINT_KEYS,
REQUIRED_NODE_KEYS, REQUIRED_NODE_KEYS,
REQUIRED_RESULTS_KEYS, REQUIRED_RESULTS_KEYS,
) )
@ -182,6 +183,20 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
] ]
for key in keys_to_delete: for key in keys_to_delete:
del value[key] del value[key]
if value.get("columns"):
for col_name, value in value[
"columns"
].items(): # pylint: disable=unused-variable
if value.get("constraints"):
keys_to_delete = [
key
for key in value
if key.lower() not in REQUIRED_CONSTRAINT_KEYS
]
for key in keys_to_delete:
del value[key]
else:
value["constraints"] = None
def remove_run_result_non_required_keys(self, run_results: List[dict]): def remove_run_result_non_required_keys(self, run_results: List[dict]):
""" """