yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-09 16:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/dbt-cloud): use job-based graphql queries (#8647)

Browse Source
This commit is contained in:
Harshal Sheth 2023-08-17 10:29:10 -07:00 committed by GitHub
parent 130f908a49
commit c5a2e8923e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
View File

@ -162,10 +162,12 @@ _DBT_FIELDS_BY_TYPE = {
} }
_DBT_GRAPHQL_QUERY = """ _DBT_GRAPHQL_QUERY = """
query DatahubMetadataQuery_{type}($jobId: Int!, $runId: Int) {{ query DatahubMetadataQuery_{type}($jobId: BigInt!, $runId: BigInt) {{
{type}(jobId: $jobId, runId: $runId) {{ job(id: $jobId, runId: $runId) {{
{type} {{
{fields} {fields}
}} }}
}}
}} }}
""" """
@ -218,7 +220,7 @@ class DBTCloudSource(DBTSourceBase):
}, },
) )
raw_nodes.extend(data[node_type]) raw_nodes.extend(data["job"][node_type])
nodes = [self._parse_into_dbt_node(node) for node in raw_nodes] nodes = [self._parse_into_dbt_node(node) for node in raw_nodes]