yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 17:04:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

issue-9734: job-id-param-added-for-dbt-cloud-ingestion (#10248)

Browse Source 
UI changes will be handled with https://github.com/open-metadata/OpenMetadata/issues/10269
This commit is contained in:
Abhishek Pandey 2023-02-22 12:18:03 +05:30 committed by GitHub
parent cae284d8d7
commit c678b54b05
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/src/metadata/examples/workflows/dbt.yaml
View File

@ -9,6 +9,8 @@ source:
# # For cloud # # For cloud
# dbtCloudAuthToken: token # dbtCloudAuthToken: token
# dbtCloudAccountId: ID # dbtCloudAccountId: ID
# dbtCloudJobId: JOB ID
# dbtCloudUrl: https://cloud.getdbt.com
# # For Local # # For Local
# dbtCatalogFilePath: path-to-catalog.json # dbtCatalogFilePath: path-to-catalog.json
# dbtManifestFilePath: path-to-manifest.json # dbtManifestFilePath: path-to-manifest.json

5
ingestion/src/metadata/utils/dbt_config.py
View File

@ -172,12 +172,17 @@ def _(config: DbtCloudConfig): # pylint: disable=too-many-locals
client = REST(client_config) client = REST(client_config)
account_id = config.dbtCloudAccountId account_id = config.dbtCloudAccountId
project_id = config.dbtCloudProjectId project_id = config.dbtCloudProjectId
job_id = config.dbtCloudJobId
logger.debug( logger.debug(
"Requesting [dbt_catalog], [dbt_manifest] and [dbt_run_results] data" "Requesting [dbt_catalog], [dbt_manifest] and [dbt_run_results] data"
) )
params_data = {"order_by": "-finished_at", "limit": "1", "status": "10"} params_data = {"order_by": "-finished_at", "limit": "1", "status": "10"}
if project_id: if project_id:
params_data["project_id"] = project_id params_data["project_id"] = project_id
if job_id:
params_data["job_definition_id"] = job_id
response = client.get(f"/accounts/{account_id}/runs", data=params_data) response = client.get(f"/accounts/{account_id}/runs", data=params_data)
runs_data = response.get("data") runs_data = response.get("data")
if runs_data: if runs_data:

5
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json
View File

@ -22,6 +22,11 @@
"description": "In case of multiple projects in a dbt cloud account, specify the project's id from which you want to extract the dbt run artifacts", "description": "In case of multiple projects in a dbt cloud account, specify the project's id from which you want to extract the dbt run artifacts",
"type": "string" "type": "string"
}, },
"dbtCloudJobId": {
"title": "dbt Cloud Job Id",
"description": "dbt cloud job id.",
"type": "string"
},
"dbtCloudUrl": { "dbtCloudUrl": {
"title": "dbt Cloud URL", "title": "dbt Cloud URL",
"description": "URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or https://emea.dbt.com/", "description": "URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or https://emea.dbt.com/",