From c678b54b05a3174267f65e49d2d30067524dfc39 Mon Sep 17 00:00:00 2001 From: Abhishek Pandey Date: Wed, 22 Feb 2023 12:18:03 +0530 Subject: [PATCH] issue-9734: job-id-param-added-for-dbt-cloud-ingestion (#10248) UI changes will be handled with https://github.com/open-metadata/OpenMetadata/issues/10269 --- ingestion/src/metadata/examples/workflows/dbt.yaml | 2 ++ ingestion/src/metadata/utils/dbt_config.py | 5 +++++ .../schema/metadataIngestion/dbtconfig/dbtCloudConfig.json | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/ingestion/src/metadata/examples/workflows/dbt.yaml b/ingestion/src/metadata/examples/workflows/dbt.yaml index 422ee01a65c..c518595e818 100644 --- a/ingestion/src/metadata/examples/workflows/dbt.yaml +++ b/ingestion/src/metadata/examples/workflows/dbt.yaml @@ -9,6 +9,8 @@ source: # # For cloud # dbtCloudAuthToken: token # dbtCloudAccountId: ID + # dbtCloudJobId: JOB ID + # dbtCloudUrl: https://cloud.getdbt.com # # For Local # dbtCatalogFilePath: path-to-catalog.json # dbtManifestFilePath: path-to-manifest.json diff --git a/ingestion/src/metadata/utils/dbt_config.py b/ingestion/src/metadata/utils/dbt_config.py index 9562dd58d74..356912d40f9 100644 --- a/ingestion/src/metadata/utils/dbt_config.py +++ b/ingestion/src/metadata/utils/dbt_config.py @@ -172,12 +172,17 @@ def _(config: DbtCloudConfig): # pylint: disable=too-many-locals client = REST(client_config) account_id = config.dbtCloudAccountId project_id = config.dbtCloudProjectId + job_id = config.dbtCloudJobId logger.debug( "Requesting [dbt_catalog], [dbt_manifest] and [dbt_run_results] data" ) params_data = {"order_by": "-finished_at", "limit": "1", "status": "10"} if project_id: params_data["project_id"] = project_id + + if job_id: + params_data["job_definition_id"] = job_id + response = client.get(f"/accounts/{account_id}/runs", data=params_data) runs_data = response.get("data") if runs_data: diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json index e4fdf3ffe7f..d335abcaf66 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json @@ -22,6 +22,11 @@ "description": "In case of multiple projects in a dbt cloud account, specify the project's id from which you want to extract the dbt run artifacts", "type": "string" }, + "dbtCloudJobId": { + "title": "dbt Cloud Job Id", + "description": "dbt cloud job id.", + "type": "string" + }, "dbtCloudUrl": { "title": "dbt Cloud URL", "description": "URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or https://emea.dbt.com/",