diff --git a/ingestion/src/metadata/examples/workflows/dbt.yaml b/ingestion/src/metadata/examples/workflows/dbt.yaml index 422ee01a65c..c518595e818 100644 --- a/ingestion/src/metadata/examples/workflows/dbt.yaml +++ b/ingestion/src/metadata/examples/workflows/dbt.yaml @@ -9,6 +9,8 @@ source: # # For cloud # dbtCloudAuthToken: token # dbtCloudAccountId: ID + # dbtCloudJobId: JOB ID + # dbtCloudUrl: https://cloud.getdbt.com # # For Local # dbtCatalogFilePath: path-to-catalog.json # dbtManifestFilePath: path-to-manifest.json diff --git a/ingestion/src/metadata/utils/dbt_config.py b/ingestion/src/metadata/utils/dbt_config.py index 9562dd58d74..356912d40f9 100644 --- a/ingestion/src/metadata/utils/dbt_config.py +++ b/ingestion/src/metadata/utils/dbt_config.py @@ -172,12 +172,17 @@ def _(config: DbtCloudConfig): # pylint: disable=too-many-locals client = REST(client_config) account_id = config.dbtCloudAccountId project_id = config.dbtCloudProjectId + job_id = config.dbtCloudJobId logger.debug( "Requesting [dbt_catalog], [dbt_manifest] and [dbt_run_results] data" ) params_data = {"order_by": "-finished_at", "limit": "1", "status": "10"} if project_id: params_data["project_id"] = project_id + + if job_id: + params_data["job_definition_id"] = job_id + response = client.get(f"/accounts/{account_id}/runs", data=params_data) runs_data = response.get("data") if runs_data: diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json index e4fdf3ffe7f..d335abcaf66 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtconfig/dbtCloudConfig.json @@ -22,6 +22,11 @@ "description": "In case of multiple projects in a dbt cloud account, specify the project's id from which you want to extract the dbt run artifacts", "type": "string" }, + "dbtCloudJobId": { + "title": "dbt Cloud Job Id", + "description": "dbt cloud job id.", + "type": "string" + }, "dbtCloudUrl": { "title": "dbt Cloud URL", "description": "URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or https://emea.dbt.com/",