yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-22 16:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Capture BQ Query Cost (#20124)

Browse Source
This commit is contained in:
Mayur Singal 2025-03-13 17:06:35 +05:30 committed by GitHub
parent d1ae36129c
commit c5c724887e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ingestion/src/metadata/ingestion/source/database/bigquery/queries.py
View File

@ -32,7 +32,8 @@ BIGQUERY_STATEMENT = textwrap.dedent(
end_time, end_time,
query as query_text, query as query_text,
null as schema_name, null as schema_name,
total_slot_ms as duration total_slot_ms as duration,
(total_bytes_billed / POWER(2, 40)) * {cost_per_tib} as cost
FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT
WHERE creation_time BETWEEN "{start_time}" AND "{end_time}" WHERE creation_time BETWEEN "{start_time}" AND "{end_time}"
{filters} {filters}

1
ingestion/src/metadata/ingestion/source/database/bigquery/query_parser.py
View File

@ -66,6 +66,7 @@ class BigqueryQueryParserSource(QueryParserSource, ABC):
region=self.service_connection.usageLocation, region=self.service_connection.usageLocation,
filters=self.get_filters(), filters=self.get_filters(),
result_limit=self.source_config.resultLimit, result_limit=self.source_config.resultLimit,
cost_per_tib=self.service_connection.costPerTB,
) )
@staticmethod @staticmethod

6
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json
View File

@ -116,6 +116,12 @@
"supportsDataDiff": { "supportsDataDiff": {
"title": "Supports Data Diff Extraction.", "title": "Supports Data Diff Extraction.",
"$ref": "../connectionBasicType.json#/definitions/supportsDataDiff" "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
},
"costPerTB": {
"title": "Cost Per TiB",
"description": "Cost per TiB for BigQuery usage",
"type": "number",
"default": 6.25
} }
}, },
"additionalProperties": false, "additionalProperties": false,

4
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/bigQueryConnection.ts
View File

@ -16,6 +16,10 @@
export interface BigQueryConnection { export interface BigQueryConnection {
connectionArguments?: { [key: string]: any }; connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string }; connectionOptions?: { [key: string]: string };
/**
* Cost per TiB for BigQuery usage
*/
costPerTB?: number;
/** /**
* GCP Credentials * GCP Credentials
*/ */