yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-02 05:33:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/bigquery): Capture all operation types when ingesting operational stats (#7723)

Browse Source
This commit is contained in:
Andrew Sikowitz 2023-03-31 03:31:28 -07:00 committed by GitHub
parent f6d7e1a325
commit a2f8c76388
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
View File

@ -41,6 +41,8 @@ logger: logging.Logger = logging.getLogger(__name__)
AggregatedDataset = GenericAggregatedDataset[BigQueryTableRef]
# See https://cloud.google.com/java/docs/reference/google-cloud-bigquery/latest/com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType
# https://pkg.go.dev/google.golang.org/genproto/googleapis/cloud/audit may be more complete
OPERATION_STATEMENT_TYPES = {
"INSERT": OperationTypeClass.INSERT,
"UPDATE": OperationTypeClass.UPDATE,
@ -48,10 +50,21 @@ OPERATION_STATEMENT_TYPES = {
"MERGE": OperationTypeClass.UPDATE,
"CREATE": OperationTypeClass.CREATE,
"CREATE_TABLE_AS_SELECT": OperationTypeClass.CREATE,
"CREATE_SCHEMA": OperationTypeClass.CREATE,
"DROP_TABLE": OperationTypeClass.DROP,
"CREATE_EXTERNAL_TABLE": OperationTypeClass.CREATE,
"CREATE_SNAPSHOT_TABLE": OperationTypeClass.CREATE,
"CREATE_VIEW": OperationTypeClass.CREATE,
"CREATE_MATERIALIZED_VIEW": OperationTypeClass.CREATE,
"CREATE_SCHEMA": OperationTypeClass.CREATE,
"DROP_TABLE": OperationTypeClass.DROP,
"DROP_EXTERNAL_TABLE": OperationTypeClass.DROP,
"DROP_SNAPSHOT_TABLE": OperationTypeClass.DROP,
"DROP_VIEW": OperationTypeClass.DROP,
"DROP_MATERIALIZED_VIEW": OperationTypeClass.DROP,
"DROP_SCHEMA": OperationTypeClass.DROP,
"ALTER_TABLE": OperationTypeClass.ALTER,
"ALTER_VIEW": OperationTypeClass.ALTER,
"ALTER_MATERIALIZED_VIEW": OperationTypeClass.ALTER,
"ALTER_SCHEMA": OperationTypeClass.ALTER,
}
READ_STATEMENT_TYPES: List[str] = ["SELECT"]