yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-17 13:45:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): bigquery - missing sqlalchemy dep and row count fix (#6553)

Browse Source
This commit is contained in:
Tamas Nemeth 2022-11-25 22:33:14 +01:00 committed by GitHub
parent 7a8e36d57d
commit d424edde41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
metadata-ingestion/setup.py
View File

@ -252,12 +252,13 @@ plugins: Dict[str, Set[str]] = {
| bigquery_common | bigquery_common
| {"sqlalchemy-bigquery>=1.4.1", "sqllineage==1.3.6", "sqlparse"}, | {"sqlalchemy-bigquery>=1.4.1", "sqllineage==1.3.6", "sqlparse"},
"bigquery-usage-legacy": bigquery_common | usage_common | {"cachetools"}, "bigquery-usage-legacy": bigquery_common | usage_common | {"cachetools"},
"bigquery": sql_common | bigquery_common | {"sqllineage==1.3.6", "sql_metadata"}, "bigquery": sql_common | bigquery_common | {"sqllineage==1.3.6", "sql_metadata", "sqlalchemy-bigquery>=1.4.1"},
"bigquery-beta": sql_common "bigquery-beta": sql_common
| bigquery_common | bigquery_common
| { | {
"sqllineage==1.3.6", "sqllineage==1.3.6",
"sql_metadata", "sql_metadata",
"sqlalchemy-bigquery>=1.4.1"
}, # deprecated, but keeping the extra for backwards compatibility }, # deprecated, but keeping the extra for backwards compatibility
"clickhouse": sql_common | clickhouse_common, "clickhouse": sql_common | clickhouse_common,
"clickhouse-usage": sql_common | usage_common | clickhouse_common, "clickhouse-usage": sql_common | usage_common | clickhouse_common,
@ -376,7 +377,8 @@ mypy_stubs = {
"types-ujson>=5.2.0", "types-ujson>=5.2.0",
"types-termcolor>=1.0.0", "types-termcolor>=1.0.0",
"types-Deprecated", "types-Deprecated",
"types-protobuf", # Mypy complains with 4.21.0.0 => error: Library stubs not installed for "google.protobuf.descriptor"
"types-protobuf<4.21.0.0",
} }
base_dev_requirements = { base_dev_requirements = {

20
metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
View File

@ -349,8 +349,8 @@ class BigQueryDataDictionary:
) )
if "last_altered" in table if "last_altered" in table
else None, else None,
size_in_bytes=table.bytes if "bytes" in table else None, size_in_bytes=table.get("bytes"),
rows_count=table.row_count if "row_count" in table else None, rows_count=table.get("row_count"),
comment=table.comment, comment=table.comment,
ddl=table.ddl, ddl=table.ddl,
expires=tables[table.table_name].expires if tables else None, expires=tables[table.table_name].expires if tables else None,
@ -361,24 +361,16 @@ class BigQueryDataDictionary:
clustering_fields=tables[table.table_name].clustering_fields clustering_fields=tables[table.table_name].clustering_fields
if tables if tables
else None, else None,
max_partition_id=table.max_partition_id max_partition_id=table.get("max_partition_id"),
if "max_partition_id" in table
else None,
max_shard_id=BigqueryTableIdentifier.get_table_and_shard( max_shard_id=BigqueryTableIdentifier.get_table_and_shard(
table.table_name table.table_name
)[1] )[1]
if len(BigqueryTableIdentifier.get_table_and_shard(table.table_name)) if len(BigqueryTableIdentifier.get_table_and_shard(table.table_name))
== 2 == 2
else None, else None,
num_partitions=table.num_partitions num_partitions=table.get("num_partitions"),
if "num_partitions" in table active_billable_bytes=table.get("active_billable_bytes"),
else None, long_term_billable_bytes=table.get("long_term_billable_bytes"),
active_billable_bytes=table.active_billable_bytes
if "active_billable_bytes" in table
else None,
long_term_billable_bytes=table.long_term_billable_bytes
if "long_term_billable_bytes" in table
else None,
) )
for table in cur for table in cur
] ]