yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-12 18:47:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/bigquery): remove last modified timestamp fallback (#10431)

Browse Source
This commit is contained in:
Harshal Sheth 2024-05-06 16:30:04 -07:00 committed by GitHub
parent 6a24ed2743
commit 1dae37a8ed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
View File

@ -275,7 +275,7 @@ class BigQuerySchemaApi:
table.get("last_altered") / 1000, tz=timezone.utc table.get("last_altered") / 1000, tz=timezone.utc
) )
if table.get("last_altered") is not None if table.get("last_altered") is not None
else table.created, else None,
size_in_bytes=table.get("bytes"), size_in_bytes=table.get("bytes"),
rows_count=table.get("row_count"), rows_count=table.get("row_count"),
comment=table.comment, comment=table.comment,
@ -339,7 +339,7 @@ class BigQuerySchemaApi:
view.get("last_altered") / 1000, tz=timezone.utc view.get("last_altered") / 1000, tz=timezone.utc
) )
if view.get("last_altered") is not None if view.get("last_altered") is not None
else view.created, else None,
comment=view.comment, comment=view.comment,
view_definition=view.view_definition, view_definition=view.view_definition,
materialized=view.table_type == BigqueryTableType.MATERIALIZED_VIEW, materialized=view.table_type == BigqueryTableType.MATERIALIZED_VIEW,
@ -487,7 +487,7 @@ class BigQuerySchemaApi:
snapshot.get("last_altered") / 1000, tz=timezone.utc snapshot.get("last_altered") / 1000, tz=timezone.utc
) )
if snapshot.get("last_altered") is not None if snapshot.get("last_altered") is not None
else snapshot.created, else None,
comment=snapshot.comment, comment=snapshot.comment,
ddl=snapshot.ddl, ddl=snapshot.ddl,
snapshot_time=snapshot.snapshot_time, snapshot_time=snapshot.snapshot_time,

1
metadata-ingestion/tests/integration/mysql/docker-compose.yml
View File

@ -4,7 +4,6 @@ services:
image: mysql:8 image: mysql:8
# image: mariadb:10.5.8 # Uncomment to run on m1 # image: mariadb:10.5.8 # Uncomment to run on m1
container_name: "testmysql" container_name: "testmysql"
command: --default-authentication-plugin=mysql_native_password
environment: environment:
MYSQL_ROOT_PASSWORD: example MYSQL_ROOT_PASSWORD: example
ports: ports:

2
metadata-ingestion/tests/unit/test_bigquery_source.py
View File

@ -829,7 +829,7 @@ def bigquery_view_2() -> BigqueryView:
return BigqueryView( return BigqueryView(
name="table2", name="table2",
created=now, created=now,
last_altered=now, last_altered=None,
comment="comment2", comment="comment2",
view_definition="CREATE VIEW 2", view_definition="CREATE VIEW 2",
materialized=True, materialized=True,