yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-24 08:28:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/iceberg): Iceberg table name (#11599)

Browse Source
This commit is contained in:
skrydal 2024-10-11 18:48:25 +02:00 committed by GitHub
parent 14c79389f5
commit ebcce1f87b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py
View File

@ -182,6 +182,7 @@ class IcebergSource(StatefulIngestionSourceBase):
custom_properties["snapshot-id"] = str(table.current_snapshot().snapshot_id)
custom_properties["manifest-list"] = table.current_snapshot().manifest_list
dataset_properties = DatasetPropertiesClass(
name=table.name()[-1],
tags=[],
description=table.metadata.properties.get("comment", None),
customProperties=custom_properties,

2
metadata-ingestion/tests/integration/iceberg/docker-compose.yml
View File

@ -1,5 +1,3 @@
version: "3"
services:
spark-iceberg:
image: tabulario/spark-iceberg:3.3.2_1.3.0

1
metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json
View File

@ -11,6 +11,7 @@
},
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"name": "another_taxis",
"customProperties": {
"owner": "root",
"created-at": "2024-06-27T17:29:32.492204247Z",

1
metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json
View File

@ -11,6 +11,7 @@
},
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"name": "taxis",
"customProperties": {
"owner": "root",
"created-at": "2024-05-22T14:08:04.001538500Z",

1
metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json
View File

@ -11,6 +11,7 @@
},
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"name": "taxis",
"customProperties": {
"owner": "root",
"created-at": "2024-05-22T14:10:22.926080700Z",