From 5236950b6d6fb0dfd3731f99ebcd314659dd18f4 Mon Sep 17 00:00:00 2001 From: Matt Chamberlin Date: Tue, 11 Jun 2024 05:37:57 -0400 Subject: [PATCH] Fixes 16595: Parse Iceberg REST table FQN from identifier (#16596) --- .../ingestion/source/database/iceberg/catalog/rest.py | 2 +- .../metadata/ingestion/source/database/iceberg/helper.py | 8 ++++---- .../ingestion/source/database/iceberg/metadata.py | 3 ++- .../database/iceberg/restCatalogConnection.json | 6 +----- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/iceberg/catalog/rest.py b/ingestion/src/metadata/ingestion/source/database/iceberg/catalog/rest.py index c5a453e280e..f82d9bf77c3 100644 --- a/ingestion/src/metadata/ingestion/source/database/iceberg/catalog/rest.py +++ b/ingestion/src/metadata/ingestion/source/database/iceberg/catalog/rest.py @@ -37,7 +37,7 @@ class IcebergRestCatalog(IcebergCatalogBase): "'connection' is not an instance of 'RestCatalogConnection'" ) - if catalog.connection.credential: + if catalog.connection.credential and catalog.connection.credential.clientSecret: client_id = catalog.connection.credential.clientId.get_secret_value() client_secret = ( catalog.connection.credential.clientSecret.get_secret_value() diff --git a/ingestion/src/metadata/ingestion/source/database/iceberg/helper.py b/ingestion/src/metadata/ingestion/source/database/iceberg/helper.py index dba5569820c..1dcd3be528e 100644 --- a/ingestion/src/metadata/ingestion/source/database/iceberg/helper.py +++ b/ingestion/src/metadata/ingestion/source/database/iceberg/helper.py @@ -37,14 +37,14 @@ def namespace_to_str(namespace: tuple[str]) -> str: return ".".join(namespace) -def get_table_name_as_str(table: pyiceberg.table.Table) -> str: - """Returns the Table Name as Tring from a PyIceberg Table. +def get_table_name_as_str(table: Tuple[str]) -> str: + """Returns the Table Name as String from a PyIceberg table identifier tuple. The PyIceberg table name is returned as tuple and we turn them into a String concatenating the items with a '.' in between. """ # We are skipping the first item because it is the schema name. - return ".".join(table.name()[1:]) + return ".".join(table[1:]) def get_column_from_partition( @@ -94,7 +94,7 @@ def get_column_partition_type( def get_owner_from_table( table: pyiceberg.table.Table, property_key: str ) -> Optional[str]: - """Retrives the owner information from given Table Property.""" + """Retrieves the owner information from given Table Property.""" return table.properties.get(property_key) diff --git a/ingestion/src/metadata/ingestion/source/database/iceberg/metadata.py b/ingestion/src/metadata/ingestion/source/database/iceberg/metadata.py index 97e4a1fbee8..855d7fbd08d 100644 --- a/ingestion/src/metadata/ingestion/source/database/iceberg/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/iceberg/metadata.py @@ -181,7 +181,8 @@ class IcebergSource(DatabaseServiceSource): for table_identifier in self.iceberg.list_tables(namespace): try: table = self.iceberg.load_table(table_identifier) - table_name = get_table_name_as_str(table) + # extract table name from table identifier, which does not include catalog name + table_name = get_table_name_as_str(table_identifier) table_fqn = fqn.build( self.metadata, entity_type=Table, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/iceberg/restCatalogConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/iceberg/restCatalogConnection.json index f48ea55840c..45a6df742ad 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/iceberg/restCatalogConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/iceberg/restCatalogConnection.json @@ -30,11 +30,7 @@ "format": "password" } }, - "additionalProperties": false, - "required": [ - "clientId", - "clientSecret" - ] + "additionalProperties": false }, "token": { "title": "Token",