From 5790b849200b45a94e1a80ef191ea18ddf605f98 Mon Sep 17 00:00:00 2001 From: Chakru <161002324+chakru-r@users.noreply.github.com> Date: Tue, 4 Mar 2025 01:55:41 +0530 Subject: [PATCH] fix(iceberg): listnamespaces includes warehouse name as root (#12761) --- metadata-service/iceberg-catalog/pyproject.toml | 6 ++++++ .../src/integrationTest/integration_test.py | 16 +++++++++++++--- .../io/datahubproject/iceberg/catalog/Utils.java | 6 +++++- .../iceberg/catalog/DataHubRestCatalogTest.java | 4 ++-- .../iceberg/catalog/UtilsTest.java | 4 ++++ 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/metadata-service/iceberg-catalog/pyproject.toml b/metadata-service/iceberg-catalog/pyproject.toml index 71998585bb..f4d974c3e6 100644 --- a/metadata-service/iceberg-catalog/pyproject.toml +++ b/metadata-service/iceberg-catalog/pyproject.toml @@ -44,3 +44,9 @@ disallow_untyped_defs = false [tool.pyright] extraPaths = ['tests'] + +[tool.pytest.ini_options] +markers = [ + "quick: marks limited tests for a quick validation", + "serial", +] \ No newline at end of file diff --git a/metadata-service/iceberg-catalog/src/integrationTest/integration_test.py b/metadata-service/iceberg-catalog/src/integrationTest/integration_test.py index 3a1351e364..f96082fbd0 100644 --- a/metadata-service/iceberg-catalog/src/integrationTest/integration_test.py +++ b/metadata-service/iceberg-catalog/src/integrationTest/integration_test.py @@ -212,6 +212,9 @@ def test_iceberg_quick(spark_session, warehouse): _test_basic_view_ops(spark_session) _test_rename_ops(spark_session) + result = spark_session.sql("show namespaces") + assert (result[result["namespace"] == "default"].count() == 1) + def _create_table(spark_session, ns, table_name): spark_session.sql("create namespace if not exists default") @@ -221,12 +224,19 @@ def _create_table(spark_session, ns, table_name): spark_session.sql(f"insert into {ns}.{table_name} values (1, 'foo' ) ") + result = spark_session.sql("show namespaces") + assert (result[result["namespace"] == "default"].count() == 1) + + result = spark_session.sql("show namespaces in default") + assert (result[result["namespace"] == f"{ns}"].count() == 1) + + def test_load_tables(spark_session, warehouse): - namespace_count = 3 - table_count = 4 + namespace_count = 2 + table_count = 2 for ns_index in range(namespace_count): - ns = f"default_ns{ns_index}" + ns = f"default.ns{ns_index}" for table_index in range(table_count): table_name = f"table_{table_index}" _create_table(spark_session, ns, table_name) diff --git a/metadata-service/iceberg-catalog/src/main/java/io/datahubproject/iceberg/catalog/Utils.java b/metadata-service/iceberg-catalog/src/main/java/io/datahubproject/iceberg/catalog/Utils.java index 455eae4f3e..b02fcc66fa 100644 --- a/metadata-service/iceberg-catalog/src/main/java/io/datahubproject/iceberg/catalog/Utils.java +++ b/metadata-service/iceberg-catalog/src/main/java/io/datahubproject/iceberg/catalog/Utils.java @@ -75,6 +75,10 @@ public class Utils { } public static String namespaceNameFromContainerUrn(Urn urn) { - return urn.toString().substring(NAMESPACE_CONTAINER_PREFIX.length()); + // Must do inverse of implementation of method containerUrn(String platformInstance, String[] + // levels) in this file + String namespaceWithPlatformInstance = + urn.toString().substring(NAMESPACE_CONTAINER_PREFIX.length()); + return namespaceWithPlatformInstance.substring(namespaceWithPlatformInstance.indexOf('.') + 1); } } diff --git a/metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/DataHubRestCatalogTest.java b/metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/DataHubRestCatalogTest.java index cde5190f31..2002357ecf 100644 --- a/metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/DataHubRestCatalogTest.java +++ b/metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/DataHubRestCatalogTest.java @@ -275,8 +275,8 @@ public class DataHubRestCatalogTest { SearchResult mockResult = mock(SearchResult.class); List entitiesList = Arrays.asList( - createSearchEntity("urn:li:container:iceberg__parent.ns1"), - createSearchEntity("urn:li:container:iceberg__parent.ns2")); + createSearchEntity("urn:li:container:iceberg__platformInstance.parent.ns1"), + createSearchEntity("urn:li:container:iceberg__platformInstance.parent.ns2")); SearchEntityArray entities = new SearchEntityArray(); entities.addAll(entitiesList); when(mockResult.getEntities()).thenReturn(entities); diff --git a/metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/UtilsTest.java b/metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/UtilsTest.java index 326f4d762e..1f8db9621b 100644 --- a/metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/UtilsTest.java +++ b/metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/UtilsTest.java @@ -42,6 +42,10 @@ public class UtilsTest { assertNotNull(containerUrn); assertEquals(containerUrn.toString(), "urn:li:container:iceberg__testInstance.db.schema"); + + Namespace namespaceFromUrn = + Namespace.of(Utils.namespaceNameFromContainerUrn(containerUrn).split("\\.")); + assertEquals(namespaceFromUrn, namespace); } @Test