yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-19 06:38:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(iceberg): listnamespaces includes warehouse name as root (#12761)

Browse Source
This commit is contained in:
Chakru 2025-03-04 01:55:41 +05:30 committed by GitHub
parent 6c57057f51
commit 5790b84920
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 30 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
metadata-service/iceberg-catalog/pyproject.toml
View File

@ -44,3 +44,9 @@ disallow_untyped_defs = false
[tool.pyright] [tool.pyright]
extraPaths = ['tests'] extraPaths = ['tests']
[tool.pytest.ini_options]
markers = [
"quick: marks limited tests for a quick validation",
"serial",
]

16
metadata-service/iceberg-catalog/src/integrationTest/integration_test.py
View File

@ -212,6 +212,9 @@ def test_iceberg_quick(spark_session, warehouse):
_test_basic_view_ops(spark_session) _test_basic_view_ops(spark_session)
_test_rename_ops(spark_session) _test_rename_ops(spark_session)
result = spark_session.sql("show namespaces")
assert (result[result["namespace"] == "default"].count() == 1)
def _create_table(spark_session, ns, table_name): def _create_table(spark_session, ns, table_name):
spark_session.sql("create namespace if not exists default") spark_session.sql("create namespace if not exists default")
@ -221,12 +224,19 @@ def _create_table(spark_session, ns, table_name):
spark_session.sql(f"insert into {ns}.{table_name} values (1, 'foo' ) ") spark_session.sql(f"insert into {ns}.{table_name} values (1, 'foo' ) ")
result = spark_session.sql("show namespaces")
assert (result[result["namespace"] == "default"].count() == 1)
result = spark_session.sql("show namespaces in default")
assert (result[result["namespace"] == f"{ns}"].count() == 1)
def test_load_tables(spark_session, warehouse): def test_load_tables(spark_session, warehouse):
namespace_count = 3 namespace_count = 2
table_count = 4 table_count = 2
for ns_index in range(namespace_count): for ns_index in range(namespace_count):
ns = f"default_ns{ns_index}" ns = f"default.ns{ns_index}"
for table_index in range(table_count): for table_index in range(table_count):
table_name = f"table_{table_index}" table_name = f"table_{table_index}"
_create_table(spark_session, ns, table_name) _create_table(spark_session, ns, table_name)

6
metadata-service/iceberg-catalog/src/main/java/io/datahubproject/iceberg/catalog/Utils.java
View File

@ -75,6 +75,10 @@ public class Utils {
} }
public static String namespaceNameFromContainerUrn(Urn urn) { public static String namespaceNameFromContainerUrn(Urn urn) {
return urn.toString().substring(NAMESPACE_CONTAINER_PREFIX.length()); // Must do inverse of implementation of method containerUrn(String platformInstance, String[]
// levels) in this file
String namespaceWithPlatformInstance =
urn.toString().substring(NAMESPACE_CONTAINER_PREFIX.length());
return namespaceWithPlatformInstance.substring(namespaceWithPlatformInstance.indexOf('.') + 1);
} }
} }

4
metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/DataHubRestCatalogTest.java
View File

@ -275,8 +275,8 @@ public class DataHubRestCatalogTest {
SearchResult mockResult = mock(SearchResult.class); SearchResult mockResult = mock(SearchResult.class);
List<SearchEntity> entitiesList = List<SearchEntity> entitiesList =
Arrays.asList( Arrays.asList(
createSearchEntity("urn:li:container:iceberg__parent.ns1"), createSearchEntity("urn:li:container:iceberg__platformInstance.parent.ns1"),
createSearchEntity("urn:li:container:iceberg__parent.ns2")); createSearchEntity("urn:li:container:iceberg__platformInstance.parent.ns2"));
SearchEntityArray entities = new SearchEntityArray(); SearchEntityArray entities = new SearchEntityArray();
entities.addAll(entitiesList); entities.addAll(entitiesList);
when(mockResult.getEntities()).thenReturn(entities); when(mockResult.getEntities()).thenReturn(entities);

4
metadata-service/iceberg-catalog/src/test/java/io/datahubproject/iceberg/catalog/UtilsTest.java
View File

@ -42,6 +42,10 @@ public class UtilsTest {
assertNotNull(containerUrn); assertNotNull(containerUrn);
assertEquals(containerUrn.toString(), "urn:li:container:iceberg__testInstance.db.schema"); assertEquals(containerUrn.toString(), "urn:li:container:iceberg__testInstance.db.schema");
Namespace namespaceFromUrn =
Namespace.of(Utils.namespaceNameFromContainerUrn(containerUrn).split("\\."));
assertEquals(namespaceFromUrn, namespace);
} }
@Test @Test