yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-27 17:02:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/snowflake): Fixed the Snowflake external URL generation issue for privatelink connections. (#14905)

Browse Source
This commit is contained in:
Tamas Nemeth 2025-10-02 17:49:59 +02:00 committed by GitHub
parent 28b866a721
commit 5b4a082c03
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 94 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py
View File

@ -73,16 +73,16 @@ class SnowsightUrlBuilder:
url_cloud_provider_suffix = ""
else:
url_cloud_provider_suffix = f".{cloud}"
if privatelink:
url = f"https://app.{account_locator}.{cloud_region_id}.privatelink.{snowflake_domain}/"
# Note: Snowsight is always accessed via the public internet (app.snowflake.com)
# even for accounts using privatelink. Privatelink only applies to database connections,
# not the Snowsight web UI.
# Standard Snowsight URL format - works for most regions
# China region may use app.snowflake.cn instead of app.snowflake.com. This is not documented, just
# guessing Based on existence of snowflake.cn domain (https://domainindex.com/domains/snowflake.cn)
if snowflake_domain == "snowflakecomputing.cn":
url = f"https://app.snowflake.cn/{cloud_region_id}{url_cloud_provider_suffix}/{account_locator}/"
else:
# Standard Snowsight URL format - works for most regions
# China region may use app.snowflake.cn instead of app.snowflake.com. This is not documented, just
# guessing Based on existence of snowflake.cn domain (https://domainindex.com/domains/snowflake.cn)
if snowflake_domain == "snowflakecomputing.cn":
url = f"https://app.snowflake.cn/{cloud_region_id}{url_cloud_provider_suffix}/{account_locator}/"
else:
url = f"https://app.snowflake.com/{cloud_region_id}{url_cloud_provider_suffix}/{account_locator}/"
url = f"https://app.snowflake.com/{cloud_region_id}{url_cloud_provider_suffix}/{account_locator}/"
return url
@staticmethod

60
metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
View File

@ -12,7 +12,7 @@
"env": "PROD",
"database": "test_db"
},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/",
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/",
"name": "TEST_DB",
"description": "Comment for TEST_DB",
"env": "PROD",
@ -26,7 +26,7 @@
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -184,13 +184,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -255,7 +255,7 @@
"database": "test_db",
"schema": "test_schema"
},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/",
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/",
"name": "TEST_SCHEMA",
"description": "comment for TEST_DB.TEST_SCHEMA",
"env": "PROD",
@ -269,7 +269,7 @@
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -640,13 +640,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -913,13 +913,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -1615,13 +1615,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -1881,13 +1881,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -1945,13 +1945,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/dynamic-table/TABLE_2/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/dynamic-table/TABLE_2/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-h1brpf",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -2534,13 +2534,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -2881,13 +2881,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -2951,13 +2951,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -3476,13 +3476,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -3643,13 +3643,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -3970,13 +3970,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},
@ -4538,13 +4538,13 @@
{
"op": "add",
"path": "/externalUrl",
"value": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/"
"value": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/"
}
]
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00-9hbbfo",
"runId": "snowflake-2022_06_07-17_00_00-j4owqs",
"lastRunId": "no-run-id-provided"
}
},

55
metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py
View File

@ -732,6 +732,61 @@ def test_create_snowsight_base_url_ap_northeast_1():
assert result == "https://app.snowflake.com/ap-northeast-1.aws/account_locator/"
def test_create_snowsight_base_url_privatelink_aws():
result = SnowsightUrlBuilder(
"test_acct", "aws_us_east_1", privatelink=True
).snowsight_base_url
assert result == "https://app.snowflake.com/us-east-1/test_acct/"
def test_create_snowsight_base_url_privatelink_gcp():
result = SnowsightUrlBuilder(
"test_account", "gcp_us_central1", privatelink=True
).snowsight_base_url
assert result == "https://app.snowflake.com/us-central1.gcp/test_account/"
def test_create_snowsight_base_url_privatelink_azure():
result = SnowsightUrlBuilder(
"test_account", "azure_eastus2", privatelink=True
).snowsight_base_url
assert result == "https://app.snowflake.com/east-us-2.azure/test_account/"
def test_snowsight_privatelink_external_urls():
url_builder = SnowsightUrlBuilder(
account_locator="test_acct",
region="aws_us_east_1",
privatelink=True,
)
# Test database URL
db_url = url_builder.get_external_url_for_database("TEST_DB")
assert (
db_url
== "https://app.snowflake.com/us-east-1/test_acct/#/data/databases/TEST_DB/"
)
# Test schema URL
schema_url = url_builder.get_external_url_for_schema("TEST_SCHEMA", "TEST_DB")
assert (
schema_url
== "https://app.snowflake.com/us-east-1/test_acct/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/"
)
# Test table URL
table_url = url_builder.get_external_url_for_table(
"TEST_TABLE",
"TEST_SCHEMA",
"TEST_DB",
domain=SnowflakeObjectDomain.TABLE,
)
assert (
table_url
== "https://app.snowflake.com/us-east-1/test_acct/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TEST_TABLE/"
)
def test_snowflake_utils() -> None:
assert_doctest(datahub.ingestion.source.snowflake.snowflake_utils)