yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-02 11:39:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #22452: [Snowflake] Add custom host support for View in Snowflake source url (#23209)

Browse Source 
(cherry picked from commit f2fd8a9107bdaaccdb6ca35f8515e1f273c3c1f7)
This commit is contained in:
Mohit Tilala 2025-09-03 14:13:03 +05:30 committed by OpenMetadata Release Bot
parent 2ec2b6af07
commit aafc6d468f
15 changed files with 230 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py
View File

@ -672,7 +672,7 @@ class SnowflakeSource(
self, database_name: Optional[str] = None, schema_name: Optional[str] = None
) -> str:
url = (
f"https://app.snowflake.com/{self.org_name.lower()}"
f"https://{self.service_connection.snowflakeSourceHost}/{self.org_name.lower()}"
f"/{self.account.lower()}/#/data/databases/{database_name}"
)
if schema_name:

80
ingestion/tests/unit/topology/database/test_snowflake.py
View File

@ -61,6 +61,22 @@ SNOWFLAKE_CONFIGURATION = {
"ingestionPipelineFQN": "snowflake.mock_pipeline",
}
SNOWFLAKE_CONFIGURATION_CUSTOM_HOST = {
**SNOWFLAKE_CONFIGURATION,
**{
"source": {
**SNOWFLAKE_CONFIGURATION["source"],
"serviceConnection": {
**SNOWFLAKE_CONFIGURATION["source"]["serviceConnection"],
"config": {
**SNOWFLAKE_CONFIGURATION["source"]["serviceConnection"]["config"],
"snowflakeSourceHost": "custom.snowflake.com",
},
},
}
},
}
SNOWFLAKE_INCREMENTAL_CONFIGURATION = {
**SNOWFLAKE_CONFIGURATION,
**{
@ -76,6 +92,7 @@ SNOWFLAKE_INCREMENTAL_CONFIGURATION = {
SNOWFLAKE_CONFIGURATIONS = {
"incremental": SNOWFLAKE_INCREMENTAL_CONFIGURATION,
"not_incremental": SNOWFLAKE_CONFIGURATION,
"custom_host": SNOWFLAKE_CONFIGURATION_CUSTOM_HOST,
}
MOCK_PIPELINE_STATUSES = [
@ -125,6 +142,8 @@ MOCK_VIEW_NAME = "COLUMNS"
MOCK_TABLE_NAME = "CALL_CENTER"
EXPECTED_SNOW_URL_VIEW = "https://app.snowflake.com/random_org/random_account/#/data/databases/SNOWFLAKE_SAMPLE_DATA/schemas/INFORMATION_SCHEMA/view/COLUMNS"
EXPECTED_SNOW_URL_TABLE = "https://app.snowflake.com/random_org/random_account/#/data/databases/SNOWFLAKE_SAMPLE_DATA/schemas/TPCDS_SF10TCL/table/CALL_CENTER"
EXPECTED_SNOW_URL_VIEW_CUSTOM = "https://custom.snowflake.com/random_org/random_account/#/data/databases/SNOWFLAKE_SAMPLE_DATA/schemas/INFORMATION_SCHEMA/view/COLUMNS"
EXPECTED_SNOW_URL_TABLE_CUSTOM = "https://custom.snowflake.com/random_org/random_account/#/data/databases/SNOWFLAKE_SAMPLE_DATA/schemas/TPCDS_SF10TCL/table/CALL_CENTER"
def get_snowflake_sources():
@ -146,6 +165,15 @@ def get_snowflake_sources():
SNOWFLAKE_CONFIGURATIONS["not_incremental"]["ingestionPipelineFQN"],
)
config_custom = OpenMetadataWorkflowConfig.model_validate(
SNOWFLAKE_CONFIGURATIONS["custom_host"]
)
sources["custom_host"] = SnowflakeSource.create(
SNOWFLAKE_CONFIGURATIONS["custom_host"]["source"],
config_custom.workflowConfig.openMetadataServerConfig,
SNOWFLAKE_CONFIGURATIONS["custom_host"]["ingestionPipelineFQN"],
)
with patch(
"metadata.ingestion.source.database.incremental_metadata_extraction.IncrementalConfigCreator._get_pipeline_statuses",
return_value=MOCK_PIPELINE_STATUSES,
@ -194,7 +222,13 @@ class SnowflakeUnitTest(TestCase):
)
def _assert_urls(self):
for source in self.sources.values():
for source_key, source in self.sources.items():
if source_key == "custom_host":
expected_table_url = EXPECTED_SNOW_URL_TABLE_CUSTOM
expected_view_url = EXPECTED_SNOW_URL_VIEW_CUSTOM
else:
expected_table_url = EXPECTED_SNOW_URL_TABLE
expected_view_url = EXPECTED_SNOW_URL_VIEW
self.assertEqual(
source.get_source_url(
database_name=MOCK_DB_NAME,
@ -202,7 +236,7 @@ class SnowflakeUnitTest(TestCase):
table_name=MOCK_TABLE_NAME,
table_type=TableType.Regular,
),
EXPECTED_SNOW_URL_TABLE,
expected_table_url,
)
self.assertEqual(
@ -212,7 +246,7 @@ class SnowflakeUnitTest(TestCase):
table_name=MOCK_VIEW_NAME,
table_type=TableType.View,
),
EXPECTED_SNOW_URL_VIEW,
expected_view_url,
)
def test_source_url(self):
@ -249,6 +283,46 @@ class SnowflakeUnitTest(TestCase):
)
)
def test_source_url_custom_host(self):
"""
Test source URL generation with custom snowflakeSourceHost
"""
with patch.object(
SnowflakeSource,
"account",
return_value="random_account",
new_callable=PropertyMock,
):
with patch.object(
SnowflakeSource,
"org_name",
return_value="random_org",
new_callable=PropertyMock,
):
custom_source = self.sources["custom_host"]
# Test custom host URL generation for table
self.assertEqual(
custom_source.get_source_url(
database_name=MOCK_DB_NAME,
schema_name=MOCK_SCHEMA_NAME_2,
table_name=MOCK_TABLE_NAME,
table_type=TableType.Regular,
),
EXPECTED_SNOW_URL_TABLE_CUSTOM,
)
# Test custom host URL generation for view
self.assertEqual(
custom_source.get_source_url(
database_name=MOCK_DB_NAME,
schema_name=MOCK_SCHEMA_NAME_1,
table_name=MOCK_VIEW_NAME,
table_type=TableType.View,
),
EXPECTED_SNOW_URL_VIEW_CUSTOM,
)
def test_stored_procedure_validator(self):
"""Review how we are building the SP signature"""

6
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json
View File

@ -181,6 +181,12 @@
"description": "Cost of credit for the Snowflake account.",
"type": "number",
"default": 3.3
},
"snowflakeSourceHost": {
"title": "Snowflake Source Host",
"description": "Snowflake source host for the Snowflake account.",
"type": "string",
"default": "app.snowflake.com"
}
},
"additionalProperties": false,

15
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md
View File

@ -164,6 +164,21 @@ $$section
Optional Configuration to keep the session active in case the ingestion job runs for longer duration.
$$
$$section
### Cost of Credit $(id="creditCost")
This value is used to calculate the Total Cost shown in the "Most Expensive Queries" section under Insights, by multiplying:
`Credits Consumed by the Query × Cost of Credit`
$$
$$section
### Snowflake Source Host $(id="snowflakeSourceHost")
Set the Snowflake host name to be used while forming entity source URL. Entity source URLs are used in `View on Snowflake` button to open any entity in Snowflake.
Default snowflake host is `app.snowflake.com`. It only needs to be updated if you are using PrivateLink or any other custom host for Snowflake.
$$
$$section
### Connection Options $(id="connectionOptions")

23
openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts
View File

@ -994,13 +994,22 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
* SSL Configuration details.
*
* SSL Configuration for OpenMetadata Server
*/
authMechanism?: AuthMechanismEnum;
sslConfig?: SSLConfigObject;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Choose Auth Config Type.
*
@ -1011,12 +1020,6 @@ export interface ConfigObject {
* Authentication type to connect to Apache Ranger.
*/
authType?: AuthConfigurationType | NoConfigAuthenticationTypes;
/**
* SSL Configuration details.
*
* SSL Configuration for OpenMetadata Server
*/
sslConfig?: SSLConfigObject;
/**
* Use slow logs to extract lineage.
*/
@ -1118,6 +1121,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

19
openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts
View File

@ -608,21 +608,24 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
* SSL Configuration details.
*/
authMechanism?: AuthMechanismEnum;
sslConfig?: Config;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Choose Auth Config Type.
*/
authType?: AuthConfigurationType | NoConfigAuthenticationTypes;
/**
* SSL Configuration details.
*/
sslConfig?: Config;
/**
* Use slow logs to extract lineage.
*/
@ -724,6 +727,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

15
openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts
View File

@ -3342,13 +3342,16 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Use slow logs to extract lineage.
*/
@ -3450,6 +3453,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

23
openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts
View File

@ -876,13 +876,22 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
* SSL Configuration details.
*
* SSL Configuration for OpenMetadata Server
*/
authMechanism?: AuthMechanismEnum;
sslConfig?: SSLConfigObject;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Choose Auth Config Type.
*
@ -893,12 +902,6 @@ export interface ConfigObject {
* Authentication type to connect to Apache Ranger.
*/
authType?: AuthConfigurationType | NoConfigAuthenticationTypes;
/**
* SSL Configuration details.
*
* SSL Configuration for OpenMetadata Server
*/
sslConfig?: SSLConfigObject;
/**
* Use slow logs to extract lineage.
*/
@ -1000,6 +1003,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

23
openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts
View File

@ -1412,13 +1412,22 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
* SSL Configuration details.
*
* SSL Configuration for OpenMetadata Server
*/
authMechanism?: AuthMechanismEnum;
sslConfig?: SSLConfigObject;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Choose Auth Config Type.
*
@ -1429,12 +1438,6 @@ export interface ConfigObject {
* Authentication type to connect to Apache Ranger.
*/
authType?: AuthConfigurationType | NoConfigAuthenticationTypes;
/**
* SSL Configuration details.
*
* SSL Configuration for OpenMetadata Server
*/
sslConfig?: SSLConfigObject;
/**
* Use slow logs to extract lineage.
*/
@ -1536,6 +1539,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

6
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/snowflakeConnection.ts
View File

@ -82,7 +82,11 @@ export interface SnowflakeConnection {
/**
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
supportsDatabase?: boolean;
supportsDataDiff?: boolean;
supportsDBTExtraction?: boolean;

15
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts
View File

@ -1149,13 +1149,16 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Use slow logs to extract lineage.
*/
@ -1257,6 +1260,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

19
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
View File

@ -727,21 +727,24 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
* SSL Configuration details.
*/
authMechanism?: AuthMechanismEnum;
sslConfig?: Config;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Choose Auth Config Type.
*/
authType?: AuthConfigurationType | NoConfigAuthenticationTypes;
/**
* SSL Configuration details.
*/
sslConfig?: Config;
/**
* Use slow logs to extract lineage.
*/
@ -843,6 +846,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

15
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts
View File

@ -3853,13 +3853,16 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Use slow logs to extract lineage.
*/
@ -3961,6 +3964,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

15
openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts
View File

@ -1193,13 +1193,16 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Use slow logs to extract lineage.
*/
@ -1301,6 +1304,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/

15
openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts
View File

@ -1229,13 +1229,16 @@ export interface ConfigObject {
*/
metastoreConnection?: HiveMetastoreConnectionDetails;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure
* communication.
*
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Authentication mode to connect to Impala.
*/
authMechanism?: AuthMechanismEnum;
/**
* Use slow logs to extract lineage.
*/
@ -1337,6 +1340,10 @@ export interface ConfigObject {
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Snowflake source host for the Snowflake account.
*/
snowflakeSourceHost?: string;
/**
* Snowflake warehouse.
*/