diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py index be830637721..bfb75584a9e 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py @@ -672,7 +672,7 @@ class SnowflakeSource( self, database_name: Optional[str] = None, schema_name: Optional[str] = None ) -> str: url = ( - f"https://app.snowflake.com/{self.org_name.lower()}" + f"https://{self.service_connection.snowflakeSourceHost}/{self.org_name.lower()}" f"/{self.account.lower()}/#/data/databases/{database_name}" ) if schema_name: diff --git a/ingestion/tests/unit/topology/database/test_snowflake.py b/ingestion/tests/unit/topology/database/test_snowflake.py index dad44cfbb92..26edbce8b80 100644 --- a/ingestion/tests/unit/topology/database/test_snowflake.py +++ b/ingestion/tests/unit/topology/database/test_snowflake.py @@ -61,6 +61,22 @@ SNOWFLAKE_CONFIGURATION = { "ingestionPipelineFQN": "snowflake.mock_pipeline", } +SNOWFLAKE_CONFIGURATION_CUSTOM_HOST = { + **SNOWFLAKE_CONFIGURATION, + **{ + "source": { + **SNOWFLAKE_CONFIGURATION["source"], + "serviceConnection": { + **SNOWFLAKE_CONFIGURATION["source"]["serviceConnection"], + "config": { + **SNOWFLAKE_CONFIGURATION["source"]["serviceConnection"]["config"], + "snowflakeSourceHost": "custom.snowflake.com", + }, + }, + } + }, +} + SNOWFLAKE_INCREMENTAL_CONFIGURATION = { **SNOWFLAKE_CONFIGURATION, **{ @@ -76,6 +92,7 @@ SNOWFLAKE_INCREMENTAL_CONFIGURATION = { SNOWFLAKE_CONFIGURATIONS = { "incremental": SNOWFLAKE_INCREMENTAL_CONFIGURATION, "not_incremental": SNOWFLAKE_CONFIGURATION, + "custom_host": SNOWFLAKE_CONFIGURATION_CUSTOM_HOST, } MOCK_PIPELINE_STATUSES = [ @@ -125,6 +142,8 @@ MOCK_VIEW_NAME = "COLUMNS" MOCK_TABLE_NAME = "CALL_CENTER" EXPECTED_SNOW_URL_VIEW = "https://app.snowflake.com/random_org/random_account/#/data/databases/SNOWFLAKE_SAMPLE_DATA/schemas/INFORMATION_SCHEMA/view/COLUMNS" EXPECTED_SNOW_URL_TABLE = "https://app.snowflake.com/random_org/random_account/#/data/databases/SNOWFLAKE_SAMPLE_DATA/schemas/TPCDS_SF10TCL/table/CALL_CENTER" +EXPECTED_SNOW_URL_VIEW_CUSTOM = "https://custom.snowflake.com/random_org/random_account/#/data/databases/SNOWFLAKE_SAMPLE_DATA/schemas/INFORMATION_SCHEMA/view/COLUMNS" +EXPECTED_SNOW_URL_TABLE_CUSTOM = "https://custom.snowflake.com/random_org/random_account/#/data/databases/SNOWFLAKE_SAMPLE_DATA/schemas/TPCDS_SF10TCL/table/CALL_CENTER" def get_snowflake_sources(): @@ -146,6 +165,15 @@ def get_snowflake_sources(): SNOWFLAKE_CONFIGURATIONS["not_incremental"]["ingestionPipelineFQN"], ) + config_custom = OpenMetadataWorkflowConfig.model_validate( + SNOWFLAKE_CONFIGURATIONS["custom_host"] + ) + sources["custom_host"] = SnowflakeSource.create( + SNOWFLAKE_CONFIGURATIONS["custom_host"]["source"], + config_custom.workflowConfig.openMetadataServerConfig, + SNOWFLAKE_CONFIGURATIONS["custom_host"]["ingestionPipelineFQN"], + ) + with patch( "metadata.ingestion.source.database.incremental_metadata_extraction.IncrementalConfigCreator._get_pipeline_statuses", return_value=MOCK_PIPELINE_STATUSES, @@ -194,7 +222,13 @@ class SnowflakeUnitTest(TestCase): ) def _assert_urls(self): - for source in self.sources.values(): + for source_key, source in self.sources.items(): + if source_key == "custom_host": + expected_table_url = EXPECTED_SNOW_URL_TABLE_CUSTOM + expected_view_url = EXPECTED_SNOW_URL_VIEW_CUSTOM + else: + expected_table_url = EXPECTED_SNOW_URL_TABLE + expected_view_url = EXPECTED_SNOW_URL_VIEW self.assertEqual( source.get_source_url( database_name=MOCK_DB_NAME, @@ -202,7 +236,7 @@ class SnowflakeUnitTest(TestCase): table_name=MOCK_TABLE_NAME, table_type=TableType.Regular, ), - EXPECTED_SNOW_URL_TABLE, + expected_table_url, ) self.assertEqual( @@ -212,7 +246,7 @@ class SnowflakeUnitTest(TestCase): table_name=MOCK_VIEW_NAME, table_type=TableType.View, ), - EXPECTED_SNOW_URL_VIEW, + expected_view_url, ) def test_source_url(self): @@ -249,6 +283,46 @@ class SnowflakeUnitTest(TestCase): ) ) + def test_source_url_custom_host(self): + """ + Test source URL generation with custom snowflakeSourceHost + """ + with patch.object( + SnowflakeSource, + "account", + return_value="random_account", + new_callable=PropertyMock, + ): + with patch.object( + SnowflakeSource, + "org_name", + return_value="random_org", + new_callable=PropertyMock, + ): + custom_source = self.sources["custom_host"] + + # Test custom host URL generation for table + self.assertEqual( + custom_source.get_source_url( + database_name=MOCK_DB_NAME, + schema_name=MOCK_SCHEMA_NAME_2, + table_name=MOCK_TABLE_NAME, + table_type=TableType.Regular, + ), + EXPECTED_SNOW_URL_TABLE_CUSTOM, + ) + + # Test custom host URL generation for view + self.assertEqual( + custom_source.get_source_url( + database_name=MOCK_DB_NAME, + schema_name=MOCK_SCHEMA_NAME_1, + table_name=MOCK_VIEW_NAME, + table_type=TableType.View, + ), + EXPECTED_SNOW_URL_VIEW_CUSTOM, + ) + def test_stored_procedure_validator(self): """Review how we are building the SP signature""" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json index f27daadb173..43387baf847 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json @@ -181,6 +181,12 @@ "description": "Cost of credit for the Snowflake account.", "type": "number", "default": 3.3 + }, + "snowflakeSourceHost": { + "title": "Snowflake Source Host", + "description": "Snowflake source host for the Snowflake account.", + "type": "string", + "default": "app.snowflake.com" } }, "additionalProperties": false, diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md index 74989cbc962..fe090233f79 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md @@ -164,6 +164,21 @@ $$section Optional Configuration to keep the session active in case the ingestion job runs for longer duration. $$ +$$section +### Cost of Credit $(id="creditCost") + +This value is used to calculate the Total Cost shown in the "Most Expensive Queries" section under Insights, by multiplying: +`Credits Consumed by the Query × Cost of Credit` +$$ + +$$section +### Snowflake Source Host $(id="snowflakeSourceHost") + +Set the Snowflake host name to be used while forming entity source URL. Entity source URLs are used in `View on Snowflake` button to open any entity in Snowflake. + +Default snowflake host is `app.snowflake.com`. It only needs to be updated if you are using PrivateLink or any other custom host for Snowflake. +$$ + $$section ### Connection Options $(id="connectionOptions") diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts index 2116edb269f..ff8cb148e51 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts @@ -994,13 +994,22 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. + * SSL Configuration details. + * + * SSL Configuration for OpenMetadata Server */ - authMechanism?: AuthMechanismEnum; + sslConfig?: SSLConfigObject; /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Choose Auth Config Type. * @@ -1011,12 +1020,6 @@ export interface ConfigObject { * Authentication type to connect to Apache Ranger. */ authType?: AuthConfigurationType | NoConfigAuthenticationTypes; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; /** * Use slow logs to extract lineage. */ @@ -1118,6 +1121,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts index 083498b2ee0..0bc9595ae47 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts @@ -608,21 +608,24 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. + * SSL Configuration details. */ - authMechanism?: AuthMechanismEnum; + sslConfig?: Config; /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Choose Auth Config Type. */ authType?: AuthConfigurationType | NoConfigAuthenticationTypes; - /** - * SSL Configuration details. - */ - sslConfig?: Config; /** * Use slow logs to extract lineage. */ @@ -724,6 +727,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts index 7e07bae07c9..4f16dc0e801 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts @@ -3342,13 +3342,16 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Use slow logs to extract lineage. */ @@ -3450,6 +3453,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts index 58bfa1d770e..3520774b602 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts @@ -876,13 +876,22 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. + * SSL Configuration details. + * + * SSL Configuration for OpenMetadata Server */ - authMechanism?: AuthMechanismEnum; + sslConfig?: SSLConfigObject; /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Choose Auth Config Type. * @@ -893,12 +902,6 @@ export interface ConfigObject { * Authentication type to connect to Apache Ranger. */ authType?: AuthConfigurationType | NoConfigAuthenticationTypes; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; /** * Use slow logs to extract lineage. */ @@ -1000,6 +1003,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts index 50e1e6cbdfb..f0b90c09e05 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts @@ -1412,13 +1412,22 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. + * SSL Configuration details. + * + * SSL Configuration for OpenMetadata Server */ - authMechanism?: AuthMechanismEnum; + sslConfig?: SSLConfigObject; /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Choose Auth Config Type. * @@ -1429,12 +1438,6 @@ export interface ConfigObject { * Authentication type to connect to Apache Ranger. */ authType?: AuthConfigurationType | NoConfigAuthenticationTypes; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; /** * Use slow logs to extract lineage. */ @@ -1536,6 +1539,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/snowflakeConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/snowflakeConnection.ts index a0f0be4077e..b3d59874d66 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/snowflakeConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/snowflakeConnection.ts @@ -82,7 +82,11 @@ export interface SnowflakeConnection { /** * Snowflake Passphrase Key used with Private Key */ - snowflakePrivatekeyPassphrase?: string; + snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; supportsDatabase?: boolean; supportsDataDiff?: boolean; supportsDBTExtraction?: boolean; diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts index ed3e7c63d3e..fa0f2fe0006 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts @@ -1149,13 +1149,16 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Use slow logs to extract lineage. */ @@ -1257,6 +1260,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts index a1ddefbf641..2b19c1b42fc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts @@ -727,21 +727,24 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. + * SSL Configuration details. */ - authMechanism?: AuthMechanismEnum; + sslConfig?: Config; /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Choose Auth Config Type. */ authType?: AuthConfigurationType | NoConfigAuthenticationTypes; - /** - * SSL Configuration details. - */ - sslConfig?: Config; /** * Use slow logs to extract lineage. */ @@ -843,6 +846,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts index 59089675d9c..28b95723fce 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts @@ -3853,13 +3853,16 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Use slow logs to extract lineage. */ @@ -3961,6 +3964,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts index ec4f55fb0e6..62e53cb397b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts @@ -1193,13 +1193,16 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Use slow logs to extract lineage. */ @@ -1301,6 +1304,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts index 200fd2c7258..fa4b9a9c4a7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts @@ -1229,13 +1229,16 @@ export interface ConfigObject { */ metastoreConnection?: HiveMetastoreConnectionDetails; /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * * Establish secure connection with Impala */ useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; /** * Use slow logs to extract lineage. */ @@ -1337,6 +1340,10 @@ export interface ConfigObject { * Snowflake Passphrase Key used with Private Key */ snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; /** * Snowflake warehouse. */