diff --git a/ingestion/src/airflow_provider_openmetadata/hooks/openmetadata.py b/ingestion/src/airflow_provider_openmetadata/hooks/openmetadata.py index 4305e69abee..ceccf8558bf 100644 --- a/ingestion/src/airflow_provider_openmetadata/hooks/openmetadata.py +++ b/ingestion/src/airflow_provider_openmetadata/hooks/openmetadata.py @@ -26,7 +26,7 @@ from metadata.generated.schema.security.client.openMetadataJWTClientConfig impor OpenMetadataJWTClientConfig, ) from metadata.generated.schema.security.ssl.validateSSLClientConfig import ( - ValidateSSLClientConfig, + ValidateSslClientConfig, ) from metadata.generated.schema.security.ssl.verifySSLConfig import VerifySSL from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -66,7 +66,7 @@ class OpenMetadataHook(BaseHook): extra = conn.extra_dejson if conn.get_extra() else {} verify_ssl = extra.get("verifySSL") or self.default_verify_ssl ssl_config = ( - ValidateSSLClientConfig(certificatePath=extra["sslConfig"]) + ValidateSslClientConfig(certificatePath=extra["sslConfig"]) if extra.get("sslConfig") else self.default_ssl_config ) diff --git a/ingestion/src/metadata/examples/workflows/superset.yaml b/ingestion/src/metadata/examples/workflows/superset.yaml index 6a0fde0db2c..bb2e9b630e7 100644 --- a/ingestion/src/metadata/examples/workflows/superset.yaml +++ b/ingestion/src/metadata/examples/workflows/superset.yaml @@ -14,6 +14,9 @@ source: # username: admin # password: admin # provider: db + # verifySSL: no-ssl / ignore / validate + # sslConfig: + # certificatePath: CA certificate path. E.g., /path/to/public.cert. Will be used if Verify SSL is set to `validate`. type: Superset sourceConfig: config: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py index b48096c2765..b962f1825fa 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py @@ -28,7 +28,7 @@ from metadata.ingestion.api.models import Either, StackTraceError from metadata.ingestion.source.dashboard.superset.mixin import SupersetSourceMixin from metadata.ingestion.source.dashboard.superset.models import ( ChartResult, - DashboradResult, + DashboardResult, ) from metadata.utils import fqn from metadata.utils.filters import filter_by_datamodel @@ -63,7 +63,7 @@ class SupersetAPISource(SupersetSourceMixin): for index, chart_result in enumerate(charts.result): self.all_charts[charts.ids[index]] = chart_result - def get_dashboards_list(self) -> Iterable[DashboradResult]: + def get_dashboards_list(self) -> Iterable[DashboardResult]: """ Get List of all dashboards """ @@ -77,7 +77,7 @@ class SupersetAPISource(SupersetSourceMixin): yield dashboard def yield_dashboard( - self, dashboard_details: DashboradResult + self, dashboard_details: DashboardResult ) -> Iterable[Either[CreateDashboardRequest]]: """ Method to Get Dashboard Entity @@ -119,7 +119,7 @@ class SupersetAPISource(SupersetSourceMixin): ) def yield_dashboard_chart( - self, dashboard_details: DashboradResult + self, dashboard_details: DashboardResult ) -> Iterable[Either[CreateChartRequest]]: """Method to fetch charts linked to dashboard""" for chart_id in self._get_charts_of_dashboard(dashboard_details): @@ -175,7 +175,7 @@ class SupersetAPISource(SupersetSourceMixin): return None def yield_datamodel( - self, dashboard_details: DashboradResult + self, dashboard_details: DashboardResult ) -> Iterable[Either[CreateDashboardDataModelRequest]]: if self.source_config.includeDataModels: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py index 85c13e5f324..f8f654296a9 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py @@ -26,6 +26,7 @@ from metadata.ingestion.source.dashboard.superset.models import ( SupersetDatasource, ) from metadata.utils.logger import ometa_logger +from metadata.utils.ssl_registry import get_verify_ssl_fn logger = ometa_logger() @@ -38,12 +39,14 @@ class SupersetAuthenticationProvider(AuthenticationProvider): def __init__(self, config: SupersetConnection): self.config = config self.service_connection = self.config + get_verify_ssl = get_verify_ssl_fn(config.connection.verifySSL) client_config = ClientConfig( base_url=config.hostPort, api_version="api/v1", auth_token=lambda: ("no_token", 0), auth_header="Authorization", allow_redirects=True, + verify=get_verify_ssl(config.connection.sslConfig), ) self.client = REST(client_config) self.generated_auth_token = None @@ -85,12 +88,14 @@ class SupersetAPIClient: def __init__(self, config: SupersetConnection): self.config = config self._auth_provider = SupersetAuthenticationProvider.create(config) + get_verify_ssl = get_verify_ssl_fn(config.connection.verifySSL) client_config = ClientConfig( base_url=config.hostPort, api_version="api/v1", auth_token=self._auth_provider.get_access_token, auth_header="Authorization", allow_redirects=True, + verify=get_verify_ssl(config.connection.sslConfig), ) self.client = REST(client_config) @@ -176,7 +181,7 @@ class SupersetAPIClient: return chart_list except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard list") + logger.warning("Failed to fetch the charts list") return SupersetChart() def fetch_charts_with_id(self, chart_id: str): @@ -200,7 +205,7 @@ class SupersetAPIClient: return datasource_list except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard list") + logger.warning("Failed to fetch the datasource list") return SupersetDatasource() diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py index 95c31562fa3..1c5deb549cc 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py @@ -36,7 +36,7 @@ from metadata.ingestion.api.models import Either, StackTraceError from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource from metadata.ingestion.source.dashboard.superset.models import ( - DashboradResult, + DashboardResult, DataSourceResult, FetchChart, FetchColumn, @@ -76,7 +76,7 @@ class SupersetSourceMixin(DashboardServiceSource): return cls(config, metadata_config) def get_dashboard_name( - self, dashboard: Union[FetchDashboard, DashboradResult] + self, dashboard: Union[FetchDashboard, DashboardResult] ) -> Optional[str]: """ Get Dashboard Name @@ -84,15 +84,15 @@ class SupersetSourceMixin(DashboardServiceSource): return dashboard.dashboard_title def get_dashboard_details( - self, dashboard: Union[FetchDashboard, DashboradResult] - ) -> Optional[Union[FetchDashboard, DashboradResult]]: + self, dashboard: Union[FetchDashboard, DashboardResult] + ) -> Optional[Union[FetchDashboard, DashboardResult]]: """ Get Dashboard Details """ return dashboard def _get_user_by_email( - self, email: Union[FetchDashboard, DashboradResult] + self, email: Union[FetchDashboard, DashboardResult] ) -> EntityReference: if email: user = self.metadata.get_user_by_email(email) @@ -102,7 +102,7 @@ class SupersetSourceMixin(DashboardServiceSource): return None def get_owner_details( - self, dashboard_details: Union[DashboradResult, FetchDashboard] + self, dashboard_details: Union[DashboardResult, FetchDashboard] ) -> EntityReference: for owner in dashboard_details.owners: if owner.email: @@ -116,7 +116,7 @@ class SupersetSourceMixin(DashboardServiceSource): return None def _get_charts_of_dashboard( - self, dashboard_details: Union[FetchDashboard, DashboradResult] + self, dashboard_details: Union[FetchDashboard, DashboardResult] ) -> Optional[List[str]]: """ Method to fetch chart ids linked to dashboard @@ -133,7 +133,7 @@ class SupersetSourceMixin(DashboardServiceSource): def yield_dashboard_lineage_details( self, - dashboard_details: Union[FetchDashboard, DashboradResult], + dashboard_details: Union[FetchDashboard, DashboardResult], db_service_name: DatabaseService, ) -> Iterable[Either[AddLineageRequest]]: """ diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/models.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/models.py index 20c3be8bb53..740f57f3cf4 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/models.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/models.py @@ -35,7 +35,7 @@ class DashOwner(BaseModel): email: Optional[str] -class DashboradResult(BaseModel): +class DashboardResult(BaseModel): dashboard_title: Optional[str] url: Optional[str] owners: Optional[List[DashOwner]] = [] @@ -48,7 +48,7 @@ class SupersetDashboardCount(BaseModel): count: Optional[int] ids: Optional[List[int]] = [] dashboard_title: Optional[str] - result: Optional[List[DashboradResult]] = [] + result: Optional[List[DashboardResult]] = [] # Chart diff --git a/openmetadata-docs/content/v1.1.x/connectors/dashboard/superset/yaml.md b/openmetadata-docs/content/v1.1.x/connectors/dashboard/superset/yaml.md index 88c6d3899bd..18ae87836d9 100644 --- a/openmetadata-docs/content/v1.1.x/connectors/dashboard/superset/yaml.md +++ b/openmetadata-docs/content/v1.1.x/connectors/dashboard/superset/yaml.md @@ -88,6 +88,15 @@ Superset only supports basic or ldap authentication through APIs so if you have **provider**: Choose between `db`(default) or `ldap` mode of Authentication provider for the Superset service. This parameter is used internally to connect to Superset's REST API. +**verifySSL**: +Client SSL verification. Make sure to configure the SSLConfig if enabled. +Possible values: +- `validate`: Validate the certificate using the public certificate (recommended). +- `ignore`: Ignore the certification validation (not recommended for production). +- `no-ssl`: SSL validation is not needed. + +**certificatePath**: CA certificate path in the instance where the ingestion run. E.g., `/path/to/public.cert`. Will be used if Verify SSL is set to `validate`. + {% /codeInfo %} {% codeInfo srNumber=2 %} @@ -177,6 +186,9 @@ source: username: admin password: admin provider: db # or provider: ldap + # verifySSL: no-ssl / ignore / validate + # sslConfig: + # certificatePath: CA certificate path. E.g., /path/to/public.cert. Will be used if Verify SSL is set to `validate`. ``` ```yaml {% srNumber=2 %} diff --git a/openmetadata-docs/content/v1.2.x-SNAPSHOT/connectors/dashboard/superset/index.md b/openmetadata-docs/content/v1.2.x-SNAPSHOT/connectors/dashboard/superset/index.md index 720c86e462f..cf15d8b6d1a 100644 --- a/openmetadata-docs/content/v1.2.x-SNAPSHOT/connectors/dashboard/superset/index.md +++ b/openmetadata-docs/content/v1.2.x-SNAPSHOT/connectors/dashboard/superset/index.md @@ -63,6 +63,16 @@ Superset only supports basic or ldap authentication through APIs so if you have - **Username**: Username to connect to Superset, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Superset to fetch the metadata. - **Password**: Password of the user account to connect with Superset. - **Provider**: Choose between `db`(default) or `ldap` mode of Authentication provider for the Superset service. This parameter is used internally to connect to Superset's REST API. +- **Verify SSL**: +Client SSL verification. Make sure to configure the SSLConfig if enabled. +Possible values: + * `validate`: Validate the certificate using the public certificate (recommended). + * `ignore`: Ignore the certification validation (not recommended for production). + * `no-ssl`: SSL validation is not needed. + +- **SSL Config**: Client SSL configuration in case we are connection to a host with SSL enabled. + +- **Certificate Path**: CA certificate path in the instance where the ingestion run. E.g., `/path/to/public.cert`. Will be used if Verify SSL is set to `validate`. #### For MySQL Connection diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/utils/supersetApiConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/utils/supersetApiConnection.json index b1777391993..322cd460765 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/utils/supersetApiConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/utils/supersetApiConnection.json @@ -30,6 +30,13 @@ "description": "Password for Superset.", "type": "string", "format": "password" + }, + "verifySSL": { + "$ref": "../../security/ssl/verifySSLConfig.json#/definitions/verifySSL", + "default": "no-ssl" + }, + "sslConfig": { + "$ref": "../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" } }, "additionalProperties": false, diff --git a/openmetadata-spec/src/main/resources/json/schema/security/ssl/validateSSLClientConfig.json b/openmetadata-spec/src/main/resources/json/schema/security/ssl/validateSSLClientConfig.json index 2df33f49ff4..7db7129e844 100644 --- a/openmetadata-spec/src/main/resources/json/schema/security/ssl/validateSSLClientConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/security/ssl/validateSSLClientConfig.json @@ -1,12 +1,13 @@ { "$id": "https://open-metadata.org/schema/security/ssl/validateSSLClientConfig.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ValidateSSLClientConfig", + "title": "Validate SSL Client Config", "description": "OpenMetadata Client configured to validate SSL certificates.", "type": "object", "javaType": "org.openmetadata.schema.security.ssl.ValidateSSLClientConfig", "properties": { "certificatePath": { + "title": "Certificate Path", "description": "CA certificate path. E.g., /path/to/public.cert. Will be used if Verify SSL is set to `validate`.", "type": "string" } diff --git a/openmetadata-spec/src/main/resources/json/schema/security/ssl/verifySSLConfig.json b/openmetadata-spec/src/main/resources/json/schema/security/ssl/verifySSLConfig.json index 7869e47c02c..1180a854494 100644 --- a/openmetadata-spec/src/main/resources/json/schema/security/ssl/verifySSLConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/security/ssl/verifySSLConfig.json @@ -1,10 +1,11 @@ { "$id": "https://open-metadata.org/schema/security/ssl/verifySSLConfig.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "VerifySSLConfig", + "title": "Verify SSL Config", "description": "Client configuration to validate SSL certificates.", "definitions": { "verifySSL": { + "title": "Verify SSL", "description": "Client SSL verification. Make sure to configure the SSLConfig if enabled.", "javaType": "org.openmetadata.schema.security.ssl.VerifySSL", "type": "string", @@ -16,6 +17,7 @@ "default": "no-ssl" }, "sslConfig": { + "title": "SSL Config", "description": "Client SSL configuration", "javaType": "org.openmetadata.schema.security.ssl.SSLConfig", "oneOf": [ diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Superset.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Superset.md index 688bb2f759c..32ecef75a79 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Superset.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Superset.md @@ -46,6 +46,23 @@ Username to connect to Superset, e.g., `user@organization.com`. This user should Password of the user account to connect with Superset. +### Verify SSL + +Client SSL verification. Make sure to configure the SSLConfig if enabled. +Possible values: +- `validate`: Validate the certificate using the public certificate (recommended). +- `ignore`: Ignore the certification validation (not recommended for production). +- `no-ssl`: SSL validation is not needed. + +### SSL Config + +Client SSL configuration in case we are connection to a host with SSL enabled. + +### Certificate Path + +CA certificate path in the instance where the ingestion run. E.g., `/path/to/public.cert`. +Will be used if Verify SSL is set to `validate`. + -------- ## Postgres Connection diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ConnectionSchemas/SupersetConnection.json b/openmetadata-ui/src/main/resources/ui/src/utils/ConnectionSchemas/SupersetConnection.json index 87bc8a9134e..38874de5783 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ConnectionSchemas/SupersetConnection.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ConnectionSchemas/SupersetConnection.json @@ -66,6 +66,36 @@ "description": "Password for Superset.", "type": "string", "format": "password" + }, + "verifySSL": { + "title": "Verify SSL", + "default": "no-ssl", + "description": "Client SSL verification. Make sure to configure the SSLConfig if enabled.", + "javaType": "org.openmetadata.schema.security.ssl.VerifySSL", + "type": "string", + "enum": ["no-ssl", "ignore", "validate"] + }, + "sslConfig": { + "description": "Client SSL configuration", + "javaType": "org.openmetadata.schema.security.ssl.SSLConfig", + "oneOf": [ + { + "$id": "https://open-metadata.org/schema/security/ssl/validateSSLClientConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Validate SSL Client Config", + "description": "OpenMetadata Client configured to validate SSL certificates.", + "type": "object", + "javaType": "org.openmetadata.schema.security.ssl.ValidateSSLClientConfig", + "properties": { + "certificatePath": { + "title": "Certificate Path", + "description": "CA certificate path. E.g., /path/to/public.cert. Will be used if Verify SSL is set to `validate`.", + "type": "string" + } + }, + "additionalProperties": false + } + ] } }, "additionalProperties": false,