From bce851372f0cecb12da9a17d74f7700e55dc687f Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Thu, 27 Feb 2025 12:18:50 +0530 Subject: [PATCH] MINOR: Use clean url in clients (#19978) (cherry picked from commit 73876a9122a85197fa093def9fa885ab0174f51b) --- ingestion/src/metadata/clients/domo_client.py | 4 ++-- .../metadata/ingestion/source/dashboard/lightdash/client.py | 3 ++- .../src/metadata/ingestion/source/dashboard/mode/client.py | 3 ++- .../metadata/ingestion/source/dashboard/qlikcloud/client.py | 2 +- .../metadata/ingestion/source/dashboard/superset/client.py | 5 +++-- .../src/metadata/ingestion/source/database/dbt/dbt_config.py | 3 ++- .../src/metadata/ingestion/source/database/sas/client.py | 3 ++- .../src/metadata/ingestion/source/metadata/atlas/client.py | 3 ++- 8 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ingestion/src/metadata/clients/domo_client.py b/ingestion/src/metadata/clients/domo_client.py index ebf8565d361..f6b66e0cb61 100644 --- a/ingestion/src/metadata/clients/domo_client.py +++ b/ingestion/src/metadata/clients/domo_client.py @@ -30,6 +30,7 @@ from metadata.generated.schema.entity.services.connections.pipeline.domoPipeline DomoPipelineConnection, ) from metadata.ingestion.ometa.client import REST, ClientConfig +from metadata.utils.helpers import clean_uri from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -103,8 +104,7 @@ class DomoClient: self.config = config HEADERS.update({"X-DOMO-Developer-Token": self.config.accessToken}) client_config: ClientConfig = ClientConfig( - # AnyUrl string ends with / and the domo API does not respond properly if it has 2 // at the end - base_url=str(self.config.instanceDomain)[:-1], + base_url=clean_uri(self.config.instanceDomain), api_version="api/", auth_header="Authorization", auth_token=lambda: ("no_token", 0), diff --git a/ingestion/src/metadata/ingestion/source/dashboard/lightdash/client.py b/ingestion/src/metadata/ingestion/source/dashboard/lightdash/client.py index 8984d91798d..e0cb571feec 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/lightdash/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/lightdash/client.py @@ -19,6 +19,7 @@ from metadata.ingestion.source.dashboard.lightdash.models import ( LightdashChart, LightdashDashboard, ) +from metadata.utils.helpers import clean_uri from metadata.utils.logger import utils_logger logger = utils_logger() @@ -34,7 +35,7 @@ class LightdashApiClient: def __init__(self, config): self.config = config client_config = ClientConfig( - base_url=str(self.config.hostPort), + base_url=clean_uri(self.config.hostPort), api_version="", access_token=self.config.apiKey.get_secret_value(), auth_header="Authorization", diff --git a/ingestion/src/metadata/ingestion/source/dashboard/mode/client.py b/ingestion/src/metadata/ingestion/source/dashboard/mode/client.py index b0887e4a910..95e89772343 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/mode/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/mode/client.py @@ -18,6 +18,7 @@ from typing import Optional from requests._internal_utils import to_native_string from metadata.ingestion.ometa.client import REST, ClientConfig +from metadata.utils.helpers import clean_uri from metadata.utils.logger import utils_logger logger = utils_logger() @@ -50,7 +51,7 @@ class ModeApiClient: def __init__(self, config): self.config = config client_config = ClientConfig( - base_url=str(config.hostPort), + base_url=clean_uri(config.hostPort), api_version="api", auth_header="Authorization", auth_token_mode="Basic", diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py index 307cc971f2f..181bd3a44ee 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py @@ -59,7 +59,7 @@ class QlikCloudClient: self.socket_connection = None client_config: ClientConfig = ClientConfig( - base_url=str(self.config.hostPort), + base_url=clean_uri(self.config.hostPort), api_version=API_VERSION, auth_header=AUTHORIZATION_HEADER, auth_token=lambda: (self.config.token.get_secret_value(), 0), diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py index a52b80acc6e..64cfbfa44aa 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py @@ -25,6 +25,7 @@ from metadata.ingestion.source.dashboard.superset.models import ( SupersetDashboardCount, SupersetDatasource, ) +from metadata.utils.helpers import clean_uri from metadata.utils.logger import ometa_logger from metadata.utils.ssl_registry import get_verify_ssl_fn @@ -41,7 +42,7 @@ class SupersetAuthenticationProvider(AuthenticationProvider): self.service_connection = self.config get_verify_ssl = get_verify_ssl_fn(config.connection.verifySSL) client_config = ClientConfig( - base_url=str(config.hostPort), + base_url=clean_uri(config.hostPort), api_version="api/v1", auth_token=lambda: ("no_token", 0), auth_header="Authorization", @@ -90,7 +91,7 @@ class SupersetAPIClient: self._auth_provider = SupersetAuthenticationProvider.create(config) get_verify_ssl = get_verify_ssl_fn(config.connection.verifySSL) client_config = ClientConfig( - base_url=str(config.hostPort), + base_url=clean_uri(config.hostPort), api_version="api/v1", auth_token=self._auth_provider.get_access_token, auth_header="Authorization", diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py index 20ef6417a9c..697de67213e 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_config.py @@ -48,6 +48,7 @@ from metadata.ingestion.source.database.dbt.constants import ( from metadata.ingestion.source.database.dbt.models import DbtFiles from metadata.readers.file.config_source_factory import get_reader from metadata.utils.credentials import set_google_credentials +from metadata.utils.helpers import clean_uri from metadata.utils.logger import ometa_logger from metadata.utils.s3_utils import list_s3_objects @@ -163,7 +164,7 @@ def _(config: DbtCloudConfig): # pylint: disable=too-many-locals expiry = 0 auth_token = config.dbtCloudAuthToken.get_secret_value(), expiry client_config = ClientConfig( - base_url=str(config.dbtCloudUrl), + base_url=clean_uri(config.dbtCloudUrl), api_version="api/v2", auth_token=lambda: auth_token, auth_header="Authorization", diff --git a/ingestion/src/metadata/ingestion/source/database/sas/client.py b/ingestion/src/metadata/ingestion/source/database/sas/client.py index 6c69ca10e8b..ec57ef59b36 100644 --- a/ingestion/src/metadata/ingestion/source/database/sas/client.py +++ b/ingestion/src/metadata/ingestion/source/database/sas/client.py @@ -19,6 +19,7 @@ from metadata.generated.schema.entity.services.connections.database.sasConnectio SASConnection, ) from metadata.ingestion.ometa.client import REST, APIError, ClientConfig +from metadata.utils.helpers import clean_uri from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -35,7 +36,7 @@ class SASClient: config.serverHost, config.username, config.password.get_secret_value() ) client_config: ClientConfig = ClientConfig( - base_url=str(config.serverHost), + base_url=clean_uri(config.serverHost), auth_header="Authorization", auth_token=self.get_auth_token, api_version="", diff --git a/ingestion/src/metadata/ingestion/source/metadata/atlas/client.py b/ingestion/src/metadata/ingestion/source/metadata/atlas/client.py index 488ffe28a6d..46e4a6918e3 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/atlas/client.py +++ b/ingestion/src/metadata/ingestion/source/metadata/atlas/client.py @@ -18,6 +18,7 @@ from metadata.generated.schema.entity.services.connections.metadata.atlasConnect AtlasConnection, ) from metadata.ingestion.ometa.client import REST, APIError, ClientConfig +from metadata.utils.helpers import clean_uri class AtlasClient: @@ -31,7 +32,7 @@ class AtlasClient: config.username, config.password.get_secret_value() ) client_config: ClientConfig = ClientConfig( - base_url=str(config.hostPort), + base_url=clean_uri(config.hostPort), auth_header="Authorization", api_version="api", auth_token=self.get_auth_token,