diff --git a/ingestion/src/metadata/cli/docker.py b/ingestion/src/metadata/cli/docker.py index 89c4dd377f2..10ca2c84ef7 100644 --- a/ingestion/src/metadata/cli/docker.py +++ b/ingestion/src/metadata/cli/docker.py @@ -219,7 +219,7 @@ def run_docker( except MemoryError: logger.debug(traceback.format_exc()) click.secho( - f"Please Allocate More memory to Docker.\nRecommended: 6GB\nCurrent: " + f"Please Allocate More memory to Docker.\nRecommended: 6GB+\nCurrent: " f"{round(float(dict(docker_info).get('mem_total')) / calc_gb)}", fg="red", ) diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery.py b/ingestion/src/metadata/ingestion/source/database/bigquery.py index b8e3e7f7af1..b0147f7ba9c 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery.py @@ -246,3 +246,4 @@ class BigquerySource(CommonDbSourceService): super().close() if self.temp_credentials: os.unlink(self.temp_credentials) + os.environ.pop("GOOGLE_CLOUD_PROJECT", "") diff --git a/ingestion/src/metadata/utils/credentials.py b/ingestion/src/metadata/utils/credentials.py index df1a9d67b5c..8b81ecb1438 100644 --- a/ingestion/src/metadata/utils/credentials.py +++ b/ingestion/src/metadata/utils/credentials.py @@ -112,6 +112,14 @@ def set_google_credentials(gcs_credentials: GCSCredentials) -> None: return if isinstance(gcs_credentials.gcsConfig, GCSValues): + if ( + gcs_credentials.gcsConfig.projectId + and not gcs_credentials.gcsConfig.privateKey + ): + logger.info( + "Overriding default projectid, using the current environment permissions authenticated via gcloud SDK." + ) + return credentials_dict = build_google_credentials_dict(gcs_credentials.gcsConfig) tmp_credentials_file = create_credential_tmp_file(credentials=credentials_dict) diff --git a/ingestion/src/metadata/utils/source_connections.py b/ingestion/src/metadata/utils/source_connections.py index d9d690b91cf..08967201b52 100644 --- a/ingestion/src/metadata/utils/source_connections.py +++ b/ingestion/src/metadata/utils/source_connections.py @@ -12,6 +12,7 @@ """ Hosts the singledispatch to build source URLs """ +import os from functools import singledispatch from urllib.parse import quote_plus @@ -359,6 +360,8 @@ def _(connection: BigQueryConnection): if not project_id and isinstance(connection.credentials.gcsConfig, GCSValues): project_id = connection.credentials.gcsConfig.projectId if project_id: + # Setting environment variable based on project id given by user / set in ADC + os.environ["GOOGLE_CLOUD_PROJECT"] = project_id return f"{connection.scheme.value}://{project_id}" return f"{connection.scheme.value}://"