From 917ccd7147a3b845b356e3e16bbdc8afabcc7e24 Mon Sep 17 00:00:00 2001
From: Ayush Shah <ayush@getcollate.io>
Date: Tue, 25 Jan 2022 14:30:09 +0530
Subject: [PATCH] Bigquery and Helpers fix - Ingestion (#2408)

* Bigquery Ingestion fix - error handling and keyerror

* Update helpers.py and column_type_parser

* Update helpers.py
---
 .../src/metadata/ingestion/source/bigquery.py | 31 ++++++++++++-------
 .../src/metadata/utils/column_type_parser.py  |  2 +-
 ingestion/src/metadata/utils/helpers.py       |  4 ++-
 3 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/ingestion/src/metadata/ingestion/source/bigquery.py b/ingestion/src/metadata/ingestion/source/bigquery.py
index b6014187d96..f4e762ded1b 100644
--- a/ingestion/src/metadata/ingestion/source/bigquery.py
+++ b/ingestion/src/metadata/ingestion/source/bigquery.py
@@ -10,10 +10,9 @@
 #  limitations under the License.
 
 import json
-import logging
 import os
 import tempfile
-from typing import Any, Optional, Tuple
+from typing import Optional, Tuple
 
 from sqlalchemy_bigquery import _types
 from sqlalchemy_bigquery._struct import STRUCT
@@ -77,20 +76,28 @@ class BigquerySource(SQLSource):
     def create(cls, config_dict, metadata_config_dict, ctx):
         config: SQLConnectionConfig = BigQueryConfig.parse_obj(config_dict)
         metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
-        if config.options.get("credentials_path"):
-            os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = config.options[
-                "credentials_path"
-            ]
-        elif config.options.get("credentials", None):
-            cred_path = create_credential_temp_file(config.options.get("credentials"))
-            os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cred_path
-            config.options["credentials_path"] = cred_path
-            del config.options["credentials"]
+        if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
+            if config.options.get("credentials_path"):
+                os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = config.options[
+                    "credentials_path"
+                ]
+            elif config.options.get("credentials", None):
+                cred_path = create_credential_temp_file(
+                    config.options.get("credentials")
+                )
+                os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cred_path
+                config.options["credentials_path"] = cred_path
+                del config.options["credentials"]
+            else:
+                raise Exception(
+                    "Please refer to the BigQuery connector documentation, especially the credentials part "
+                    "https://docs.open-metadata.org/connectors/bigquery"
+                )
         return cls(config, metadata_config, ctx)
 
     def close(self):
         super().close()
-        if self.config.options["credentials_path"]:
+        if self.config.options.get("credentials_path"):
             os.unlink(self.config.options["credentials_path"])
 
     def standardize_schema_table_names(
diff --git a/ingestion/src/metadata/utils/column_type_parser.py b/ingestion/src/metadata/utils/column_type_parser.py
index bc0288d0ce9..58377f49106 100644
--- a/ingestion/src/metadata/utils/column_type_parser.py
+++ b/ingestion/src/metadata/utils/column_type_parser.py
@@ -96,7 +96,7 @@ class ColumnTypeParser:
         "MONEY": "NUMBER",
         "NCHAR": "CHAR",
         "NTEXT": "TEXT",
-        "NULL": "NULL",
+        "NULL": "VARCHAR",
         "NUMBER": "NUMBER",
         "NUMERIC": "NUMERIC",
         "NVARCHAR": "VARCHAR",
diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py
index 050923daa0c..f31f38a8ca4 100644
--- a/ingestion/src/metadata/utils/helpers.py
+++ b/ingestion/src/metadata/utils/helpers.py
@@ -64,7 +64,9 @@ def get_database_service_or_create(
         return service
     else:
         password = (
-            config.password.get_secret_value() if hasattr(config, "password") else None
+            config.password.get_secret_value()
+            if hasattr(config, "password") and config.password
+            else None
         )
         service = {
             "databaseConnection": {