Dbt Fixes and removed urllib from DBT and ometa (#5928)

* Adding default values

* removed urllib and dbt fixes

* removed urllib and dbt fixes

Co-authored-by: Onkar Ravgan <onkarravgan@Onkars-MacBook-Pro.local>

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -15,7 +15,6 @@ models from the JSON schemas and provides a typed approach to
 working with OpenMetadata entities.
 """
 
-import urllib
 from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
 
 from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin
@@ -551,9 +550,8 @@ class OpenMetadata(
         url_limit = f"?limit={limit}"
         url_after = f"&after={after}" if after else ""
         url_fields = f"&fields={','.join(fields)}" if fields else ""
-        url_params = f"&{urllib.parse.urlencode(params)}" if params else ""
         resp = self.client.get(
-            f"{suffix}{url_limit}{url_after}{url_fields}{url_params}"
+            path=f"{suffix}{url_limit}{url_after}{url_fields}", data=params
         )
 
         if self._use_raw_data:

ingestion/src/metadata/ingestion/source/database/dbt_source.py

@@ -73,8 +73,8 @@ class DBTMixin:
                     model_name = (
                         mnode["alias"] if "alias" in mnode.keys() else mnode["name"]
                     )
-                    database = mnode["database"]
-                    schema = mnode["schema"]
+                    database = mnode["database"] if mnode["database"] else "default"
+                    schema = mnode["schema"] if mnode["schema"] else "default"
                     raw_sql = mnode.get("raw_sql", "")
                     model = DataModel(
                         modelType=ModelType.DBT,
@@ -108,8 +108,12 @@ class DBTMixin:
                         self.metadata,
                         entity_type=Table,
                         service_name=self.config.serviceName,
-                        database_name=parent_node["database"],
-                        schema_name=parent_node["schema"],
+                        database_name=parent_node["database"]
+                        if parent_node["database"]
+                        else "default",
+                        schema_name=parent_node["schema"]
+                        if parent_node["schema"]
+                        else "default",
                         table_name=parent_node["name"],
                     )
                     if parent_fqn:

ingestion/src/metadata/utils/dbt_config.py

@@ -13,10 +13,11 @@ Hosts the singledispatch to get DBT files
 """
 import json
 import traceback
-import urllib.request
 from functools import singledispatch
 from typing import Optional, Tuple
 
+import requests
+
 from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
     DbtCloudConfig,
     DbtGCSConfig,
@@ -60,11 +61,9 @@ def _(config: DbtLocalConfig):
 @get_dbt_details.register
 def _(config: DbtHttpConfig):
     try:
-        catalog_file = urllib.request.urlopen(config.dbtCatalogHttpPath)
-        manifest_file = urllib.request.urlopen(config.dbtManifestHttpPath)
-        dbt_catalog = catalog_file.read().decode()
-        dbt_manifest = manifest_file.read().decode()
-        return json.loads(dbt_catalog), json.loads(dbt_manifest)
+        dbt_catalog = requests.get(config.dbtCatalogHttpPath)
+        dbt_manifest = requests.get(config.dbtManifestHttpPath)
+        return json.loads(dbt_catalog.text), json.loads(dbt_manifest.text)
     except Exception as exc:
         logger.error(traceback.format_exc())
         logger.error(f"Error fetching dbt files from file server {repr(exc)}")