From ba725b81e75603825dc9fbcda0e930eb6b15b07d Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Thu, 7 Jul 2022 20:32:32 +0530 Subject: [PATCH] Dbt Fixes and removed urllib from DBT and ometa (#5928) * Adding default values * removed urllib and dbt fixes * removed urllib and dbt fixes Co-authored-by: Onkar Ravgan --- ingestion/src/metadata/ingestion/ometa/ometa_api.py | 4 +--- .../metadata/ingestion/source/database/dbt_source.py | 12 ++++++++---- ingestion/src/metadata/utils/dbt_config.py | 11 +++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 88de9ecbb60..f6d9104fd79 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -15,7 +15,6 @@ models from the JSON schemas and provides a typed approach to working with OpenMetadata entities. """ -import urllib from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin @@ -551,9 +550,8 @@ class OpenMetadata( url_limit = f"?limit={limit}" url_after = f"&after={after}" if after else "" url_fields = f"&fields={','.join(fields)}" if fields else "" - url_params = f"&{urllib.parse.urlencode(params)}" if params else "" resp = self.client.get( - f"{suffix}{url_limit}{url_after}{url_fields}{url_params}" + path=f"{suffix}{url_limit}{url_after}{url_fields}", data=params ) if self._use_raw_data: diff --git a/ingestion/src/metadata/ingestion/source/database/dbt_source.py b/ingestion/src/metadata/ingestion/source/database/dbt_source.py index f3ae62c7214..5fa1c9323b4 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt_source.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt_source.py @@ -73,8 +73,8 @@ class DBTMixin: model_name = ( mnode["alias"] if "alias" in mnode.keys() else mnode["name"] ) - database = mnode["database"] - schema = mnode["schema"] + database = mnode["database"] if mnode["database"] else "default" + schema = mnode["schema"] if mnode["schema"] else "default" raw_sql = mnode.get("raw_sql", "") model = DataModel( modelType=ModelType.DBT, @@ -108,8 +108,12 @@ class DBTMixin: self.metadata, entity_type=Table, service_name=self.config.serviceName, - database_name=parent_node["database"], - schema_name=parent_node["schema"], + database_name=parent_node["database"] + if parent_node["database"] + else "default", + schema_name=parent_node["schema"] + if parent_node["schema"] + else "default", table_name=parent_node["name"], ) if parent_fqn: diff --git a/ingestion/src/metadata/utils/dbt_config.py b/ingestion/src/metadata/utils/dbt_config.py index aef41400fae..cfe076b1273 100644 --- a/ingestion/src/metadata/utils/dbt_config.py +++ b/ingestion/src/metadata/utils/dbt_config.py @@ -13,10 +13,11 @@ Hosts the singledispatch to get DBT files """ import json import traceback -import urllib.request from functools import singledispatch from typing import Optional, Tuple +import requests + from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import ( DbtCloudConfig, DbtGCSConfig, @@ -60,11 +61,9 @@ def _(config: DbtLocalConfig): @get_dbt_details.register def _(config: DbtHttpConfig): try: - catalog_file = urllib.request.urlopen(config.dbtCatalogHttpPath) - manifest_file = urllib.request.urlopen(config.dbtManifestHttpPath) - dbt_catalog = catalog_file.read().decode() - dbt_manifest = manifest_file.read().decode() - return json.loads(dbt_catalog), json.loads(dbt_manifest) + dbt_catalog = requests.get(config.dbtCatalogHttpPath) + dbt_manifest = requests.get(config.dbtManifestHttpPath) + return json.loads(dbt_catalog.text), json.loads(dbt_manifest.text) except Exception as exc: logger.error(traceback.format_exc()) logger.error(f"Error fetching dbt files from file server {repr(exc)}")