yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-14 09:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Dbt Fixes and removed urllib from DBT and ometa (#5928)

Browse Source 
* Adding default values

* removed urllib and dbt fixes

* removed urllib and dbt fixes

Co-authored-by: Onkar Ravgan <onkarravgan@Onkars-MacBook-Pro.local>
This commit is contained in:
Onkar Ravgan 2022-07-07 20:32:32 +05:30 committed by GitHub
parent 1651a6fec0
commit ba725b81e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingestion/src/metadata/ingestion/ometa/ometa_api.py
View File

@ -15,7 +15,6 @@ models from the JSON schemas and provides a typed approach to
working with OpenMetadata entities. working with OpenMetadata entities.
""" """
import urllib
from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin
@ -551,9 +550,8 @@ class OpenMetadata(
url_limit = f"?limit={limit}" url_limit = f"?limit={limit}"
url_after = f"&after={after}" if after else "" url_after = f"&after={after}" if after else ""
url_fields = f"&fields={','.join(fields)}" if fields else "" url_fields = f"&fields={','.join(fields)}" if fields else ""
url_params = f"&{urllib.parse.urlencode(params)}" if params else ""
resp = self.client.get( resp = self.client.get(
f"{suffix}{url_limit}{url_after}{url_fields}{url_params}" path=f"{suffix}{url_limit}{url_after}{url_fields}", data=params
) )
if self._use_raw_data: if self._use_raw_data:

12
ingestion/src/metadata/ingestion/source/database/dbt_source.py
View File

@ -73,8 +73,8 @@ class DBTMixin:
model_name = ( model_name = (
mnode["alias"] if "alias" in mnode.keys() else mnode["name"] mnode["alias"] if "alias" in mnode.keys() else mnode["name"]
) )
database = mnode["database"] database = mnode["database"] if mnode["database"] else "default"
schema = mnode["schema"] schema = mnode["schema"] if mnode["schema"] else "default"
raw_sql = mnode.get("raw_sql", "") raw_sql = mnode.get("raw_sql", "")
model = DataModel( model = DataModel(
modelType=ModelType.DBT, modelType=ModelType.DBT,
@ -108,8 +108,12 @@ class DBTMixin:
self.metadata, self.metadata,
entity_type=Table, entity_type=Table,
service_name=self.config.serviceName, service_name=self.config.serviceName,
database_name=parent_node["database"], database_name=parent_node["database"]
schema_name=parent_node["schema"], if parent_node["database"]
else "default",
schema_name=parent_node["schema"]
if parent_node["schema"]
else "default",
table_name=parent_node["name"], table_name=parent_node["name"],
) )
if parent_fqn: if parent_fqn:

11
ingestion/src/metadata/utils/dbt_config.py
View File

@ -13,10 +13,11 @@ Hosts the singledispatch to get DBT files
""" """
import json import json
import traceback import traceback
import urllib.request
from functools import singledispatch from functools import singledispatch
from typing import Optional, Tuple from typing import Optional, Tuple
import requests
from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import ( from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
DbtCloudConfig, DbtCloudConfig,
DbtGCSConfig, DbtGCSConfig,
@ -60,11 +61,9 @@ def _(config: DbtLocalConfig):
@get_dbt_details.register @get_dbt_details.register
def _(config: DbtHttpConfig): def _(config: DbtHttpConfig):
try: try:
catalog_file = urllib.request.urlopen(config.dbtCatalogHttpPath) dbt_catalog = requests.get(config.dbtCatalogHttpPath)
manifest_file = urllib.request.urlopen(config.dbtManifestHttpPath) dbt_manifest = requests.get(config.dbtManifestHttpPath)
dbt_catalog = catalog_file.read().decode() return json.loads(dbt_catalog.text), json.loads(dbt_manifest.text)
dbt_manifest = manifest_file.read().decode()
return json.loads(dbt_catalog), json.loads(dbt_manifest)
except Exception as exc: except Exception as exc:
logger.error(traceback.format_exc()) logger.error(traceback.format_exc())
logger.error(f"Error fetching dbt files from file server {repr(exc)}") logger.error(f"Error fetching dbt files from file server {repr(exc)}")