yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-28 17:23:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix for #2597 - added check with column type mapping (#2598)

Browse Source 
* Fix for #2597 - added check with column type mapping

* Optimized Column Type Parsing

* Optimized Column Type Parsing

* SQL Source Bigquery Policy Tags check added
This commit is contained in:
Ayush Shah 2022-02-16 11:05:12 +05:30 committed by GitHub
parent f1c62a70b7
commit 4aee82bfd8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/src/metadata/ingestion/source/sql_source.py
View File

@ -576,7 +576,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]):
col_dict = Column(**parsed_string)
try:
if (
self.config.enable_policy_tags
hasattr(self.config, "enable_policy_tags")
and "policy_tags" in column
and column["policy_tags"]
):

28
ingestion/src/metadata/utils/column_type_parser.py
View File

@ -1,5 +1,5 @@
import re
from typing import Any, Dict, List, Optional, Type, Union
from typing import Any, Dict, List, Type, Union
from sqlalchemy.sql import sqltypes as types
from sqlalchemy.types import TypeEngine
@ -156,21 +156,17 @@ class ColumnTypeParser:
@staticmethod
def get_column_type(column_type: Any) -> str:
type_class: Optional[str] = None
if isinstance(column_type, types.NullType):
return "NULL"
for sql_type in ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.keys():
if str(column_type) == sql_type:
type_class = ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE[sql_type]
break
if type_class is None or type_class == "NULL":
for col_type in ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.keys():
if str(column_type).split("(")[0].split("<")[0].upper() in col_type:
type_class = ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(col_type)
break
else:
type_class = None
return type_class
if not ColumnTypeParser._COLUMN_TYPE_MAPPING.get(type(column_type)):
if not ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(str(column_type)):
if not ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(
str(column_type).split("(")[0].split("<")[0].upper()
):
return ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get("VARCHAR")
return ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(
str(column_type).split("(")[0].split("<")[0].upper()
)
return ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(str(column_type))
return ColumnTypeParser._COLUMN_TYPE_MAPPING.get(type(column_type))
@staticmethod
def _parse_datatype_string(