yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-29 17:49:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix for #2597 - added check with column type mapping (#2598)

Browse Source 
* Fix for #2597 - added check with column type mapping

* Optimized Column Type Parsing

* Optimized Column Type Parsing

* SQL Source Bigquery Policy Tags check added
This commit is contained in:
Ayush Shah 2022-02-16 11:05:12 +05:30 committed by GitHub
parent f1c62a70b7
commit 4aee82bfd8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/src/metadata/ingestion/source/sql_source.py
View File

@ -576,7 +576,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]):
col_dict = Column(**parsed_string) col_dict = Column(**parsed_string)
try: try:
if ( if (
self.config.enable_policy_tags hasattr(self.config, "enable_policy_tags")
and "policy_tags" in column and "policy_tags" in column
and column["policy_tags"] and column["policy_tags"]
): ):

28
ingestion/src/metadata/utils/column_type_parser.py
View File

@ -1,5 +1,5 @@
import re import re
from typing import Any, Dict, List, Optional, Type, Union from typing import Any, Dict, List, Type, Union
from sqlalchemy.sql import sqltypes as types from sqlalchemy.sql import sqltypes as types
from sqlalchemy.types import TypeEngine from sqlalchemy.types import TypeEngine
@ -156,21 +156,17 @@ class ColumnTypeParser:
@staticmethod @staticmethod
def get_column_type(column_type: Any) -> str: def get_column_type(column_type: Any) -> str:
type_class: Optional[str] = None if not ColumnTypeParser._COLUMN_TYPE_MAPPING.get(type(column_type)):
if isinstance(column_type, types.NullType): if not ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(str(column_type)):
return "NULL" if not ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(
for sql_type in ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.keys(): str(column_type).split("(")[0].split("<")[0].upper()
if str(column_type) == sql_type: ):
type_class = ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE[sql_type] return ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get("VARCHAR")
break return ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(
if type_class is None or type_class == "NULL": str(column_type).split("(")[0].split("<")[0].upper()
for col_type in ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.keys(): )
if str(column_type).split("(")[0].split("<")[0].upper() in col_type: return ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(str(column_type))
type_class = ColumnTypeParser._SOURCE_TYPE_TO_OM_TYPE.get(col_type) return ColumnTypeParser._COLUMN_TYPE_MAPPING.get(type(column_type))
break
else:
type_class = None
return type_class
@staticmethod @staticmethod
def _parse_datatype_string( def _parse_datatype_string(