yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-30 12:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion): add output converters for ODBC unsuported datatype in… (#6134)

Browse Source 
Co-authored-by: Pavel Klammert | LOGEX <Pavel.Klammert@logex.com>
This commit is contained in:
Pavel Klammert 2022-11-15 23:11:00 +01:00 committed by GitHub
parent fc26cf3f27
commit 9862eff95a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py
View File

@ -128,9 +128,22 @@ class SQLServerSource(SQLAlchemySource):
for inspector in self.get_inspectors(): for inspector in self.get_inspectors():
db_name: str = self.get_db_name(inspector) db_name: str = self.get_db_name(inspector)
with inspector.engine.connect() as conn: with inspector.engine.connect() as conn:
if self.config.use_odbc:
self._add_output_converters(conn)
self._populate_table_descriptions(conn, db_name) self._populate_table_descriptions(conn, db_name)
self._populate_column_descriptions(conn, db_name) self._populate_column_descriptions(conn, db_name)
@staticmethod
def _add_output_converters(conn: Connection) -> None:
def handle_sql_variant_as_string(value):
return value.decode('utf-16le')
# see https://stackoverflow.com/questions/45677374/pandas-pyodbc-odbc-sql-type-150-is-not-yet-supported
# and https://stackoverflow.com/questions/11671170/adding-output-converter-to-pyodbc-connection-in-sqlalchemy
try:
conn.connection.add_output_converter(-150, handle_sql_variant_as_string)
except AttributeError as e:
logger.debug(f"Failed to mount output converter for MSSQL data type -150 due to {e}")
def _populate_table_descriptions(self, conn: Connection, db_name: str) -> None: def _populate_table_descriptions(self, conn: Connection, db_name: str) -> None:
# see https://stackoverflow.com/questions/5953330/how-do-i-map-the-id-in-sys-extended-properties-to-an-object-name # see https://stackoverflow.com/questions/5953330/how-do-i-map-the-id-in-sys-extended-properties-to-an-object-name
# also see https://www.mssqltips.com/sqlservertip/5384/working-with-sql-server-extended-properties/ # also see https://www.mssqltips.com/sqlservertip/5384/working-with-sql-server-extended-properties/