yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-26 17:45:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): handle case when view definition handler is not implemented (#2796)

Browse Source
This commit is contained in:
Harshal Sheth 2021-06-29 17:49:57 -07:00 committed by GitHub
parent 2ee9ecfbc3
commit 57ddb41086
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
metadata-ingestion/src/datahub/ingestion/source/sql_common.py
View File

@ -4,6 +4,7 @@ from dataclasses import dataclass, field
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type
from sqlalchemy import create_engine, inspect from sqlalchemy import create_engine, inspect
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.sql import sqltypes as types from sqlalchemy.sql import sqltypes as types
from datahub.configuration.common import AllowDenyPattern, ConfigModel from datahub.configuration.common import AllowDenyPattern, ConfigModel
@ -233,7 +234,7 @@ class SQLAlchemySource(Source):
def loop_tables( def loop_tables(
self, self,
inspector: Any, inspector: Inspector,
schema: str, schema: str,
sql_config: SQLAlchemyConfig, sql_config: SQLAlchemyConfig,
) -> Iterable[SqlWorkUnit]: ) -> Iterable[SqlWorkUnit]:
@ -248,7 +249,9 @@ class SQLAlchemySource(Source):
columns = inspector.get_columns(table, schema) columns = inspector.get_columns(table, schema)
try: try:
table_info: dict = inspector.get_table_comment(table, schema) # SQLALchemy stubs are incomplete and missing this method.
# PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223.
table_info: dict = inspector.get_table_comment(table, schema) # type: ignore
except NotImplementedError: except NotImplementedError:
description: Optional[str] = None description: Optional[str] = None
properties: Dict[str, str] = {} properties: Dict[str, str] = {}
@ -284,7 +287,7 @@ class SQLAlchemySource(Source):
def loop_views( def loop_views(
self, self,
inspector: Any, inspector: Inspector,
schema: str, schema: str,
sql_config: SQLAlchemyConfig, sql_config: SQLAlchemyConfig,
) -> Iterable[SqlWorkUnit]: ) -> Iterable[SqlWorkUnit]:
@ -299,7 +302,9 @@ class SQLAlchemySource(Source):
columns = inspector.get_columns(view, schema) columns = inspector.get_columns(view, schema)
try: try:
view_info: dict = inspector.get_table_comment(view, schema) # SQLALchemy stubs are incomplete and missing this method.
# PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223.
view_info: dict = inspector.get_table_comment(view, schema) # type: ignore
except NotImplementedError: except NotImplementedError:
description: Optional[str] = None description: Optional[str] = None
properties: Dict[str, str] = {} properties: Dict[str, str] = {}
@ -309,8 +314,11 @@ class SQLAlchemySource(Source):
# The "properties" field is a non-standard addition to SQLAlchemy's interface. # The "properties" field is a non-standard addition to SQLAlchemy's interface.
properties = view_info.get("properties", {}) properties = view_info.get("properties", {})
view_definition = inspector.get_view_definition(view) try:
if view_definition is None: view_definition = inspector.get_view_definition(view, schema)
if view_definition is None:
view_definition = ""
except NotImplementedError:
view_definition = "" view_definition = ""
properties["view_definition"] = view_definition properties["view_definition"] = view_definition
properties["is_view"] = "True" properties["is_view"] = "True"