yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-28 18:38:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(ingest/delta-lake): cleanup import error handling (#8230)

Browse Source 
Co-authored-by: Tamas Nemeth <treff7es@gmail.com>
This commit is contained in:
Harshal Sheth 2023-07-06 13:53:03 -07:00 committed by GitHub
parent 3a21c27f06
commit 4daee108c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 12 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py
View File

@ -6,15 +6,12 @@ from deltalake import DeltaTable
try:
from deltalake.exceptions import TableNotFoundError
DELTALAKE_VERSION_GTE_0_10_0 = True
_MUST_CHECK_TABLE_NOT_FOUND_MESSAGE = False
except ImportError:
# For deltalake < 0.10.0
from deltalake import PyDeltaTableError # type: ignore[attr-defined]
DELTALAKE_VERSION_GTE_0_10_0 = False
# For deltalake < 0.10.0.
from deltalake import PyDeltaTableError as TableNotFoundError # type: ignore
_MUST_CHECK_TABLE_NOT_FOUND_MESSAGE = True
from datahub.ingestion.source.delta_lake.config import DeltaLakeSourceConfig
@ -33,15 +30,15 @@ def read_delta_table(
storage_options=opts,
without_files=not delta_lake_config.require_files,
)
except Exception as e:
if (DELTALAKE_VERSION_GTE_0_10_0 and isinstance(e, TableNotFoundError)) or (
not DELTALAKE_VERSION_GTE_0_10_0
and isinstance(e, PyDeltaTableError)
and "Not a Delta table" in str(e)
):
pass
else:
except TableNotFoundError as e:
# For deltalake < 0.10.0, we need to check the error message to make sure
# that this is a table not found error. Newer versions have a dedicated
# exception class for this.
if _MUST_CHECK_TABLE_NOT_FOUND_MESSAGE and "Not a Delta table" not in str(e):
raise e
else:
# Otherwise, the table was genuinely not found and we return None.
pass
return None