yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-12 08:52:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Datalake s3 get list of files (#9294)

Browse Source 
* Fix: changed func to get list of files

* Fix: minor change

* Fix: minor change

* Fix: adding exception
This commit is contained in:
NiharDoshi99 2022-12-15 14:33:25 +05:30 committed by GitHub
parent 8fe3b09d21
commit c2221fa596
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingestion/src/metadata/ingestion/source/database/datalake.py
View File

@ -233,10 +233,10 @@ class DatalakeSource(DatabaseServiceSource): # pylint: disable=too-many-public-
try: try:
paginator = self.client.get_paginator("list_objects_v2") paginator = self.client.get_paginator("list_objects_v2")
for page in paginator.paginate(**kwargs): for page in paginator.paginate(**kwargs):
yield from page["Contents"] yield from page.get("Contents", [])
except Exception as exc: except Exception as exc:
logger.debug(traceback.format_exc()) logger.debug(traceback.format_exc())
logger.warning(f"Unexpected exception to yield s3 object [{page}]: {exc}") logger.warning(f"Unexpected exception to yield s3 object: {exc}")
def get_tables_name_and_type( # pylint: disable=too-many-branches def get_tables_name_and_type( # pylint: disable=too-many-branches
self, self,