yujunjun/llama-hub
1
0
Fork 0
mirror of https://github.com/run-llama/llama-hub.git synced 2025-11-09 14:24:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

skip folders in s3

Browse Source
This commit is contained in:
EmptyCrown 2023-02-10 16:08:55 -08:00
parent adaa2d78a6
commit 8948cff28d
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
loader_hub/s3/base.py
View File

@ -75,6 +75,8 @@ class S3Reader(BaseReader):
else: else:
bucket = s3.Bucket(self.bucket) bucket = s3.Bucket(self.bucket)
for obj in bucket.objects.filter(Prefix=self.prefix): for obj in bucket.objects.filter(Prefix=self.prefix):
if obj.key.endswith("/"): # skip folders
continue
suffix = Path(obj.key).suffix suffix = Path(obj.key).suffix
filepath = ( filepath = (
f"{temp_dir}/{next(tempfile._get_candidate_names())}{suffix}" f"{temp_dir}/{next(tempfile._get_candidate_names())}{suffix}"