skip folders in s3

2025-11-03 19:30:13 +00:00 · 2023-02-10 16:08:55 -08:00 · 2023-02-10 16:08:55 -08:00 · 8948cff28d
commit 8948cff28d
parent adaa2d78a6
1 changed files with 2 additions and 0 deletions
--- a/loader_hub/s3/base.py
+++ b/loader_hub/s3/base.py
@ -75,6 +75,8 @@ class S3Reader(BaseReader):
            else:
                bucket = s3.Bucket(self.bucket)
                for obj in bucket.objects.filter(Prefix=self.prefix):
+                    if obj.key.endswith("/"):  # skip folders
+                        continue
                    suffix = Path(obj.key).suffix
                    filepath = (
                        f"{temp_dir}/{next(tempfile._get_candidate_names())}{suffix}"