From a90eb94951b7c4615b7c25bc82e9b921c4dd80cd Mon Sep 17 00:00:00 2001 From: Jake Poznanski Date: Mon, 7 Oct 2024 15:25:48 -0700 Subject: [PATCH] Fix dataloader bug --- pdelfin/train/dataloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdelfin/train/dataloader.py b/pdelfin/train/dataloader.py index 9a22c39..d38c02d 100644 --- a/pdelfin/train/dataloader.py +++ b/pdelfin/train/dataloader.py @@ -51,7 +51,7 @@ def load_jsonl_into_ds(s3_glob_path: str, first_n_files: int = None) -> Dataset: """ Loads JSONL files from the specified S3 path into a Hugging Face Dataset. """ - all_json_files = s3_glob_path(s3_glob_path) + all_json_files = list_dataset_files(s3_glob_path) if first_n_files: all_json_files = all_json_files[:first_n_files]