yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-10-11 16:22:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix dataloader bug

Browse Source
This commit is contained in:
Jake Poznanski 2024-10-07 15:25:48 -07:00
parent 3d36545fa5
commit a90eb94951
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pdelfin/train/dataloader.py
View File

@ -51,7 +51,7 @@ def load_jsonl_into_ds(s3_glob_path: str, first_n_files: int = None) -> Dataset:
"""
Loads JSONL files from the specified S3 path into a Hugging Face Dataset.
"""
all_json_files = s3_glob_path(s3_glob_path)
all_json_files = list_dataset_files(s3_glob_path)
if first_n_files:
all_json_files = all_json_files[:first_n_files]