yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-10-27 08:04:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Filtering based on cpu count

Browse Source
This commit is contained in:
Jake Poznanski 2024-10-07 15:40:29 -07:00
parent a90eb94951
commit ec09408ca9
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pdelfin/train/utils.py
View File

@ -59,7 +59,7 @@ def make_dataset(config: TrainConfig, processor: AutoProcessor) -> tuple[Dataset
for source in config.train_data.sources for source in config.train_data.sources
] ]
) )
.filter(partial(filter_by_max_seq_len, processor=processor)) .filter(partial(filter_by_max_seq_len, processor=processor), num_proc=multiprocessing.cpu_count())
.with_transform(partial(batch_prepare_data_for_qwen2_training, processor=processor)) .with_transform(partial(batch_prepare_data_for_qwen2_training, processor=processor))
) )