yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-09-25 08:05:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use max size properly

Browse Source
This commit is contained in:
Sebastian Raschka 2024-04-02 13:29:23 -05:00 committed by GitHub
parent 4a617b8343
commit 2fab89d47e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ch05/03_bonus_pretraining_on_gutenberg/prepare_dataset.py
View File

@ -69,5 +69,5 @@ if __name__ == "__main__":
for name in files if name.endswith((".txt", ".txt.utf8")) and "raw" not in path]
print(f"{len(all_files)} file(s) to process.")
file_counter = combine_files(all_files, args.output_dir)
file_counter = combine_files(all_files, args.output_dir, max_size_mb=args.max_size_mb)
print(f"{file_counter} file(s) saved in {os.path.abspath(args.output_dir)}")