yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-12-03 10:30:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

main push to sync github ruleset

Browse Source
This commit is contained in:
rasbt 2025-09-14 11:59:52 -05:00
parent 8f3e5b024d
commit 1412b139f2
No known key found for this signature in database
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ch02/05_bpe-from-scratch/tests.py
View File

@ -80,7 +80,7 @@ def test_tokenizer_training(imported_module, verdict_file):
BPETokenizerSimple = getattr(imported_module, "BPETokenizerSimple", None)
tokenizer = BPETokenizerSimple()
with open(verdict_file, "r", encoding="utf-8") as f: # added ../01_main-chapter-code/
with open(verdict_file, "r", encoding="utf-8") as f: # added ../01_main-chapter-code/
text = f.read()
tokenizer.train(text, vocab_size=1000, allowed_special={"<|endoftext|>"})