yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-12-01 01:20:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

- added no chat template comparison in test_chat_wrap_and_equivalence

Browse Source 
- removed duplicate comparison
This commit is contained in:
casinca 2025-09-15 19:30:31 +02:00
parent df504397a8
commit 3a5ee8cfa1
1 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
pkg/llms_from_scratch/tests/test_qwen3.py
View File

@ -457,21 +457,19 @@ def test_chat_wrap_and_equivalence(add_gen, add_think):
add_thinking=add_think,
)
# Our encode vs HF template
ours = qt.encode(prompt)
ref = hf_tok.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=add_gen,
enable_thinking=add_think,
)
ours = qt.encode(prompt)
ref = hf_tok.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=add_gen,
enable_thinking=add_think,
)
# Base models: compare raw encoding (no chat template)
if "Base" in repo_id:
ours = qt.encode(prompt) # should use no chat template
ref = hf_tok.encode(prompt) # raw encoding without chat template
else:
# Instruct models: compare with chat template
ours = qt.encode(prompt) # will use chat template
ref = hf_tok.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=add_gen,
enable_thinking=add_think,
)
if add_gen and not add_think:
pass # skip edge case as this is not something we use in practice