mirror of
https://github.com/rasbt/LLMs-from-scratch.git
synced 2025-12-01 09:31:14 +00:00
- added no chat template comparison in test_chat_wrap_and_equivalence
- removed duplicate comparison
This commit is contained in:
parent
df504397a8
commit
3a5ee8cfa1
@ -457,15 +457,13 @@ def test_chat_wrap_and_equivalence(add_gen, add_think):
|
||||
add_thinking=add_think,
|
||||
)
|
||||
|
||||
# Our encode vs HF template
|
||||
ours = qt.encode(prompt)
|
||||
ref = hf_tok.apply_chat_template(
|
||||
messages,
|
||||
tokenize=True,
|
||||
add_generation_prompt=add_gen,
|
||||
enable_thinking=add_think,
|
||||
)
|
||||
ours = qt.encode(prompt)
|
||||
# Base models: compare raw encoding (no chat template)
|
||||
if "Base" in repo_id:
|
||||
ours = qt.encode(prompt) # should use no chat template
|
||||
ref = hf_tok.encode(prompt) # raw encoding without chat template
|
||||
else:
|
||||
# Instruct models: compare with chat template
|
||||
ours = qt.encode(prompt) # will use chat template
|
||||
ref = hf_tok.apply_chat_template(
|
||||
messages,
|
||||
tokenize=True,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user