yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-11-15 01:24:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

cosmetics

Browse Source
This commit is contained in:
Sebastian Raschka 2024-05-05 08:15:46 -05:00 committed by GitHub
parent 244593ce01
commit 3328b29521
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ch06/01_main-chapter-code/previous_chapters.py
View File

@ -312,10 +312,10 @@ def load_weights_into_gpt(gpt, params):
def text_to_token_ids(text, tokenizer): def text_to_token_ids(text, tokenizer):
encoded = tokenizer.encode(text, allowed_special={'<|endoftext|>'}) encoded = tokenizer.encode(text, allowed_special={'<|endoftext|>'})
encoded_tensor = torch.tensor(encoded).unsqueeze(0) # add batch dimension encoded_tensor = torch.tensor(encoded).unsqueeze(0) # add batch dimension
return encoded_tensor return encoded_tensor
def token_ids_to_text(token_ids, tokenizer): def token_ids_to_text(token_ids, tokenizer):
flat = token_ids.squeeze(0) # remove batch dimension flat = token_ids.squeeze(0) # remove batch dimension
return tokenizer.decode(flat.tolist()) return tokenizer.decode(flat.tolist())