yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-09-26 08:34:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix gradient comment

Browse Source
This commit is contained in:
rasbt 2024-06-09 20:23:18 -05:00
parent c3c7e64a63
commit b9ed5811c3
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ch06/02_bonus_additional-experiments/additional-experiments.py
View File

@ -261,7 +261,7 @@ def train_classifier_simple(model, train_loader, val_loader, optimizer, device,
# Use gradient accumulation if accumulation_steps > 1 # Use gradient accumulation if accumulation_steps > 1
if batch_idx % accumulation_steps == 0: if batch_idx % accumulation_steps == 0:
optimizer.step() # Update model weights using loss gradients optimizer.step() # Update model weights using loss gradients
optimizer.zero_grad() # Reset loss gradients from previous epoch optimizer.zero_grad() # Reset loss gradients from previous batch iteration
examples_seen += input_batch.shape[0] # New: track examples instead of tokens examples_seen += input_batch.shape[0] # New: track examples instead of tokens
global_step += 1 global_step += 1