From aa084656e0283309f0c012a7dfda2a1237884dbb Mon Sep 17 00:00:00 2001
From: rasbt <mail@sebastianraschka.com>
Date: Thu, 23 May 2024 17:19:01 -0500
Subject: [PATCH] update how to retrieve learning rate

---
 appendix-D/01_main-chapter-code/appendix-D.ipynb | 8 ++++----
 ch05/05_bonus_hparam_tuning/hparam_search.py     | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/appendix-D/01_main-chapter-code/appendix-D.ipynb b/appendix-D/01_main-chapter-code/appendix-D.ipynb
index 993bfb8..15cf906 100644
--- a/appendix-D/01_main-chapter-code/appendix-D.ipynb
+++ b/appendix-D/01_main-chapter-code/appendix-D.ipynb
@@ -231,7 +231,7 @@
     "        # Apply the calculated learning rate to the optimizer\n",
     "        for param_group in optimizer.param_groups:\n",
     "            param_group[\"lr\"] = lr\n",
-    "        track_lrs.append(optimizer.param_groups[0][\"lr\"])\n",
+    "        track_lrs.append(optimizer.defaults[\"lr\"])\n",
     "    \n",
     "        # Calculate loss and update weights\n",
     "        # ..."
@@ -318,7 +318,7 @@
     "        # Apply the calculated learning rate to the optimizer\n",
     "        for param_group in optimizer.param_groups:\n",
     "            param_group[\"lr\"] = lr\n",
-    "        track_lrs.append(optimizer.param_groups[0][\"lr\"])\n",
+    "        track_lrs.append(optimizer.defaults[\"lr\"])\n",
     "    \n",
     "        # Calculate loss and update weights"
    ]
@@ -529,7 +529,7 @@
     "    tokens_seen, global_step = 0, -1\n",
     "\n",
     "    # Retrieve the maximum learning rate from the optimizer\n",
-    "    peak_lr = optimizer.param_groups[0][\"lr\"]\n",
+    "    peak_lr = optimizer.defaults[\"lr\"]\n",
     "\n",
     "    # Calculate the total number of iterations in the training process\n",
     "    total_training_steps = len(train_loader) * n_epochs\n",
@@ -780,7 +780,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,
diff --git a/ch05/05_bonus_hparam_tuning/hparam_search.py b/ch05/05_bonus_hparam_tuning/hparam_search.py
index 94cfb0d..fd558a9 100644
--- a/ch05/05_bonus_hparam_tuning/hparam_search.py
+++ b/ch05/05_bonus_hparam_tuning/hparam_search.py
@@ -65,13 +65,13 @@ def train_model(model, train_loader, val_loader, optimizer, device,
                 initial_lr=3e-05, min_lr=1e-6):
     global_step = 0
 
-    max_lr = optimizer.param_groups[0]["lr"]
+    max_lr = optimizer.defaults["lr"]
 
     # Calculate total number of iterations
     total_training_iters = len(train_loader) * n_epochs
 
     # Calculate the learning rate increment at each step during warmup
-    lr_increment = (optimizer.param_groups[0]["lr"] - initial_lr) / warmup_iters
+    lr_increment = (optimizer.defaults["lr"] - initial_lr) / warmup_iters
 
     for epoch in range(n_epochs):
         model.train()