yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-11-04 03:40:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add note about context_length (#549)

Browse Source 
* Add note about context_length

* update note

---------

Co-authored-by: rasbt <mail@sebastianraschka.com>
This commit is contained in:
Greg Gandenberger 2025-02-27 08:36:41 -06:00 committed by GitHub
parent b6f9c8956f
commit 80c95987c1
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ch03/01_main-chapter-code/ch03.ipynb
View File

@ -1628,6 +1628,10 @@
"\n", "\n",
" def forward(self, x):\n", " def forward(self, x):\n",
" b, num_tokens, d_in = x.shape # New batch dimension b\n", " b, num_tokens, d_in = x.shape # New batch dimension b\n",
" # For inputs where `num_tokens` exceeds `context_length`, this will result in errors\n",
" # in the mask creation further below.\n",
" # In practice, this is not a problem since the LLM (chapters 4-7) ensures that inputs \n",
" # do not exceed `context_length` before reaching this forward method. \n",
" keys = self.W_key(x)\n", " keys = self.W_key(x)\n",
" queries = self.W_query(x)\n", " queries = self.W_query(x)\n",
" values = self.W_value(x)\n", " values = self.W_value(x)\n",
@ -1837,6 +1841,10 @@
"\n", "\n",
" def forward(self, x):\n", " def forward(self, x):\n",
" b, num_tokens, d_in = x.shape\n", " b, num_tokens, d_in = x.shape\n",
" # As in `CausalAttention`, for inputs where `num_tokens` exceeds `context_length`, \n",
" # this will result in errors in the mask creation further below. \n",
" # In practice, this is not a problem since the LLM (chapters 4-7) ensures that inputs \n",
" # do not exceed `context_length` before reaching this forwar\n",
"\n", "\n",
" keys = self.W_key(x) # Shape: (b, num_tokens, d_out)\n", " keys = self.W_key(x) # Shape: (b, num_tokens, d_out)\n",
" queries = self.W_query(x)\n", " queries = self.W_query(x)\n",
@ -2029,7 +2037,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.11.4" "version": "3.10.16"
} }
}, },
"nbformat": 4, "nbformat": 4,