yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-08-28 18:40:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix qk_norm comment (#769)

Browse Source
This commit is contained in:
Sebastian Raschka 2025-08-15 08:38:48 -05:00 committed by GitHub
parent b14325e56d
commit e9c1c1da38
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ch05/11_qwen3/standalone-qwen3.ipynb
View File

@ -436,7 +436,7 @@
" \"n_layers\": 28, # Number of layers\n", " \"n_layers\": 28, # Number of layers\n",
" \"hidden_dim\": 3072, # Size of the intermediate dimension in FeedForward\n", " \"hidden_dim\": 3072, # Size of the intermediate dimension in FeedForward\n",
" \"head_dim\": 128, # Size of the heads in GQA\n", " \"head_dim\": 128, # Size of the heads in GQA\n",
" \"qk_norm\": True, # Whether to normalize queries and values in GQA\n", " \"qk_norm\": True, # Whether to normalize queries and keys in GQA\n",
" \"n_kv_groups\": 8, # Key-Value groups for grouped-query attention\n", " \"n_kv_groups\": 8, # Key-Value groups for grouped-query attention\n",
" \"rope_base\": 1_000_000.0, # The base in RoPE's \"theta\"\n", " \"rope_base\": 1_000_000.0, # The base in RoPE's \"theta\"\n",
" \"dtype\": torch.bfloat16, # Lower-precision dtype to reduce memory usage\n", " \"dtype\": torch.bfloat16, # Lower-precision dtype to reduce memory usage\n",

2
pkg/llms_from_scratch/qwen3.py
View File

@ -22,7 +22,7 @@ QWEN_CONFIG_06_B = {
"n_layers": 28, # Number of layers "n_layers": 28, # Number of layers
"hidden_dim": 3072, # Size of the intermediate dimension in FeedForward "hidden_dim": 3072, # Size of the intermediate dimension in FeedForward
"head_dim": 128, # Size of the heads in GQA "head_dim": 128, # Size of the heads in GQA
"qk_norm": True, # Whether to normalize queries and values in GQA "qk_norm": True, # Whether to normalize queries and keys in GQA
"n_kv_groups": 8, # Key-Value groups for grouped-query attention "n_kv_groups": 8, # Key-Value groups for grouped-query attention
"rope_base": 1_000_000.0, # The base in RoPE's "theta" "rope_base": 1_000_000.0, # The base in RoPE's "theta"
"dtype": torch.bfloat16, # Lower-precision dtype to reduce memory usage "dtype": torch.bfloat16, # Lower-precision dtype to reduce memory usage