yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-08-29 11:00:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix code comment: embed_dim -> d_out (#698)

Browse Source
This commit is contained in:
Sebastian Raschka 2025-06-22 16:36:39 -05:00 committed by GitHub
parent a5ea296259
commit 984cca3f64
3 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ch03/02_bonus_efficient-multihead-attention/mha-implementations.ipynb
View File

@ -352,7 +352,7 @@
" def __init__(self, d_in, d_out, num_heads, context_length, dropout=0.0, qkv_bias=False):\n", " def __init__(self, d_in, d_out, num_heads, context_length, dropout=0.0, qkv_bias=False):\n",
" super().__init__()\n", " super().__init__()\n",
"\n", "\n",
" assert d_out % num_heads == 0, \"embed_dim is indivisible by num_heads\"\n", " assert d_out % num_heads == 0, \"d_out is indivisible by num_heads\"\n",
"\n", "\n",
" self.num_heads = num_heads\n", " self.num_heads = num_heads\n",
" self.context_length = context_length\n", " self.context_length = context_length\n",
@ -588,7 +588,7 @@
" def __init__(self, d_in, d_out, num_heads, context_length, dropout=0.0, qkv_bias=False):\n", " def __init__(self, d_in, d_out, num_heads, context_length, dropout=0.0, qkv_bias=False):\n",
" super().__init__()\n", " super().__init__()\n",
"\n", "\n",
" assert d_out % num_heads == 0, \"embed_dim is indivisible by num_heads\"\n", " assert d_out % num_heads == 0, \"d_out is indivisible by num_heads\"\n",
"\n", "\n",
" self.num_heads = num_heads\n", " self.num_heads = num_heads\n",
" self.context_length = context_length\n", " self.context_length = context_length\n",

2
ch05/10_llm-training-speed/01_opt_single_gpu.py
View File

@ -65,7 +65,7 @@ class PyTorchMultiHeadAttention(nn.Module):
def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False): def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False):
super().__init__() super().__init__()
assert d_out % num_heads == 0, "embed_dim is indivisible by num_heads" assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
self.num_heads = num_heads self.num_heads = num_heads
self.head_dim = d_out // num_heads self.head_dim = d_out // num_heads

2
ch05/10_llm-training-speed/02_opt_multi_gpu_ddp.py
View File

@ -108,7 +108,7 @@ class PyTorchMultiHeadAttention(nn.Module):
def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False): def __init__(self, d_in, d_out, num_heads, dropout=0.0, qkv_bias=False):
super().__init__() super().__init__()
assert d_out % num_heads == 0, "embed_dim is indivisible by num_heads" assert d_out % num_heads == 0, "d_out is indivisible by num_heads"
self.num_heads = num_heads self.num_heads = num_heads
self.head_dim = d_out // num_heads self.head_dim = d_out // num_heads