yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-12-14 16:47:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

use blocksize in positional embedding

Browse Source
This commit is contained in:
rasbt 2024-01-15 08:15:33 -06:00
parent 9e85f13ba9
commit dfe2c3b46f
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ch03/01_main-chapter-code/multihead-attention.ipynb
View File

@ -88,7 +88,7 @@
"\n", "\n",
"\n", "\n",
"token_embedding_layer = nn.Embedding(vocab_size, output_dim)\n", "token_embedding_layer = nn.Embedding(vocab_size, output_dim)\n",
"pos_embedding_layer = nn.Embedding(vocab_size, output_dim)\n", "pos_embedding_layer = torch.nn.Embedding(block_size, output_dim)\n",
"\n", "\n",
"max_length = 4\n", "max_length = 4\n",
"dataloader = create_dataloader(raw_text, batch_size=8, max_length=max_length, stride=5)" "dataloader = create_dataloader(raw_text, batch_size=8, max_length=max_length, stride=5)"
@ -324,6 +324,14 @@
"\n", "\n",
"print(\"context_vecs.shape:\", context_vecs.shape)" "print(\"context_vecs.shape:\", context_vecs.shape)"
] ]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f8d4be84-28bb-41d5-996c-4936acffd411",
"metadata": {},
"outputs": [],
"source": []
} }
], ],
"metadata": { "metadata": {