yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-11-04 03:40:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

format the other GPT architecture sizes

Browse Source
This commit is contained in:
rasbt 2024-02-10 17:47:56 -06:00
parent 40477c55b3
commit 496b52f842
1 changed files with 20 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
ch04/01_main-chapter-code/ch04.ipynb
View File

@ -884,7 +884,7 @@
"source": [
"torch.manual_seed(123)\n",
"\n",
"x = torch.rand(2, 6, 768)\n",
"x = torch.rand(2, 6, 768) # Shape: [batch_size, num_tokens, emb_dim]\n",
"block = TransformerBlock(GPT_CONFIG_124M)\n",
"output = block(x)\n",
"\n",
@ -1140,27 +1140,27 @@
"id": "309a3be4-c20a-4657-b4e0-77c97510b47c",
"metadata": {},
"source": [
"- Exercise: you can try the other configurations as well:\n",
"- Exercise: you can try the following other configurations, which are referenced in the [GPT-2 paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf), as well.\n",
"\n",
"- GPT2-small (the 124M configuration we implemented):\n",
" - \"emb_dim\" = 768\n",
" - \"n_layers\" = 12\n",
" - \"n_heads\" = 12\n",
" - **GPT2-small** (the 124M configuration we already implemented):\n",
" - \"emb_dim\" = 768\n",
" - \"n_layers\" = 12\n",
" - \"n_heads\" = 12\n",
"\n",
"- GPT2-medium:\n",
" - \"emb_dim\" = 1024\n",
" - \"n_layers\" = 24\n",
" - \"n_heads\" = 16\n",
"\n",
"- GPT2-large:\n",
" - \"emb_dim\" = 1280\n",
" - \"n_layers\" = 36\n",
" - \"n_heads\" = 20\n",
"\n",
"- GPT2-XL:\n",
" - \"emb_dim\" = 1600\n",
" - \"n_layers\" = 48\n",
" - \"n_heads\" = 25"
" - **GPT2-medium:**\n",
" - \"emb_dim\" = 1024\n",
" - \"n_layers\" = 24\n",
" - \"n_heads\" = 16\n",
" \n",
" - **GPT2-large:**\n",
" - \"emb_dim\" = 1280\n",
" - \"n_layers\" = 36\n",
" - \"n_heads\" = 20\n",
" \n",
" - **GPT2-XL:**\n",
" - \"emb_dim\" = 1600\n",
" - \"n_layers\" = 48\n",
" - \"n_heads\" = 25"
]
},
{