yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-08-04 14:52:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

note about dropout

Browse Source
This commit is contained in:
rasbt 2024-06-19 17:37:42 -05:00
parent bbb2a0c3d5
commit 85827e0a0b
No known key found for this signature in database
GPG Key ID: 3C6E5C7C075611DB
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py
View File

@ -180,7 +180,7 @@ if __name__ == "__main__":
"emb_dim": 12, # Embedding dimension "emb_dim": 12, # Embedding dimension
"n_heads": 2, # Number of attention heads "n_heads": 2, # Number of attention heads
"n_layers": 2, # Number of layers "n_layers": 2, # Number of layers
"drop_rate": 0.0, # Dropout rate "drop_rate": 0.0, # Dropout rate, deactivated via 0.0 as dropout in LLMs is not recommended anymore
"qkv_bias": False # Query-key-value bias "qkv_bias": False # Query-key-value bias
} }