From 85827e0a0b2279a65822a8a876e7fc9780e3b40f Mon Sep 17 00:00:00 2001 From: rasbt Date: Wed, 19 Jun 2024 17:37:42 -0500 Subject: [PATCH] note about dropout --- ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py b/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py index a4cb7de..0bbf2b1 100644 --- a/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py +++ b/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py @@ -180,7 +180,7 @@ if __name__ == "__main__": "emb_dim": 12, # Embedding dimension "n_heads": 2, # Number of attention heads "n_layers": 2, # Number of layers - "drop_rate": 0.0, # Dropout rate + "drop_rate": 0.0, # Dropout rate, deactivated via 0.0 as dropout in LLMs is not recommended anymore "qkv_bias": False # Query-key-value bias }