yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-10-31 01:41:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix incorrect indentation (#536)

Browse Source
This commit is contained in:
Kasen 2025-02-19 04:47:31 +08:00 committed by GitHub
parent 18dd8ab633
commit 0a5214b804
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ch02/05_bpe-from-scratch/bpe-from-scratch.ipynb
View File

@ -382,7 +382,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": null,
"id": "3e4a15ec-2667-4f56-b7c1-34e8071b621d", "id": "3e4a15ec-2667-4f56-b7c1-34e8071b621d",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@ -499,7 +499,7 @@
" if lines and lines[0].startswith(\"#\"):\n", " if lines and lines[0].startswith(\"#\"):\n",
" lines = lines[1:]\n", " lines = lines[1:]\n",
"\n", "\n",
" for rank, line in enumerate(lines):\n", " for line in lines:\n",
" pair = tuple(line.strip().split())\n", " pair = tuple(line.strip().split())\n",
" if len(pair) == 2:\n", " if len(pair) == 2:\n",
" token1, token2 = pair\n", " token1, token2 = pair\n",
@ -511,10 +511,10 @@
" merged_token_id = self.inverse_vocab[merged_token]\n", " merged_token_id = self.inverse_vocab[merged_token]\n",
" self.bpe_merges[(token_id1, token_id2)] = merged_token_id\n", " self.bpe_merges[(token_id1, token_id2)] = merged_token_id\n",
" # print(f\"Loaded merge: '{token1}' + '{token2}' -> '{merged_token}' (ID: {merged_token_id})\")\n", " # print(f\"Loaded merge: '{token1}' + '{token2}' -> '{merged_token}' (ID: {merged_token_id})\")\n",
" else:\n",
" print(f\"Merged token '{merged_token}' not found in vocab. Skipping.\")\n",
" else:\n", " else:\n",
" print(f\"Merged token '{merged_token}' not found in vocab. Skipping.\")\n", " print(f\"Skipping pair {pair} as one of the tokens is not in the vocabulary.\")\n",
" else:\n",
" print(f\"Skipping pair {pair} as one of the tokens is not in the vocabulary.\")\n",
"\n", "\n",
" def encode(self, text):\n", " def encode(self, text):\n",
" \"\"\"\n", " \"\"\"\n",