From 3198363c4f45b28e63a6cc9a79a380ed8b16bccf Mon Sep 17 00:00:00 2001 From: rasbt Date: Mon, 4 Mar 2024 06:42:58 -0600 Subject: [PATCH] add wording from three to four --- ch02/01_main-chapter-code/ch02.ipynb | 139 ++++++++++++++++----------- 1 file changed, 81 insertions(+), 58 deletions(-) diff --git a/ch02/01_main-chapter-code/ch02.ipynb b/ch02/01_main-chapter-code/ch02.ipynb index 72c43fa..13299c0 100644 --- a/ch02/01_main-chapter-code/ch02.ipynb +++ b/ch02/01_main-chapter-code/ch02.ipynb @@ -26,7 +26,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "torch version: 2.2.1\n", + "torch version: 2.1.0\n", "tiktoken version: 0.5.1\n" ] } @@ -559,7 +559,7 @@ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[16], line 5\u001b[0m\n\u001b[1;32m 1\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m SimpleTokenizerV1(vocab)\n\u001b[1;32m 3\u001b[0m text \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHello, do you like tea. Is this-- a test?\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 5\u001b[0m \u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[12], line 9\u001b[0m, in \u001b[0;36mSimpleTokenizerV1.encode\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m 7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[1;32m 8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[0;32m----> 9\u001b[0m ids \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr_to_int\u001b[49m\u001b[43m[\u001b[49m\u001b[43ms\u001b[49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43ms\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mpreprocessed\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n", + "Cell \u001b[0;32mIn[12], line 9\u001b[0m, in \u001b[0;36mSimpleTokenizerV1.encode\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m 7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[1;32m 8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[0;32m----> 9\u001b[0m ids \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstr_to_int[s] \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n", "Cell \u001b[0;32mIn[12], line 9\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[1;32m 8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[0;32m----> 9\u001b[0m ids \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr_to_int\u001b[49m\u001b[43m[\u001b[49m\u001b[43ms\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n", "\u001b[0;31mKeyError\u001b[0m: 'Hello'" ] @@ -585,7 +585,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "ce9df29c-6c5b-43f1-8c1a-c7f7b79db78f", "metadata": {}, "outputs": [], @@ -601,17 +601,17 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "id": "57c3143b-e860-4d3b-a22a-de22b547a6a9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "1159" + "1161" ] }, - "execution_count": 17, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -622,7 +622,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "id": "50e51bb1-ae05-4aa8-a9ff-455b65ed1959", "metadata": {}, "outputs": [ @@ -630,11 +630,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "('yet', 1154)\n", - "('you', 1155)\n", "('younger', 1156)\n", "('your', 1157)\n", - "('yourself', 1158)\n" + "('yourself', 1158)\n", + "('<|endoftext|>', 1159)\n", + "('<|unk|>', 1160)\n" ] } ], @@ -653,7 +653,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "id": "948861c5-3f30-4712-a234-725f20d26f68", "metadata": {}, "outputs": [], @@ -689,7 +689,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "id": "effcef79-e0a5-4f4a-a43a-31dd94b9250a", "metadata": {}, "outputs": [ @@ -714,22 +714,34 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "id": "ddfe7346-398d-4bf8-99f1-5b071244ce95", "metadata": {}, "outputs": [ { - "ename": "KeyError", - "evalue": "'<|unk|>'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[21], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[19], line 12\u001b[0m, in \u001b[0;36mSimpleTokenizerV2.encode\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m 8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[1;32m 9\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item \u001b[38;5;28;01mif\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstr_to_int \n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m<|unk|>\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[0;32m---> 12\u001b[0m ids \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr_to_int\u001b[49m\u001b[43m[\u001b[49m\u001b[43ms\u001b[49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43ms\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mpreprocessed\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n", - "Cell \u001b[0;32mIn[19], line 12\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[1;32m 9\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item \u001b[38;5;28;01mif\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstr_to_int \n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m<|unk|>\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[0;32m---> 12\u001b[0m ids \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr_to_int\u001b[49m\u001b[43m[\u001b[49m\u001b[43ms\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n", - "\u001b[0;31mKeyError\u001b[0m: '<|unk|>'" - ] + "data": { + "text/plain": [ + "[1160,\n", + " 5,\n", + " 362,\n", + " 1155,\n", + " 642,\n", + " 1000,\n", + " 10,\n", + " 1159,\n", + " 57,\n", + " 1013,\n", + " 981,\n", + " 1009,\n", + " 738,\n", + " 1013,\n", + " 1160,\n", + " 7]" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -738,10 +750,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "0c350ff6-2734-4e84-9ec7-d578baa4ae1b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'<|unk|>, do you like tea? <|endoftext|> In the sunlit terraces of the <|unk|>.'" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tokenizer.decode(tokenizer.encode(text))" ] @@ -769,7 +792,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 24, "id": "ede1d41f-934b-4bf4-8184-54394a257a94", "metadata": {}, "outputs": [], @@ -779,7 +802,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 25, "id": "48967a77-7d17-42bf-9e92-fc619d63a59e", "metadata": {}, "outputs": [ @@ -800,7 +823,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "id": "6ad3312f-a5f7-4efc-9d7d-8ea09d7b5128", "metadata": {}, "outputs": [], @@ -810,7 +833,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "id": "5ff2cd85-7cfb-4325-b390-219938589428", "metadata": {}, "outputs": [ @@ -832,7 +855,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "id": "d26a48bb-f82e-41a8-a955-a1c9cf9d50ab", "metadata": {}, "outputs": [ @@ -860,7 +883,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 29, "id": "848d5ade-fd1f-46c3-9e31-1426e315c71b", "metadata": {}, "outputs": [ @@ -891,7 +914,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 30, "id": "e84424a7-646d-45b6-99e3-80d15fb761f2", "metadata": {}, "outputs": [], @@ -901,7 +924,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 31, "id": "dfbff852-a92f-48c8-a46d-143a0f109f40", "metadata": {}, "outputs": [ @@ -934,7 +957,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 32, "id": "d97b031e-ed55-409d-95f2-aeb38c6fe366", "metadata": {}, "outputs": [ @@ -959,7 +982,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 33, "id": "f57bd746-dcbf-4433-8e24-ee213a8c34a1", "metadata": {}, "outputs": [ @@ -1001,7 +1024,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 34, "id": "e1770134-e7f3-4725-a679-e04c3be48cac", "metadata": {}, "outputs": [ @@ -1009,7 +1032,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "PyTorch version: 2.2.1\n" + "PyTorch version: 2.1.0\n" ] } ], @@ -1028,7 +1051,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 35, "id": "74b41073-4c9f-46e2-a1bd-d38e4122b375", "metadata": {}, "outputs": [], @@ -1061,7 +1084,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 36, "id": "5eb30ebe-97b3-43c5-9ff1-a97d621b3c4e", "metadata": {}, "outputs": [], @@ -1091,7 +1114,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 37, "id": "df31d96c-6bfd-4564-a956-6192242d7579", "metadata": {}, "outputs": [], @@ -1102,7 +1125,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 38, "id": "9226d00c-ad9a-4949-a6e4-9afccfc7214f", "metadata": {}, "outputs": [ @@ -1124,7 +1147,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 39, "id": "10deb4bc-4de1-4d20-921e-4b1c7a0e1a6d", "metadata": {}, "outputs": [ @@ -1152,7 +1175,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 40, "id": "1916e7a6-f03d-4f09-91a6-d0bdbac5a58c", "metadata": {}, "outputs": [ @@ -1214,12 +1237,12 @@ "id": "44e014ca-1fc5-4b90-b6fa-c2097bb92c0b", "metadata": {}, "source": [ - "- Suppose we have the following three input examples with input ids 5, 1, 3, and 2 (after tokenization):" + "- Suppose we have the following four input examples with input ids 5, 1, 3, and 2 (after tokenization):" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 41, "id": "15a6304c-9474-4470-b85d-3991a49fa653", "metadata": {}, "outputs": [], @@ -1237,7 +1260,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 42, "id": "93cb2cee-9aa6-4bb8-8977-c65661d16eda", "metadata": {}, "outputs": [], @@ -1259,7 +1282,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 43, "id": "a686eb61-e737-4351-8f1c-222913d47468", "metadata": {}, "outputs": [ @@ -1300,7 +1323,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 44, "id": "e43600ba-f287-4746-8ddf-d0f71a9023ca", "metadata": {}, "outputs": [ @@ -1327,7 +1350,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 45, "id": "50280ead-0363-44c8-8c35-bb885d92c8b7", "metadata": {}, "outputs": [ @@ -1365,7 +1388,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 46, "id": "0b9e344d-03a6-4f2c-b723-67b6a20c5041", "metadata": {}, "outputs": [], @@ -1387,7 +1410,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 47, "id": "ad56a263-3d2e-4d91-98bf-d0b68d3c7fc3", "metadata": {}, "outputs": [], @@ -1400,7 +1423,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 48, "id": "84416b60-3707-4370-bcbc-da0b62f2b64d", "metadata": {}, "outputs": [ @@ -1430,7 +1453,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 49, "id": "7766ec38-30d0-4128-8c31-f49f063c43d1", "metadata": {}, "outputs": [ @@ -1457,7 +1480,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 50, "id": "cc048e20-7ac8-417e-81f5-8fe6f9a4fe07", "metadata": {}, "outputs": [], @@ -1468,7 +1491,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 51, "id": "c369a1e7-d566-4b53-b398-d6adafb44105", "metadata": {}, "outputs": [ @@ -1495,7 +1518,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 52, "id": "b22fab89-526e-43c8-9035-5b7018e34288", "metadata": {}, "outputs": [ @@ -1547,7 +1570,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.12" } }, "nbformat": 4,