From c8825b7c22e904d6598de9714bef65d43f0bda5f Mon Sep 17 00:00:00 2001 From: rasbt Date: Fri, 27 Oct 2023 06:19:40 -0500 Subject: [PATCH] add exercise solutions --- .../exercise-solutions.ipynb | 176 ++++++++++ ch02/01_main-chapter-code/dataloader.ipynb | 2 +- .../exercise-solutions.ipynb | 328 ++++++++++++++++++ 3 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 appendix-A/03_main-chapter-code/exercise-solutions.ipynb create mode 100644 ch02/01_main-chapter-code/exercise-solutions.ipynb diff --git a/appendix-A/03_main-chapter-code/exercise-solutions.ipynb b/appendix-A/03_main-chapter-code/exercise-solutions.ipynb new file mode 100644 index 0000000..f934b2d --- /dev/null +++ b/appendix-A/03_main-chapter-code/exercise-solutions.ipynb @@ -0,0 +1,176 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exercise A.3" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "class NeuralNetwork(torch.nn.Module):\n", + " def __init__(self, num_inputs, num_outputs):\n", + " super().__init__()\n", + "\n", + " self.layers = torch.nn.Sequential(\n", + " \n", + " # 1st hidden layer\n", + " torch.nn.Linear(num_inputs, 30),\n", + " torch.nn.ReLU(),\n", + "\n", + " # 2nd hidden layer\n", + " torch.nn.Linear(30, 20),\n", + " torch.nn.ReLU(),\n", + "\n", + " # output layer\n", + " torch.nn.Linear(20, num_outputs),\n", + " )\n", + "\n", + " def forward(self, x):\n", + " logits = self.layers(x)\n", + " return logits" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total number of trainable model parameters: 752\n" + ] + } + ], + "source": [ + "model = NeuralNetwork(2, 2)\n", + "\n", + "num_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "print(\"Total number of trainable model parameters:\", num_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exercise A.4" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "qGgnamiyLJxp" + }, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "a = torch.rand(100, 200)\n", + "b = torch.rand(200, 300)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "CvGvIeVkLzXE", + "outputId": "44d027be-0787-4348-9c06-4e559d94d0e1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "63.8 µs ± 8.7 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" + ] + } + ], + "source": [ + "%timeit a @ b" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "OmRtZLa9L2ZG" + }, + "outputs": [], + "source": [ + "a, b = a.to(\"cuda\"), b.to(\"cuda\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "duLEhXDPL6k0", + "outputId": "3486471d-fd62-446f-9855-2d01f41fd101" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "13.8 µs ± 425 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n" + ] + } + ], + "source": [ + "%timeit a @ b" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Zqqa-To2L749" + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "V100", + "machine_shape": "hm", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/ch02/01_main-chapter-code/dataloader.ipynb b/ch02/01_main-chapter-code/dataloader.ipynb index 32a5b1c..2da0c32 100644 --- a/ch02/01_main-chapter-code/dataloader.ipynb +++ b/ch02/01_main-chapter-code/dataloader.ipynb @@ -142,7 +142,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.11.4" } }, "nbformat": 4, diff --git a/ch02/01_main-chapter-code/exercise-solutions.ipynb b/ch02/01_main-chapter-code/exercise-solutions.ipynb new file mode 100644 index 0000000..5ad3685 --- /dev/null +++ b/ch02/01_main-chapter-code/exercise-solutions.ipynb @@ -0,0 +1,328 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ab88d307-61ba-45ef-89bc-e3569443dfca", + "metadata": {}, + "source": [ + "# Chapter 2 Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "id": "6f678e62-7bcb-4405-86ae-dce94f494303", + "metadata": {}, + "source": [ + "# Exercise 2.1" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7614337f-f639-42c9-a99b-d33f74fa8a03", + "metadata": {}, + "outputs": [], + "source": [ + "import tiktoken\n", + "\n", + "tokenizer = tiktoken.get_encoding(\"gpt2\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "664397bc-6daa-4b88-90aa-e8fc1fbd5846", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[33901]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.encode(\"Ak\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d3664332-e6bb-447e-8b96-203aafde8b24", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[86]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.encode(\"w\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "2773c09d-c136-4372-a2be-04b58d292842", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[343]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.encode(\"ir\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "8a6abd32-1e0a-4038-9dd2-673f47bcdeb5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[86]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.encode(\"w\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "26ae940a-9841-4e27-a1df-b83fc8a488b3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[220]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.encode(\" \")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a606c39a-6747-4cd8-bb38-e3183f80908d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[959]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.encode(\"ier\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "47c7268d-8fdc-4957-bc68-5be6113f45a7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Akwirw ier'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.decode([33901, 86, 343, 86, 220, 959])" + ] + }, + { + "cell_type": "markdown", + "id": "29e5034a-95ed-46d8-9972-589354dc9fd4", + "metadata": {}, + "source": [ + "# Exercise 2.2" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "4d50af16-937b-49e0-8ffd-42d30cbb41c9", + "metadata": {}, + "outputs": [], + "source": [ + "import tiktoken\n", + "import torch\n", + "from torch.utils.data import Dataset, DataLoader\n", + "\n", + "\n", + "class GPTDatasetV1(Dataset):\n", + " def __init__(self, txt, tokenizer, max_length, stride):\n", + " self.tokenizer = tokenizer\n", + " self.input_ids = []\n", + " self.target_ids = []\n", + "\n", + " # Tokenize the entire text\n", + " token_ids = tokenizer.encode(txt)\n", + "\n", + " # Use a sliding window to chunk the book into overlapping sequences of max_length\n", + " for i in range(0, len(token_ids) - max_length, stride):\n", + " input_chunk = token_ids[i:i + max_length]\n", + " target_chunk = token_ids[i + 1: i + max_length + 1]\n", + " self.input_ids.append(torch.tensor(input_chunk))\n", + " self.target_ids.append(torch.tensor(target_chunk))\n", + "\n", + " def __len__(self):\n", + " return len(self.input_ids)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.input_ids[idx], self.target_ids[idx]\n", + "\n", + "\n", + "def create_dataloader(txt, batch_size=4, max_length=256, stride=128):\n", + " # Initialize the tokenizer\n", + " tokenizer = tiktoken.get_encoding(\"gpt2\")\n", + "\n", + " # Create dataset\n", + " dataset = GPTDatasetV1(txt, tokenizer, max_length, stride)\n", + "\n", + " # Create dataloader\n", + " dataloader = DataLoader(dataset, batch_size=batch_size)\n", + "\n", + " return dataloader\n", + "\n", + "\n", + "with open(\"the-verdict.txt\", \"r\", encoding=\"utf-8\") as f:\n", + " raw_text = f.read()\n", + "\n", + "tokenizer = tiktoken.get_encoding(\"gpt2\")\n", + "encoded_text = tokenizer.encode(raw_text)\n", + "\n", + "vocab_size = 50257\n", + "output_dim = 256\n", + "token_embedding_layer = torch.nn.Embedding(vocab_size, output_dim)\n", + "pos_embedding_layer = torch.nn.Embedding(vocab_size, output_dim)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "0128eefa-d7c8-4f76-9851-566dfa7c3745", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 40, 367],\n", + " [2885, 1464],\n", + " [1807, 3619],\n", + " [ 402, 271]])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataloader = create_dataloader(raw_text, batch_size=4, max_length=2, stride=2)\n", + "\n", + "for batch in dataloader:\n", + " x, y = batch\n", + " break\n", + "\n", + "x" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "ff5c1e90-c6de-4a87-adf6-7e19f603291c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 40, 367, 2885, 1464, 1807, 3619, 402, 271],\n", + " [ 2885, 1464, 1807, 3619, 402, 271, 10899, 2138],\n", + " [ 1807, 3619, 402, 271, 10899, 2138, 257, 7026],\n", + " [ 402, 271, 10899, 2138, 257, 7026, 15632, 438]])" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataloader = create_dataloader(raw_text, batch_size=4, max_length=8, stride=2)\n", + "\n", + "for batch in dataloader:\n", + " x, y = batch\n", + " break\n", + "\n", + "x" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}