From 4eea9ce12c05a32ed11242bb36f462c7be2ee034 Mon Sep 17 00:00:00 2001 From: Sebastian Raschka Date: Sun, 9 Jun 2024 10:35:26 -0500 Subject: [PATCH] ch07 first draft (#203) --- README.md | 2 +- ch07/01_main-chapter-code/ch07.ipynb | 1667 +++++++++++++++++ ch07/01_main-chapter-code/gpt_download.py | 99 + .../01_main-chapter-code/previous_chapters.py | 468 +++++ requirements.txt | 2 +- 5 files changed, 2236 insertions(+), 2 deletions(-) create mode 100644 ch07/01_main-chapter-code/ch07.ipynb create mode 100644 ch07/01_main-chapter-code/gpt_download.py create mode 100644 ch07/01_main-chapter-code/previous_chapters.py diff --git a/README.md b/README.md index 8a862e4..4be4aa1 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Alternatively, you can view this and other files on GitHub at [https://github.co | Ch 4: Implementing a GPT Model from Scratch | - [ch04.ipynb](ch04/01_main-chapter-code/ch04.ipynb)
- [gpt.py](ch04/01_main-chapter-code/gpt.py) (summary)
- [exercise-solutions.ipynb](ch04/01_main-chapter-code/exercise-solutions.ipynb) | [./ch04](./ch04) | | Ch 5: Pretraining on Unlabeled Data | - [ch05.ipynb](ch05/01_main-chapter-code/ch05.ipynb)
- [gpt_train.py](ch05/01_main-chapter-code/gpt_train.py) (summary)
- [gpt_generate.py](ch05/01_main-chapter-code/gpt_generate.py) (summary)
- [exercise-solutions.ipynb](ch05/01_main-chapter-code/exercise-solutions.ipynb) | [./ch05](./ch05) | | Ch 6: Finetuning for Text Classification | - [ch06.ipynb](ch06/01_main-chapter-code/ch06.ipynb)
- [gpt-class-finetune.py](ch06/01_main-chapter-code/gpt-class-finetune.py)
- [exercise-solutions.ipynb](ch06/01_main-chapter-code/exercise-solutions.ipynb) | [./ch06](./ch06) | -| Ch 7: Finetuning to Follow Instructions | Q2 2024 | ... | +| Ch 7: Finetuning to Follow Instructions | - [ch07.ipynb](ch07/01_main-chapter-code/ch07.ipynb) | [./ch07](./ch07) | | Appendix A: Introduction to PyTorch | - [code-part1.ipynb](appendix-A/01_main-chapter-code/code-part1.ipynb)
- [code-part2.ipynb](appendix-A/01_main-chapter-code/code-part2.ipynb)
- [DDP-script.py](appendix-A/01_main-chapter-code/DDP-script.py)
- [exercise-solutions.ipynb](appendix-A/01_main-chapter-code/exercise-solutions.ipynb) | [./appendix-A](./appendix-A) | | Appendix B: References and Further Reading | No code | - | | Appendix C: Exercise Solutions | No code | - | diff --git a/ch07/01_main-chapter-code/ch07.ipynb b/ch07/01_main-chapter-code/ch07.ipynb new file mode 100644 index 0000000..e4cf66a --- /dev/null +++ b/ch07/01_main-chapter-code/ch07.ipynb @@ -0,0 +1,1667 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "12e91914-5f51-43fa-b65b-625e73b4d17b", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "Supplementary code for the Build a Large Language Model From Scratch book by Sebastian Raschka
\n", + "
Code repository: https://github.com/rasbt/LLMs-from-scratch\n", + "\n", + "		\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "c2520ec3-722f-4f44-bdd1-885b13e7afbf", + "metadata": {}, + "source": [ + "# Chapter 7: Finetuning To Follow Instructions" + ] + }, + { + "cell_type": "markdown", + "id": "a984b9ef-af93-415a-9ec7-97385f28af7b", + "metadata": {}, + "source": [ + "- Comments & notes in progress ..." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4e19327b-6c02-4881-ad02-9b6d3ec0b1b4", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4e19327b-6c02-4881-ad02-9b6d3ec0b1b4", + "outputId": "538e79af-011b-4a60-f288-2d0312a2b5a6" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "matplotlib version: 3.8.4\n", + "tiktoken version: 0.6.0\n", + "torch version: 2.2.2\n", + "tqdm version: 4.66.2\n", + "tensorflow version: 2.16.1\n" + ] + } + ], + "source": [ + "from importlib.metadata import version\n", + "\n", + "pkgs = [\n", + " \"matplotlib\", # Plotting library\n", + " \"tiktoken\", # Tokenizer\n", + " \"torch\", # Deep learning library\n", + " \"tqdm\", # Progress bar\n", + " \"tensorflow\", # For OpenAI's pretrained weights\n", + "]\n", + "for p in pkgs:\n", + " print(f\"{p} version: {version(p)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "8bbc68e9-75b3-41f1-ac2c-e071c3cd0813", + "metadata": {}, + "source": [ + "## 7.1 Introduction to instruction finetuning" + ] + }, + { + "cell_type": "markdown", + "id": "5384f0cf-ef3c-4436-a5fa-59bd25649f86", + "metadata": {}, + "source": [ + "## 7.2 Preparing a dataset for supervised instruction finetuning" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0G3axLw6kY1N", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0G3axLw6kY1N", + "outputId": "2a9a1c83-9c46-49a5-f9df-fce3320f7db2" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1100\n" + ] + } + ], + "source": [ + "import json\n", + "import os\n", + "import urllib\n", + "\n", + "\n", + "def download_and_load_file(file_path, url):\n", + "\n", + " if not os.path.exists(file_path):\n", + " with urllib.request.urlopen(url) as response:\n", + " text_data = response.read().decode('utf-8')\n", + " with open(file_path, \"w\", encoding=\"utf-8\") as file:\n", + " file.write(text_data)\n", + " else:\n", + " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", + " text_data = file.read()\n", + "\n", + " with open(file_path, \"r\") as file:\n", + " data = json.load(file)\n", + "\n", + " return data\n", + "\n", + "\n", + "file_path = \"instruction-data.json\"\n", + "url = \"https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/main/ch07/01_main-chapter-code/instruction-data.json\"\n", + "\n", + "data = download_and_load_file(file_path, url)\n", + "print(len(data))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "-LiuBMsHkzQV", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-LiuBMsHkzQV", + "outputId": "fc3b22fd-9a53-405e-9c25-2a5873d343d1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'instruction': 'Evaluate the following phrase by transforming it into the spelling given.', 'input': 'freind --> friend', 'output': 'The spelling of the given phrase \"freind\" is incorrect, the correct spelling is \"friend\".'}\n" + ] + } + ], + "source": [ + "print(data[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "uFInFxDDk2Je", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "uFInFxDDk2Je", + "outputId": "84cb1aad-233a-488a-f6b0-6cb977834367" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'instruction': \"Change the sentence 'You should have called me.' into a question.\", 'input': '', 'output': 'Should you have called me?'}\n" + ] + } + ], + "source": [ + "print(data[-1])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "aFZVopbIlNfx", + "metadata": { + "id": "aFZVopbIlNfx" + }, + "outputs": [], + "source": [ + "train_portion = int(len(data) * 0.85) # 85% for training\n", + "test_portion = int(len(data) * 0.1) # 10% for testing\n", + "val_portion = len(data) - train_portion - test_portion # Remaining 5% for validation\n", + "\n", + "train_data = data[:train_portion]\n", + "test_data = data[train_portion:train_portion + test_portion]\n", + "val_data = data[train_portion + test_portion:]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "-zf6oht6bIUQ", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-zf6oht6bIUQ", + "outputId": "bf33cd9a-2778-4365-c51d-d394c817c4fb" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "935\n", + "55\n", + "110\n" + ] + } + ], + "source": [ + "print(len(train_data))\n", + "print(len(val_data))\n", + "print(len(test_data))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "Jhk37nnJnkBh", + "metadata": { + "id": "Jhk37nnJnkBh" + }, + "outputs": [], + "source": [ + "def format_input(entry):\n", + " instruction_text = (\n", + " f\"Below is an instruction that describes a task. \"\n", + " f\"Write a response that appropriately completes the request.\"\n", + " f\"\\n\\n### Instruction:\\n{entry['instruction']}\"\n", + " )\n", + "\n", + " input_text = f\"\\n\\n### Input:\\n{entry['input']}\" if entry[\"input\"] else \"\"\n", + " instruction_text + input_text\n", + "\n", + " return instruction_text + input_text" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "F9UQRfjzo4Js", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "F9UQRfjzo4Js", + "outputId": "b56e6c03-f603-4e9d-c1b6-b4a70403caf9" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Evaluate the following phrase by transforming it into the spelling given.\n", + "\n", + "### Input:\n", + "freind --> friend\n" + ] + } + ], + "source": [ + "print(format_input(train_data[0]))" + ] + }, + { + "cell_type": "markdown", + "id": "fcaaf606-f913-4445-8301-632ae10d387d", + "metadata": {}, + "source": [ + "## 7.3 Creating data loaders for an instruction dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "K6MWf0lhu8GP", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "K6MWf0lhu8GP", + "outputId": "bb01c511-4023-4b74-9781-8385da75b391" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[50256]\n" + ] + } + ], + "source": [ + "import tiktoken\n", + "\n", + "tokenizer = tiktoken.get_encoding(\"gpt2\")\n", + "print(tokenizer.encode(\"<|endoftext|>\", allowed_special={\"<|endoftext|>\"}))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "adc29dc4-f1c7-4c71-937b-95119d6239bb", + "metadata": { + "id": "adc29dc4-f1c7-4c71-937b-95119d6239bb" + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch.utils.data import Dataset\n", + "\n", + "\n", + "class InstructionDataset(Dataset):\n", + " def __init__(self, data, tokenizer):\n", + " self.data = data\n", + "\n", + " # Pre-tokenize texts\n", + " self.encoded_texts = []\n", + " for entry in data:\n", + " instruction_plus_input = format_input(entry)\n", + " response_text = f\"\\n\\n### Response:\\n{entry['output']}\"\n", + " full_text = instruction_plus_input + response_text\n", + " self.encoded_texts.append(\n", + " tokenizer.encode(\n", + " full_text, allowed_special={\"<|endoftext|>\"}\n", + " )\n", + " )\n", + "\n", + " def __getitem__(self, index):\n", + " return self.encoded_texts[index]\n", + "\n", + " def __len__(self):\n", + " return len(self.data)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "W2jvh-OP9MFV", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "W2jvh-OP9MFV", + "outputId": "7878ef5f-635a-491a-99b2-07b3319daefc" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(1.1269)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Explain index masking\n", + "\n", + "targets = torch.tensor([0, 1])\n", + "inputs = torch.tensor(\n", + " [[-1., 1.],\n", + " [-0.5, 1.5]]\n", + ")\n", + "\n", + "torch.nn.functional.cross_entropy(inputs, targets)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "nvVMuil89v9N", + "metadata": { + "id": "nvVMuil89v9N" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(0.7936)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "targets = torch.tensor([0, 1, 1])\n", + "inputs = torch.tensor(\n", + " [[-1., 1.],\n", + " [-0.5, 1.5],\n", + " [-0.5, 1.5]]\n", + ")\n", + "torch.nn.functional.cross_entropy(inputs, targets)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "RTyB1vah9p56", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RTyB1vah9p56", + "outputId": "f1c132ad-85db-411d-cfc8-1d9ab3aec79d" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(1.1269)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "targets = torch.tensor([0, 1, -100])\n", + "inputs = torch.tensor(\n", + " [[-1., 1.],\n", + " [-0.5, 1.5],\n", + " [-0.5, 1.5]]\n", + ")\n", + "torch.nn.functional.cross_entropy(inputs, targets)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "41ec6e2d-9eb2-4124-913e-d2af39be4cf2", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_collate_fn(\n", + " batch, \n", + " pad_token_id=50256,\n", + " ignore_index=-100,\n", + " allowed_max_length=None, \n", + " device=\"cpu\"\n", + "):\n", + " # Find the longest sequence in the batch\n", + " batch_max_length = max(len(item) for item in batch)\n", + "\n", + " # Pad and prepare inputs and targets\n", + " inputs_lst, targets_lst = [], []\n", + " for item in batch:\n", + " # Pad sequences to max_length\n", + " padded = item + [pad_token_id] * (batch_max_length - len(item))\n", + " inputs = torch.tensor(padded[:-1]) # Truncate the last token for inputs\n", + " targets = torch.tensor(padded[1:]) # Shift +1 to the right for targets\n", + "\n", + " # Replace all but the first padding tokens in targets by ignore_index\n", + " mask = targets == pad_token_id\n", + " indices = torch.nonzero(mask).squeeze()\n", + " if indices.numel() > 1:\n", + " targets[indices[1:]] = ignore_index\n", + "\n", + " # Optionally truncate to maximum sequence length\n", + " if allowed_max_length is not None:\n", + " inputs = inputs[:allowed_max_length]\n", + " targets = targets[:allowed_max_length]\n", + "\n", + " inputs_lst.append(inputs)\n", + " targets_lst.append(targets)\n", + "\n", + " inputs_tensor = torch.stack(inputs_lst).to(device)\n", + " targets_tensor = torch.stack(targets_lst).to(device)\n", + "\n", + " return inputs_tensor, targets_tensor" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "cdf5eec4-9ebe-4be0-9fca-9a47bee88fdc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[ 0, 1, 2, 3, 4, 50256],\n", + " [ 7, 8, 9, 50256, 50256, 50256]]),\n", + " tensor([[ 1, 2, 3, 4, 50256, -100],\n", + " [ 8, 9, 50256, -100, -100, -100]]))" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "inputs_1 = [0, 1, 2, 3, 4, 50256, 50256]\n", + "inputs_2 = [7, 8, 9]\n", + "\n", + "batch = (\n", + " inputs_1,\n", + " inputs_2\n", + ")\n", + "\n", + "custom_collate_fn(batch)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "etpqqWh8phKc", + "metadata": { + "id": "etpqqWh8phKc" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Device: cpu\n" + ] + } + ], + "source": [ + "from functools import partial\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(\"Device:\", device)\n", + "\n", + "customized_collate_fn = partial(custom_collate_fn, device=device)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "BtWkgir6Hlpe", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "BtWkgir6Hlpe", + "outputId": "8e3a969d-e1f6-4574-cc07-3f8401068555" + }, + "outputs": [], + "source": [ + "from torch.utils.data import DataLoader\n", + "\n", + "\n", + "num_workers = 0\n", + "batch_size = 8\n", + "\n", + "torch.manual_seed(123)\n", + "\n", + "train_dataset = InstructionDataset(train_data, tokenizer)\n", + "train_loader = DataLoader(\n", + " train_dataset,\n", + " batch_size=batch_size,\n", + " collate_fn=customized_collate_fn,\n", + " shuffle=True,\n", + " drop_last=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "1d097dc8-ad34-4f05-b435-e4147965f532", + "metadata": { + "id": "1d097dc8-ad34-4f05-b435-e4147965f532" + }, + "outputs": [], + "source": [ + "val_dataset = InstructionDataset(val_data, tokenizer)\n", + "val_loader = DataLoader(\n", + " val_dataset,\n", + " batch_size=batch_size,\n", + " collate_fn=customized_collate_fn,\n", + " shuffle=False,\n", + " drop_last=False\n", + ")\n", + "\n", + "test_dataset = InstructionDataset(test_data, tokenizer)\n", + "test_loader = DataLoader(\n", + " test_dataset,\n", + " batch_size=batch_size,\n", + " collate_fn=customized_collate_fn,\n", + " shuffle=False,\n", + " drop_last=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "GGs1AI3vHpnX", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "GGs1AI3vHpnX", + "outputId": "eaabe39c-bb78-4fec-979c-6382c192a79f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loader:\n", + "torch.Size([8, 60]) torch.Size([8, 60])\n", + "torch.Size([8, 75]) torch.Size([8, 75])\n", + "torch.Size([8, 72]) torch.Size([8, 72])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 64]) torch.Size([8, 64])\n", + "torch.Size([8, 71]) torch.Size([8, 71])\n", + "torch.Size([8, 79]) torch.Size([8, 79])\n", + "torch.Size([8, 66]) torch.Size([8, 66])\n", + "torch.Size([8, 61]) torch.Size([8, 61])\n", + "torch.Size([8, 74]) torch.Size([8, 74])\n", + "torch.Size([8, 61]) torch.Size([8, 61])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 66]) torch.Size([8, 66])\n", + "torch.Size([8, 76]) torch.Size([8, 76])\n", + "torch.Size([8, 68]) torch.Size([8, 68])\n", + "torch.Size([8, 78]) torch.Size([8, 78])\n", + "torch.Size([8, 70]) torch.Size([8, 70])\n", + "torch.Size([8, 65]) torch.Size([8, 65])\n", + "torch.Size([8, 82]) torch.Size([8, 82])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 79]) torch.Size([8, 79])\n", + "torch.Size([8, 70]) torch.Size([8, 70])\n", + "torch.Size([8, 68]) torch.Size([8, 68])\n", + "torch.Size([8, 64]) torch.Size([8, 64])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 59]) torch.Size([8, 59])\n", + "torch.Size([8, 58]) torch.Size([8, 58])\n", + "torch.Size([8, 68]) torch.Size([8, 68])\n", + "torch.Size([8, 62]) torch.Size([8, 62])\n", + "torch.Size([8, 64]) torch.Size([8, 64])\n", + "torch.Size([8, 75]) torch.Size([8, 75])\n", + "torch.Size([8, 65]) torch.Size([8, 65])\n", + "torch.Size([8, 70]) torch.Size([8, 70])\n", + "torch.Size([8, 90]) torch.Size([8, 90])\n", + "torch.Size([8, 64]) torch.Size([8, 64])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 66]) torch.Size([8, 66])\n", + "torch.Size([8, 65]) torch.Size([8, 65])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 64]) torch.Size([8, 64])\n", + "torch.Size([8, 74]) torch.Size([8, 74])\n", + "torch.Size([8, 88]) torch.Size([8, 88])\n", + "torch.Size([8, 58]) torch.Size([8, 58])\n", + "torch.Size([8, 87]) torch.Size([8, 87])\n", + "torch.Size([8, 82]) torch.Size([8, 82])\n", + "torch.Size([8, 82]) torch.Size([8, 82])\n", + "torch.Size([8, 69]) torch.Size([8, 69])\n", + "torch.Size([8, 64]) torch.Size([8, 64])\n", + "torch.Size([8, 73]) torch.Size([8, 73])\n", + "torch.Size([8, 75]) torch.Size([8, 75])\n", + "torch.Size([8, 66]) torch.Size([8, 66])\n", + "torch.Size([8, 74]) torch.Size([8, 74])\n", + "torch.Size([8, 82]) torch.Size([8, 82])\n", + "torch.Size([8, 68]) torch.Size([8, 68])\n", + "torch.Size([8, 66]) torch.Size([8, 66])\n", + "torch.Size([8, 59]) torch.Size([8, 59])\n", + "torch.Size([8, 59]) torch.Size([8, 59])\n", + "torch.Size([8, 65]) torch.Size([8, 65])\n", + "torch.Size([8, 79]) torch.Size([8, 79])\n", + "torch.Size([8, 70]) torch.Size([8, 70])\n", + "torch.Size([8, 60]) torch.Size([8, 60])\n", + "torch.Size([8, 57]) torch.Size([8, 57])\n", + "torch.Size([8, 70]) torch.Size([8, 70])\n", + "torch.Size([8, 66]) torch.Size([8, 66])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 62]) torch.Size([8, 62])\n", + "torch.Size([8, 86]) torch.Size([8, 86])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 70]) torch.Size([8, 70])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 70]) torch.Size([8, 70])\n", + "torch.Size([8, 60]) torch.Size([8, 60])\n", + "torch.Size([8, 64]) torch.Size([8, 64])\n", + "torch.Size([8, 66]) torch.Size([8, 66])\n", + "torch.Size([8, 64]) torch.Size([8, 64])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 59]) torch.Size([8, 59])\n", + "torch.Size([8, 71]) torch.Size([8, 71])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 69]) torch.Size([8, 69])\n", + "torch.Size([8, 56]) torch.Size([8, 56])\n", + "torch.Size([8, 71]) torch.Size([8, 71])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 61]) torch.Size([8, 61])\n", + "torch.Size([8, 73]) torch.Size([8, 73])\n", + "torch.Size([8, 79]) torch.Size([8, 79])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 69]) torch.Size([8, 69])\n", + "torch.Size([8, 90]) torch.Size([8, 90])\n", + "torch.Size([8, 60]) torch.Size([8, 60])\n", + "torch.Size([8, 65]) torch.Size([8, 65])\n", + "torch.Size([8, 79]) torch.Size([8, 79])\n", + "torch.Size([8, 80]) torch.Size([8, 80])\n", + "torch.Size([8, 73]) torch.Size([8, 73])\n", + "torch.Size([8, 81]) torch.Size([8, 81])\n", + "torch.Size([8, 62]) torch.Size([8, 62])\n", + "torch.Size([8, 82]) torch.Size([8, 82])\n", + "torch.Size([8, 67]) torch.Size([8, 67])\n", + "torch.Size([8, 66]) torch.Size([8, 66])\n", + "torch.Size([8, 76]) torch.Size([8, 76])\n", + "torch.Size([8, 90]) torch.Size([8, 90])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 60]) torch.Size([8, 60])\n", + "torch.Size([8, 74]) torch.Size([8, 74])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 65]) torch.Size([8, 65])\n", + "torch.Size([8, 77]) torch.Size([8, 77])\n", + "torch.Size([8, 65]) torch.Size([8, 65])\n", + "torch.Size([8, 63]) torch.Size([8, 63])\n", + "torch.Size([8, 82]) torch.Size([8, 82])\n", + "torch.Size([8, 65]) torch.Size([8, 65])\n", + "torch.Size([8, 73]) torch.Size([8, 73])\n", + "torch.Size([8, 68]) torch.Size([8, 68])\n" + ] + } + ], + "source": [ + "print(\"Train loader:\")\n", + "for x, y in train_loader:\n", + " print(x.shape, y.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "21b8fd02-014f-4481-9b71-5bfee8f9dfcd", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "21b8fd02-014f-4481-9b71-5bfee8f9dfcd", + "outputId": "71ce098a-36b7-44fa-8c7c-f63db448fe40" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "21106, 318, 281, 12064, 326, 8477, 257, 4876, 13, 19430, 257, 2882, 326, 20431, 32543, 262, 2581, 13, 198, 198, 21017, 46486, 25, 198, 30003, 6525, 262, 6827, 1262, 257, 985, 576, 13, 198, 198, 21017, 23412, 25, 198, 464, 5156, 318, 845, 13779, 13, 198, 198, 21017, 18261, 25, 198, 464, 5156, 318, 355, 13779, 355, 257, 4936, 13, 50256, 50256, 50256, 50256, 50256, 50256, 50256, 50256, " + ] + } + ], + "source": [ + "for i in x[0]:\n", + " print(i.item(), end=\", \")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "51649ab4-1a7e-4a9e-92c5-950a24fde211", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "51649ab4-1a7e-4a9e-92c5-950a24fde211", + "outputId": "4cf98eac-b7f7-4687-b264-4508c0865865" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "318, 281, 12064, 326, 8477, 257, 4876, 13, 19430, 257, 2882, 326, 20431, 32543, 262, 2581, 13, 198, 198, 21017, 46486, 25, 198, 30003, 6525, 262, 6827, 1262, 257, 985, 576, 13, 198, 198, 21017, 23412, 25, 198, 464, 5156, 318, 845, 13779, 13, 198, 198, 21017, 18261, 25, 198, 464, 5156, 318, 355, 13779, 355, 257, 4936, 13, 50256, -100, -100, -100, -100, -100, -100, -100, -100, " + ] + } + ], + "source": [ + "for i in y[0]:\n", + " print(i.item(), end=\", \")" + ] + }, + { + "cell_type": "markdown", + "id": "d6aad445-8f19-4238-b9bf-db80767fb91a", + "metadata": { + "id": "d6aad445-8f19-4238-b9bf-db80767fb91a" + }, + "source": [ + "## 7.4 Loading a pretrained LLM" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "0d249d67-5eba-414e-9bd2-972ebf01329d", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0d249d67-5eba-414e-9bd2-972ebf01329d", + "outputId": "0ccd8d13-4f8a-44ce-ea22-0b6ea36bb06e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "File already exists and is up-to-date: gpt2/355M/checkpoint\n", + "File already exists and is up-to-date: gpt2/355M/encoder.json\n", + "File already exists and is up-to-date: gpt2/355M/hparams.json\n", + "File already exists and is up-to-date: gpt2/355M/model.ckpt.data-00000-of-00001\n", + "File already exists and is up-to-date: gpt2/355M/model.ckpt.index\n", + "File already exists and is up-to-date: gpt2/355M/model.ckpt.meta\n", + "File already exists and is up-to-date: gpt2/355M/vocab.bpe\n" + ] + } + ], + "source": [ + "from gpt_download import download_and_load_gpt2\n", + "from previous_chapters import GPTModel, load_weights_into_gpt\n", + "\n", + "\n", + "BASE_CONFIG = {\n", + " \"vocab_size\": 50257, # Vocabulary size\n", + " \"context_length\": 1024, # Context length\n", + " \"drop_rate\": 0.0, # Dropout rate\n", + " \"qkv_bias\": True # Query-key-value bias\n", + "}\n", + "\n", + "model_configs = {\n", + " \"gpt2-small (124M)\": {\"emb_dim\": 768, \"n_layers\": 12, \"n_heads\": 12},\n", + " \"gpt2-medium (355M)\": {\"emb_dim\": 1024, \"n_layers\": 24, \"n_heads\": 16},\n", + " \"gpt2-large (774M)\": {\"emb_dim\": 1280, \"n_layers\": 36, \"n_heads\": 20},\n", + " \"gpt2-xl (1558M)\": {\"emb_dim\": 1600, \"n_layers\": 48, \"n_heads\": 25},\n", + "}\n", + "\n", + "CHOOSE_MODEL = \"gpt2-medium (355M)\"\n", + "\n", + "BASE_CONFIG.update(model_configs[CHOOSE_MODEL])\n", + "\n", + "model_size = CHOOSE_MODEL.split(\" \")[-1].lstrip(\"(\").rstrip(\")\")\n", + "settings, params = download_and_load_gpt2(model_size=model_size, models_dir=\"gpt2\")\n", + "\n", + "model = GPTModel(BASE_CONFIG)\n", + "load_weights_into_gpt(model, params)\n", + "model.eval();" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "7bd32b7c-5b44-4d25-a09f-46836802ca74", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7bd32b7c-5b44-4d25-a09f-46836802ca74", + "outputId": "de446b9d-7667-48a5-c34a-f3c5cf70459b" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Convert the active sentence to passive: 'The chef cooks the meal every day.'\n", + "\n", + "### Response:\n", + "\n", + "The chef cooks the meal every day.\n", + "\n", + "### Instruction:\n", + "\n", + "Convert the active sentence to passive: 'The chef cooks the\n" + ] + } + ], + "source": [ + "from previous_chapters import (\n", + " generate,\n", + " text_to_token_ids,\n", + " token_ids_to_text\n", + ")\n", + "\n", + "torch.manual_seed(123)\n", + "\n", + "token_ids = generate(\n", + " model=model,\n", + " idx=text_to_token_ids(format_input(val_data[0]), tokenizer),\n", + " max_new_tokens=35,\n", + " context_size=BASE_CONFIG[\"context_length\"],\n", + ")\n", + "\n", + "print(token_ids_to_text(token_ids, tokenizer))" + ] + }, + { + "cell_type": "markdown", + "id": "70d27b9d-a942-4cf5-b797-848c5f01e723", + "metadata": { + "id": "70d27b9d-a942-4cf5-b797-848c5f01e723" + }, + "source": [ + "## 7.5 Finetuning the LLM on instruction data" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "65444865-df87-4d98-9faf-875e1c4be860", + "metadata": { + "id": "65444865-df87-4d98-9faf-875e1c4be860" + }, + "outputs": [], + "source": [ + "from previous_chapters import (\n", + " calc_loss_loader,\n", + " train_model_simple\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "d99fc6f8-63b2-43da-adbb-a7b6b92c8dd5", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "d99fc6f8-63b2-43da-adbb-a7b6b92c8dd5", + "outputId": "0c815e75-9357-42e6-fdf3-3ea13ffa4da4" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training loss: 3.8234103202819822\n", + "Validation loss: 3.7612109184265137\n" + ] + } + ], + "source": [ + "model.to(device)\n", + "\n", + "torch.manual_seed(123) # For reproducibility due to the shuffling in the data loader\n", + "\n", + "with torch.no_grad(): # Disable gradient tracking for efficiency because we are not training, yet\n", + " train_loss = calc_loss_loader(train_loader, model, device, num_batches=5)\n", + " val_loss = calc_loss_loader(val_loader, model, device, num_batches=5)\n", + "\n", + "print(\"Training loss:\", train_loss)\n", + "print(\"Validation loss:\", val_loss)" + ] + }, + { + "cell_type": "markdown", + "id": "db4b57fb-e689-4550-931c-6d34a932487c", + "metadata": {}, + "source": [ + "- Runtimes:\n", + "\n", + "
\n", + " \n", + "| Model | Platform | Runtime |\n", + "|--------------------|-----------------------|----------------|\n", + "| gpt2-medium (355M) | CPU (M3 MacBook Air) | 23.67 minutes |\n", + "| gpt2-medium (355M) | GPU (A100) | 1.29 minutes |\n", + "| gpt2-small (124M) | CPU (M3 MacBook Air) | 8.61 |\n", + "| gpt2-small (124M) | GPU (A100) | 0.59 minutes |\n", + "\n", + "
\n", + "\n", + "- Remainder of the notebook was run on M3 MacBook Air with the `\"gpt2-medium (355M)\"` model" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "78bcf83a-1fff-4540-97c1-765c4016d5e3", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "78bcf83a-1fff-4540-97c1-765c4016d5e3", + "outputId": "315368d9-5484-4527-f42d-b0d650d6aa23" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ep 1 (Step 000000): Train loss 2.636, Val loss 2.627\n", + "Ep 1 (Step 000005): Train loss 1.173, Val loss 1.103\n", + "Ep 1 (Step 000010): Train loss 0.873, Val loss 0.947\n", + "Ep 1 (Step 000015): Train loss 0.856, Val loss 0.907\n", + "Ep 1 (Step 000020): Train loss 0.777, Val loss 0.882\n", + "Ep 1 (Step 000025): Train loss 0.754, Val loss 0.860\n", + "Ep 1 (Step 000030): Train loss 0.799, Val loss 0.838\n", + "Ep 1 (Step 000035): Train loss 0.715, Val loss 0.810\n", + "Ep 1 (Step 000040): Train loss 0.673, Val loss 0.807\n", + "Ep 1 (Step 000045): Train loss 0.634, Val loss 0.791\n", + "Ep 1 (Step 000050): Train loss 0.663, Val loss 0.784\n", + "Ep 1 (Step 000055): Train loss 0.760, Val loss 0.764\n", + "Ep 1 (Step 000060): Train loss 0.721, Val loss 0.745\n", + "Ep 1 (Step 000065): Train loss 0.654, Val loss 0.736\n", + "Ep 1 (Step 000070): Train loss 0.535, Val loss 0.730\n", + "Ep 1 (Step 000075): Train loss 0.569, Val loss 0.729\n", + "Ep 1 (Step 000080): Train loss 0.606, Val loss 0.726\n", + "Ep 1 (Step 000085): Train loss 0.511, Val loss 0.710\n", + "Ep 1 (Step 000090): Train loss 0.563, Val loss 0.691\n", + "Ep 1 (Step 000095): Train loss 0.501, Val loss 0.682\n", + "Ep 1 (Step 000100): Train loss 0.504, Val loss 0.678\n", + "Ep 1 (Step 000105): Train loss 0.566, Val loss 0.671\n", + "Ep 1 (Step 000110): Train loss 0.556, Val loss 0.668\n", + "Ep 1 (Step 000115): Train loss 0.509, Val loss 0.665\n", + "Below is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: Convert the active sentence to passive: 'The chef cooks the meal every day.' ### Response: The meal is prepared every day by the chef.<|endoftext|>The following is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: Convert the active sentence to passive:\n", + "Ep 2 (Step 000120): Train loss 0.436, Val loss 0.672\n", + "Ep 2 (Step 000125): Train loss 0.451, Val loss 0.687\n", + "Ep 2 (Step 000130): Train loss 0.447, Val loss 0.682\n", + "Ep 2 (Step 000135): Train loss 0.405, Val loss 0.681\n", + "Ep 2 (Step 000140): Train loss 0.407, Val loss 0.680\n", + "Ep 2 (Step 000145): Train loss 0.370, Val loss 0.681\n", + "Ep 2 (Step 000150): Train loss 0.382, Val loss 0.676\n", + "Ep 2 (Step 000155): Train loss 0.413, Val loss 0.676\n", + "Ep 2 (Step 000160): Train loss 0.414, Val loss 0.685\n", + "Ep 2 (Step 000165): Train loss 0.379, Val loss 0.688\n", + "Ep 2 (Step 000170): Train loss 0.322, Val loss 0.683\n", + "Ep 2 (Step 000175): Train loss 0.338, Val loss 0.670\n", + "Ep 2 (Step 000180): Train loss 0.393, Val loss 0.659\n", + "Ep 2 (Step 000185): Train loss 0.417, Val loss 0.659\n", + "Ep 2 (Step 000190): Train loss 0.342, Val loss 0.649\n", + "Ep 2 (Step 000195): Train loss 0.330, Val loss 0.635\n", + "Ep 2 (Step 000200): Train loss 0.312, Val loss 0.634\n", + "Ep 2 (Step 000205): Train loss 0.355, Val loss 0.630\n", + "Ep 2 (Step 000210): Train loss 0.371, Val loss 0.629\n", + "Ep 2 (Step 000215): Train loss 0.394, Val loss 0.633\n", + "Ep 2 (Step 000220): Train loss 0.302, Val loss 0.646\n", + "Ep 2 (Step 000225): Train loss 0.344, Val loss 0.659\n", + "Ep 2 (Step 000230): Train loss 0.292, Val loss 0.656\n", + "Below is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: Convert the active sentence to passive: 'The chef cooks the meal every day.' ### Response: The meal is cooked everyday by the chef.<|endoftext|>The following is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: What is the capital of the United Kingdom\n", + "Ep 3 (Step 000235): Train loss 0.327, Val loss 0.663\n", + "Ep 3 (Step 000240): Train loss 0.275, Val loss 0.693\n", + "Ep 3 (Step 000245): Train loss 0.275, Val loss 0.707\n", + "Ep 3 (Step 000250): Train loss 0.246, Val loss 0.698\n", + "Ep 3 (Step 000255): Train loss 0.277, Val loss 0.688\n", + "Ep 3 (Step 000260): Train loss 0.268, Val loss 0.687\n", + "Ep 3 (Step 000265): Train loss 0.269, Val loss 0.694\n", + "Ep 3 (Step 000270): Train loss 0.282, Val loss 0.707\n", + "Ep 3 (Step 000275): Train loss 0.275, Val loss 0.701\n", + "Ep 3 (Step 000280): Train loss 0.293, Val loss 0.709\n", + "Ep 3 (Step 000285): Train loss 0.291, Val loss 0.711\n", + "Ep 3 (Step 000290): Train loss 0.288, Val loss 0.710\n", + "Ep 3 (Step 000295): Train loss 0.268, Val loss 0.703\n", + "Ep 3 (Step 000300): Train loss 0.262, Val loss 0.691\n", + "Ep 3 (Step 000305): Train loss 0.268, Val loss 0.688\n", + "Ep 3 (Step 000310): Train loss 0.270, Val loss 0.692\n", + "Ep 3 (Step 000315): Train loss 0.234, Val loss 0.697\n", + "Ep 3 (Step 000320): Train loss 0.252, Val loss 0.696\n", + "Ep 3 (Step 000325): Train loss 0.235, Val loss 0.701\n", + "Ep 3 (Step 000330): Train loss 0.239, Val loss 0.697\n", + "Ep 3 (Step 000335): Train loss 0.229, Val loss 0.687\n", + "Ep 3 (Step 000340): Train loss 0.246, Val loss 0.684\n", + "Ep 3 (Step 000345): Train loss 0.243, Val loss 0.676\n", + "Below is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: Convert the active sentence to passive: 'The chef cooks the meal every day.' ### Response: The chef cooks the meal every day.<|endoftext|>The following is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: What is the capital of the United Kingdom? \n", + "Training completed in 23.67 minutes.\n" + ] + } + ], + "source": [ + "import time\n", + "\n", + "start_time = time.time()\n", + "\n", + "torch.manual_seed(123)\n", + "\n", + "optimizer = torch.optim.AdamW(model.parameters(), lr=0.00005, weight_decay=0.1)\n", + "\n", + "num_epochs = 3\n", + "\n", + "train_losses, val_losses, tokens_seen = train_model_simple(\n", + " model, train_loader, val_loader, optimizer, device,\n", + " num_epochs=num_epochs, eval_freq=5, eval_iter=5,\n", + " start_context=format_input(val_data[0]), tokenizer=tokenizer\n", + ")\n", + "\n", + "end_time = time.time()\n", + "execution_time_minutes = (end_time - start_time) / 60\n", + "print(f\"Training completed in {execution_time_minutes:.2f} minutes.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "1Vdh7jmHI1we", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 307 + }, + "id": "1Vdh7jmHI1we", + "outputId": "97990a7a-605b-4634-9c6f-085d800eed71" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAEiCAYAAAA21pHjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABg7klEQVR4nO3deVxU1fsH8M/MwGzs+w6yCYhsohBiqUmhmYpZmvl1yS0LM7+Wmt/K9VdamlpplplSWWlqmuUW7om4g4IgiSKgsqjIDjMwc35/XBkdWWQZmAGe9+t1XzD3nnvvc7jMPHPuOfdeHmOMgRBCCCE6ia/tAAghhBBSP0rUhBBCiA6jRE0IIYToMErUhBBCiA6jRE0IIYToMErUhBBCiA6jRE0IIYToMErUhBBCiA6jRE0IIYToMErUhHQgN27cAI/HQ2JiorZDIYRoCCVqQnQMj8drcFq4cKG2QySEtCE9bQdACFGXk5Oj+n3r1q2YP38+0tLSVPMMDQ21ERYhREuoRU2IjrG1tVVNJiYm4PF4qtfW1tZYuXIlHB0dIRKJEBgYiP3799e7LYVCgYkTJ8Lb2xtZWVkAgD/++AM9evSAWCyGm5sbFi1ahOrqatU6PB4PGzZswPDhwyGVSuHp6Yndu3erlt+/fx9jxoyBlZUVJBIJPD09sWnTpnpj2L59O/z8/CCRSGBhYYGIiAiUlZWplm/YsAE+Pj4Qi8Xw9vbG119/rbZ+dnY2Ro4cCVNTU5ibm2PYsGG4ceOGavmECRMQFRWFFStWwM7ODhYWFoiOjkZVVVWj/+aE6DRGCNFZmzZtYiYmJqrXK1euZMbGxuzXX39lV65cYXPmzGH6+vrs33//ZYwxlpGRwQCwhIQEVllZyYYPH86CgoJYfn4+Y4yx48ePM2NjYxYTE8OuXbvG/v77b9alSxe2cOFC1T4AMEdHR/bLL7+wq1evshkzZjBDQ0N27949xhhj0dHRLDAwkJ09e5ZlZGSw2NhYtnv37jrjv337NtPT02MrV65kGRkZ7NKlS2zt2rWspKSEMcbY5s2bmZ2dHduxYwe7fv0627FjBzM3N2cxMTGMMcbkcjnz8fFhEydOZJcuXWIpKSnstddeY15eXkwmkzHGGBs/fjwzNjZm06ZNY6mpqezPP/9kUqmUrV+/XrMHgxAtoURNiA57PFHb29uzjz/+WK1Mr1692FtvvcUYe5io//nnHzZgwADWp08fVlhYqCo7YMAA9sknn6it/9NPPzE7OzvVawDsww8/VL0uLS1lANi+ffsYY4wNGTKEvf76642K//z58wwAu3HjRp3L3d3d2S+//KI2b8mSJSwsLEwVm5eXF1MqlarlMpmMSSQSduDAAcYYl6hdXFxYdXW1qswrr7zCRo0a1agYCdF11EdNSDtRXFyM27dvIzw8XG1+eHg4Ll68qDZv9OjRcHR0xOHDhyGRSFTzL168iLi4OHz88ceqeQqFApWVlSgvL4dUKgUA+Pv7q5YbGBjA2NgY+fn5AIA333wTI0aMwIULF/D8888jKioKvXv3rjPmgIAADBgwAH5+foiMjMTzzz+Pl19+GWZmZigrK8O1a9cwadIkTJkyRbVOdXU1TExMVPGmp6fDyMhIbbuVlZW4du2a6rWvry8EAoHqtZ2dHZKSkhr4axLSflCiJqQDeuGFF7B582bEx8fj2WefVc0vLS3FokWL8NJLL9VaRywWq37X19dXW8bj8aBUKgEAgwYNQmZmJvbu3YvY2FgMGDAA0dHRWLFiRa1tCgQCxMbG4uTJk/j777/x1Vdf4YMPPsDp06dVXwq+++47hIaG1lqvJt7g4GD8/PPPtbZtZWXVqHgJae8oURPSThgbG8Pe3h5xcXHo27evan5cXBxCQkLUyr755pvo3r07hg4dij179qjK9+jRA2lpafDw8GhRLFZWVhg/fjzGjx+Pp59+GrNnz64zUQNc0gwPD0d4eDjmz58PFxcX7Ny5E7NmzYK9vT2uX7+OMWPG1Llujx49sHXrVlhbW8PY2LhFMRPSXlGiJqQdmT17NhYsWAB3d3cEBgZi06ZNSExMrLPF+fbbb0OhUODFF1/Evn370KdPH8yfPx8vvvginJ2d8fLLL4PP5+PixYtITk7G//3f/zUqhvnz5yM4OBi+vr6QyWT466+/4OPjU2fZ06dP49ChQ3j++edhbW2N06dP486dO6ryixYtwowZM2BiYoKBAwdCJpPh3LlzuH//PmbNmoUxY8Zg+fLlGDZsGBYvXgxHR0dkZmbi999/x5w5c+Do6Nj8PyYh7QQlakLakRkzZqCoqAjvvvsu8vPz0a1bN+zevRuenp51lp85cyaUSiVeeOEF7N+/H5GRkfjrr7+wePFifPrpp9DX14e3tzcmT57c6BiEQiHmzZuHGzduQCKR4Omnn8aWLVvqLGtsbIzjx49j9erVKC4uhouLCz7//HMMGjQIADB58mRIpVIsX74cs2fPhoGBAfz8/DBz5kwAgFQqxfHjxzF37ly89NJLKCkpgYODAwYMGEAtbNJp8BhjTNtBEEIIIaRudMMTQgghRIdRoiaEEEJ0GCVqQgghRIdRoiaEEEJ0GCVqQgghRIdRoiaEEEJ0GCXqZlq7di26dOkCsViM0NBQnDlzps32vXTpUvTq1QtGRkawtrZGVFSU2vOKAaBfv37g8Xhq07Rp09TKZGVlYfDgwZBKpbC2tsbs2bPVHncIAEePHkWPHj0gEong4eGBmJiYWvE092+xcOHCWjF6e3urlldWViI6OhoWFhYwNDTEiBEjkJeXp1N1qNGlS5dadeHxeIiOjgagu8fj+PHjGDJkCOzt7cHj8bBr1y615YwxzJ8/H3Z2dpBIJIiIiMDVq1fVyhQUFGDMmDEwNjaGqakpJk2ahNLSUrUyly5dwtNPPw2xWAwnJyd89tlntWLZtm0bvL29IRaL4efnh7179zYplobqUlVVhblz58LPzw8GBgawt7fHuHHjcPv2bbV91HUcly1b1qZ1edIxmTBhQq0YBw4cqHPH5En1qOv9wuPxsHz5cp06HjpBiw8Eabe2bNnChEIh27hxI7t8+TKbMmUKMzU1ZXl5eW2y/8jISLZp0yaWnJzMEhMT2QsvvMCcnZ1ZaWmpqkzfvn3ZlClTWE5OjmoqKipSLa+urmbdu3dnERERLCEhge3du5dZWlqyefPmqcpcv36dSaVSNmvWLJaSksK++uorJhAI2P79+zXyt1iwYAHz9fVVi/HOnTuq5dOmTWNOTk7s0KFD7Ny5c+ypp55ivXv31qk61MjPz1erR2xsLAPAjhw5otPHY+/eveyDDz5gv//+OwPAdu7cqbZ82bJlzMTEhO3atYtdvHiRDR06lLm6urKKigpVmYEDB7KAgAB26tQp9s8//zAPDw82evRo1fKioiJmY2PDxowZw5KTk9mvv/7KJBIJ+/bbb1Vl4uLimEAgYJ999hlLSUlhH374IdPX12dJSUmNjqWhuhQWFrKIiAi2detWduXKFRYfH89CQkJYcHCwWn1dXFzY4sWL1Y7To++rtqjLk47J+PHj2cCBA9ViLCgoUCujC8fkSfV4NP6cnBy2ceNGxuPx2LVr13TqeOgCStTNEBISwqKjo1WvFQoFs7e3Z0uXLtVKPPn5+QwAO3bsmGpe37592TvvvFPvOnv37mV8Pp/l5uaq5q1bt44ZGxurnvM7Z84c5uvrq7beqFGjWGRkpOp1S/4WCxYsYAEBAXUuKywsZPr6+mzbtm2qeampqQwAi4+P15k61Oedd95h7u7uqscztofj8fiHqVKpZLa2tmz58uWqeYWFhUwkErFff/2VMcZYSkoKA8DOnj2rKrNv3z7G4/HYrVu3GGOMff3118zMzExVD8YYmzt3LvPy8lK9HjlyJBs8eLBaPKGhoeyNN95odCwN1aUuZ86cYQBYZmamap6LiwtbtWpVveu0dV3qS9TDhg2rN0ZdPCaNOR7Dhg1jzz77rNo8XTse2kKnvptILpfj/PnziIiIUM3j8/mIiIhAfHy8VmIqKioCAJibm6vN//nnn2FpaYnu3btj3rx5KC8vVy2Lj4+Hn58fbGxsVPMiIyNRXFyMy5cvq8o8Ws+aMjX11MTf4urVq7C3t4ebmxvGjBmDrKwsAMD58+dRVVWltm1vb284Ozurtq0rdXicXC7H5s2bMXHiRPB4PNX89nA8HpWRkYHc3Fy17ZmYmCA0NFTtGJiamqJnz56qMhEREeDz+Th9+rSqzDPPPAOhUKgWd1paGu7fv9+oujUmlqYqKioCj8eDqamp2vxly5bBwsICQUFBWL58uVr3g67U5ejRo7C2toaXlxfefPNN3Lt3Ty3G9nZM8vLysGfPHkyaNKnWsvZwPFob3eu7ie7evQuFQqH2gQoANjY2uHLlSpvHo1QqMXPmTISHh6N79+6q+a+99hpcXFxgb2+PS5cuYe7cuUhLS8Pvv/8OAMjNza2zDjXLGipTXFyMiooK3L9/v0V/i9DQUMTExMDLyws5OTlYtGgRnn76aSQnJyM3NxdCobDWh6iNjc0T42vLOtRl165dKCwsxIQJE1Tz2sPxeFzNfuva3qMxWVtbqy3X09ODubm5WhlXV9d662ZmZlZv3R7dxpNiaYrKykrMnTsXo0ePVrtn+IwZM9CjRw+Ym5vj5MmTmDdvHnJycrBy5UqdqcvAgQPx0ksvwdXVFdeuXcP//vc/DBo0CPHx8RAIBO3ymPzwww8wMjKq9fjV9nA82gIl6nYuOjoaycnJOHHihNr8qVOnqn738/ODnZ0dBgwYgGvXrsHd3b2tw6xTzYMZAMDf3x+hoaFwcXHBb7/9BolEosXIWub777/HoEGDYG9vr5rXHo5HZ1FVVYWRI0eCMYZ169apLZs1a5bqd39/fwiFQrzxxhtYunQpRCJRW4dap1dffVX1u5+fH/z9/eHu7o6jR49iwIABWoys+TZu3IgxY8aoPRMdaB/Hoy3Qqe8msrS0hEAgqDX6OC8vD7a2tm0ay/Tp0/HXX3/hyJEjT3zcX2hoKAAgPT0dAGBra1tnHWqWNVTG2NgYEolE438LU1NTdO3aFenp6bC1tYVcLkdhYWG929bFOmRmZuLgwYNPfBpVezgeNes0tD1bW1vk5+erLa+urkZBQYFGjtOjy58US2PUJOnMzEzExsY+8QlcoaGhqK6uxo0bN3SuLjXc3NxgaWmp9r/Uno7JP//8g7S0tEY9wa09HI/WQIm6iYRCIYKDg3Ho0CHVPKVSiUOHDiEsLKxNYmCMYfr06di5cycOHz5c69RPXRITEwEAdnZ2AICwsDAkJSWpvaFrPri6deumKvNoPWvK1NRT03+L0tJSXLt2DXZ2dggODoa+vr7attPS0pCVlaXati7WYdOmTbC2tsbgwYMbLNcejoerqytsbW3VtldcXIzTp0+rHYPCwkKcP39eVebw4cNQKpWqLyNhYWE4fvw4qqqq1OL28vKCmZlZo+rWmFiepCZJX716FQcPHoSFhcUT10lMTASfz1edStaVujzq5s2buHfvntr/Uns5JgB3Bio4OBgBAQFPLNsejker0PZotvZoy5YtTCQSsZiYGJaSksKmTp3KTE1N1UbstqY333yTmZiYsKNHj6pdtlBeXs4YYyw9PZ0tXryYnTt3jmVkZLA//viDubm5sWeeeUa1jZrLgZ5//nmWmJjI9u/fz6ysrOq8HGj27NksNTWVrV27ts7LgZr7t3j33XfZ0aNHWUZGBouLi2MRERHM0tKS5efnM8a4y7OcnZ3Z4cOH2blz51hYWBgLCwvTqTo8SqFQMGdnZzZ37ly1+bp8PEpKSlhCQgJLSEhgANjKlStZQkKCaiT0smXLmKmpKfvjjz/YpUuX2LBhw+q8PCsoKIidPn2anThxgnl6eqpdClRYWMhsbGzY2LFjWXJyMtuyZQuTSqW1LqHR09NjK1asYKmpqWzBggV1XkLTUCwN1UUul7OhQ4cyR0dHlpiYqPa+qRkxfPLkSbZq1SqWmJjIrl27xjZv3sysrKzYuHHj2rQuDdWjpKSEvffeeyw+Pp5lZGSwgwcPsh49ejBPT09WWVmpU8fkSf9bjHGXV0mlUrZu3bpa/5u6cjx0ASXqZvrqq6+Ys7MzEwqFLCQkhJ06darN9g2gzmnTpk2MMcaysrLYM888w8zNzZlIJGIeHh5s9uzZatftMsbYjRs32KBBg5hEImGWlpbs3XffZVVVVWpljhw5wgIDA5lQKGRubm6qfTyquX+LUaNGMTs7OyYUCpmDgwMbNWoUS09PVy2vqKhgb731FjMzM2NSqZQNHz6c5eTk6FQdHnXgwAEGgKWlpanN1+XjceTIkTr/l8aPH88Y4y5d+eijj5iNjQ0TiURswIABtep37949Nnr0aGZoaMiMjY3Z66+/zkpKStTKXLx4kfXp04eJRCLm4ODAli1bViuW3377jXXt2pUJhULm6+vL9uzZo7b8SbE0VJeMjIx63zc117qfP3+ehYaGMhMTEyYWi5mPjw/75JNP1BJgW9SloXqUl5ez559/nllZWTF9fX3m4uLCpkyZUuuLmC4ckyf9bzHG2LfffsskEgkrLCystW9dOR66gMcYY63aZCeEEEJIs1EfNSGEEKLDKFETQgghOowSNSGEEKLDKFETQgghOowSNSGEEKLDKFETQgghOowSdTPJZDIsXLgQMplM26G0WEepS0epB9Bx6kL10D0dpS4dpR6NQddRN1NxcTFMTExQVFT0xPsF67qOUpeOUg+g49SF6qF7OkpdOko9GoNa1IQQQogOo0RNCCGE6LBO9zzq6upqJCQkwMbGBnx+87+nlJSUAABu3bqF4uJiTYWnFR2lLh2lHkDHqQvVQ/d0lLq093oolUrk5eUhKCgIenoNp+JO10d99uxZhISEaDsMQgghBGfOnEGvXr0aLNPpWtQ2NjYAuD9OzfNbCSGEkLaUk5ODkJAQVU5qSKdL1DWnu+3s7ODo6KjlaAghhHRmjemCpcFkhBBCiA6jRE0IIYToMErUhBBCiA7rdH3UhBDSEIVCgaqqKm2HQdo5fX19CAQCjWyLEnULJN0sQl5xJbo7mMDWRKztcAghLcAYQ25uLgoLC7UdCukgTE1NYWtrCx6P16LtUKJugf/bk4LTGQX4anQQhgTYazscQkgL1CRpa2trSKXSFn+4ks6LMYby8nLk5+cDQIsvBaZE3QKD5fsRpXcRkpzxQMBwbYdDCGkmhUKhStIWFhbaDod0ABKJBACQn58Pa2vrFp0Gp8FkLRAgO4/RekcgLkjTdiiEkBao6ZOWSqVajoR0JDX/Ty0d80CJugWU+gYAACYr1XIkhBBNoNPdRJM09f9EiboFlEJD7hd5iXYDIYQQ0mFRom4BJjQCAPDkZVqOhBBCNKdLly5YvXp1o8sfPXoUPB6v1UfMx8TEwNTUtFX3oYu0mqiXLl2KXr16wcjICNbW1oiKikJaWsP9vTExMeDxeGqTWKydS6N4Iq5FLaiiFjUhpO09/ln4+LRw4cJmbffs2bOYOnVqo8v37t0bOTk5MDExadb+SMO0Our72LFjiI6ORq9evVBdXY3//e9/eP7555GSkgIDA4N61zM2NlZL6NrqV+KJuBa1oIpa1ISQtpeTk6P6fevWrZg/f77aZ6OhoaHqd8YYFArFE599DABWVlZNikMoFMLW1rZJ65DG02qLev/+/ZgwYQJ8fX0REBCAmJgYZGVl4fz58w2ux+PxYGtrq5oa85iw1qAn4RK1vqJcK/snhHRuj34OmpiYqH02XrlyBUZGRti3bx+Cg4MhEolw4sQJXLt2DcOGDYONjQ0MDQ3Rq1cvHDx4UG27j5/65vF42LBhA4YPHw6pVApPT0/s3r1btfzxU981p6gPHDgAHx8fGBoaYuDAgWpfLKqrqzFjxgyYmprCwsICc+fOxfjx4xEVFdWkv8G6devg7u4OoVAILy8v/PTTT6pljDEsXLgQzs7OEIlEsLe3x4wZM1TLv/76a3h6ekIsFsPGxgYvv/xyk/bdVnSqj7qoqAgAYG5u3mC50tJSuLi4wMnJCcOGDcPly5frLSuTyVBcXKyaSko0d5paX8Kd5hFSoiakw2GMoVxerZWJMaaxerz//vtYtmwZUlNT4e/vj9LSUrzwwgs4dOgQEhISMHDgQAwZMgRZWVkNbmfRokUYOXIkLl26hBdeeAFjxoxBQUFBveXLy8uxYsUK/PTTTzh+/DiysrLw3nvvqZZ/+umn+Pnnn7Fp0ybExcWhuLgYu3btalLddu7ciXfeeQfvvvsukpOT8cYbb+D111/HkSNHAAA7duzAqlWr8O233+Lq1avYtWsX/Pz8AADnzp3DjBkzsHjxYqSlpWH//v145plnmrT/tqIzNzxRKpWYOXMmwsPD0b1793rLeXl5YePGjfD390dRURFWrFiB3r174/Lly3U+X3rp0qVYtGhRq8Ssb8AlapGSEjUhHU1FlQLd5h/Qyr5TFkdCKtTMx/PixYvx3HPPqV6bm5sjICBA9XrJkiXYuXMndu/ejenTp9e7nQkTJmD06NEAgE8++QRffvklzpw5g4EDB9ZZvqqqCt988w3c3d0BANOnT8fixYtVy7/66ivMmzcPw4dzN4tas2YN9u7d26S6rVixAhMmTMBbb70FAJg1axZOnTqFFStWoH///sjKyoKtrS0iIiKgr68PZ2dnhISEAACysrJgYGCAF198EUZGRnBxcUFQUFCT9t9WdKZFHR0djeTkZGzZsqXBcmFhYRg3bhwCAwPRt29f/P7777CyssK3335bZ/l58+ahqKhINaWkpGgsZpEBd+pbwihRE0J0U8+ePdVel5aW4r333oOPjw9MTU1haGiI1NTUJ7ao/f39Vb8bGBjA2NhYdYvMukilUlWSBrjbaNaULyoqQl5enippAoBAIEBwcHCT6paamorw8HC1eeHh4UhNTQUAvPLKK6ioqICbmxumTJmCnTt3orq6GgDw3HPPwcXFBW5ubhg7dix+/vlnlJfr5me5TrSop0+fjr/++gvHjx+vs1XcEH19fQQFBSE9Pb3O5SKRCCKRSPW6uLi4RbE+SmxgCgCQsAowxuhmCYR0IBJ9AVIWR2pt35ry+MDc9957D7GxsVixYgU8PDwgkUjw8ssvQy6XN7gdfX19tdc8Hg9KpbJJ5TV5Sr8xnJyckJaWhoMHDyI2NhZvvfUWli9fjmPHjsHIyAgXLlzA0aNH8ffff2P+/PlYuHAhzp49q3OXgGm1Rc0Yw/Tp07Fz504cPnwYrq6uTd6GQqFAUlJSi2963hxSI1MAgCEqIKuu/x+WENL+8Hg8SIV6Wpla80t/XFwcJkyYgOHDh8PPzw+2tra4ceNGq+2vLiYmJrCxscHZs2dV8xQKBS5cuNCk7fj4+CAuLk5tXlxcHLp166Z6LZFIMGTIEHz55Zc4evQo4uPjkZSUBADQ09NDREQEPvvsM1y6dAk3btzA4cOHW1Cz1qHVFnV0dDR++eUX/PHHHzAyMkJubi4A7iDW3NB83LhxcHBwwNKlSwFw/S1PPfUUPDw8UFhYiOXLlyMzMxOTJ09u8/glhmaIV3RDKcQILK+E2KT+S8oIIUQXeHp64vfff8eQIUPA4/Hw0UcfNdgybi1vv/02li5dCg8PD3h7e+Orr77C/fv3m/QlZfbs2Rg5ciSCgoIQERGBP//8E7///rtqFHtMTAwUCgVCQ0MhlUqxefNmSCQSuLi44K+//sL169fxzDPPwMzMDHv37oVSqYSXl1drVbnZtJqo161bBwDo16+f2vxNmzZhwoQJALgOfz7/YcP//v37mDJlCnJzc2FmZobg4GCcPHlS7RtUW+FLjDGFvxClsmocqQKaduUhIYS0vZUrV2LixIno3bs3LC0tMXfuXI12CTbW3LlzkZubi3HjxkEgEGDq1KmIjIxs0lOmoqKi8MUXX2DFihV455134Orqik2bNqlyiqmpKZYtW4ZZs2ZBoVDAz88Pf/75JywsLGBqaorff/8dCxcuRGVlJTw9PfHrr7/C19e3lWrcfDzW1p0GWnbz5k04OTkhOzu7yf3hdXnqk0PILa7En9P7wM+R7spDSHtUWVmJjIwMuLq6au1Oh52dUqmEj48PRo4ciSVLlmg7HI1o6P+qKblIJwaTtWeGYj2gGCitbNljzAghpDPJzMzE33//jb59+0Imk2HNmjXIyMjAa6+9pu3QdI7OXJ7VXn1d/h7SROOhd+u0tkMhhJB2g8/nIyYmBr169UJ4eDiSkpJw8OBB+Pj4aDs0nUMt6hYS8pQQ8aogr2j7Ph5CCGmvnJycao3YJnWjFnULfWe3COGVX+CGcc8nFyaEEEKaiFrULVRp6IRb4KFYTt95CCGEaB5llxYyEnPfdUplNJiMEEKI5lGibiH/8nj8T+9n2OUd13YohBBCOiA69d1CbmWJeElvD44UGWs7FEIIIR0QtahbiCcyBADwq0q1HAkhhJCOiBJ1C/FF3KMu9ShRE0LaqX79+mHmzJmq1126dMHq1asbXIfH42HXrl0t3remttOQhQsXIjAwsFX30ZooUbeQQMwlan2Fbj7HlBDScQ0ZMgQDBw6sc9k///wDHo+HS5cuNXm7Z8+exdSpU1sanpr6kmVOTg4GDRqk0X11NJSoW0hPyt3fW6go03IkhJDOZtKkSYiNjcXNmzdrLdu0aRN69uwJf3//Jm/XysoKUqlUEyE+ka2tLUQiUZvsq72iRN1CQik3iEykrNByJISQzubFF1+ElZUVYmJi1OaXlpZi27ZtmDRpEu7du4fRo0fDwcEBUqkUfn5++PXXXxvc7uOnvq9evYpnnnkGYrEY3bp1Q2xsbK115s6di65du0IqlcLNzQ0fffQRqqq4y1ZjYmKwaNEiXLx4ETweDzweTxXz46e+k5KS8Oyzz0IikcDCwgJTp05FaenDrsUJEyYgKioKK1asgJ2dHSwsLBAdHa3aV2MolUosXrwYjo6OEIlECAwMxP79+1XL5XI5pk+fDjs7O4jFYri4uKgetcwYw8KFC+Hs7AyRSAR7e3vMmDGj0ftuDhr13UIiA65FLVbSqW9COiR5M86WCUSA4MHHq6IaUMgAHh/Qlzx5u8LGP9deT08P48aNQ0xMDD744APVs5y3bdsGhUKB0aNHo7S0FMHBwZg7dy6MjY2xZ88ejB07Fu7u7ggJCXniPpRKJV566SXY2Njg9OnTKCoqUuvPrmFkZISYmBjY29sjKSkJU6ZMgZGREebMmYNRo0YhOTkZ+/fvVz0r2sSk9tMGy8rKEBkZibCwMJw9exb5+fmYPHkypk+frvZl5MiRI7Czs8ORI0eQnp6OUaNGITAwEFOmTGnU3+2LL77A559/jm+//RZBQUHYuHEjhg4disuXL8PT0xNffvkldu/ejd9++w3Ozs7Izs5GdnY2AGDHjh1YtWoVtmzZAl9fX+Tm5uLixYuN2m9zUaJuIbEh16KWogJKJQOf3/iHnhNC2oFP7Ju+zisxgO9w7vcrfwLbJgAufYDX9zwss9oPKL9Xe92FRU3a1cSJE7F8+XIcO3ZM9RzmTZs2YcSIETAxMYGJiQnee+89Vfm3334bBw4cwG+//daoRH3w4EFcuXIFBw4cgL0997f45JNPavUrf/jhh6rfu3Tpgvfeew9btmzBnDlzIJFIYGhoCD09Pdja2ta7r19++QWVlZX48ccfYWDAfWFZs2YNhgwZgk8//RQ2NjYAADMzM6xZswYCgQDe3t4YPHgwDh061OhEvWLFCsydOxevvvoqAODTTz/FkSNHsHr1aqxduxZZWVnw9PREnz59wOPx4OLiolo3KysLtra2iIiIgL6+PpydnRv1d2wJOvXdQlJDUwCAASpRJq/WbjCEkE7H29sbvXv3xsaNGwEA6enp+OeffzBp0iQAgEKhwJIlS+Dn5wdzc3MYGhriwIEDyMrKatT2U1NT4eTkpErSABAWFlar3NatWxEeHg5bW1sYGhriww8/bPQ+Ht1XQECAKkkDQHh4OJRKJdLS0lTzfH19IRAIVK/t7OyQn5/fqH0UFxfj9u3bCA8PV5sfHh6O1NRUANzp9cTERHh5eWHGjBn4+++/VeVeeeUVVFRUwM3NDVOmTMHOnTtRXd26n/3Uom6hmj5qQ14lcirlMBLrazkiQohG/e9209cRPDI4ynsItw3eY+2imUkti+sRkyZNwttvv421a9di06ZNcHd3R9++fQEAy5cvxxdffIHVq1fDz88PBgYGmDlzJuRyucb2Hx8fjzFjxmDRokWIjIyEiYkJtmzZgs8//1xj+3iUvr765yyPx4NSqdTY9nv06IGMjAzs27cPBw8exMiRIxEREYHt27fDyckJaWlpOHjwIGJjY/HWW2+pzmg8HpemUIu6hXgPrqMGgPLSpp2yIoS0A0KDpk+CR9pAAj1u3qP90w1ttxlGjhwJPp+PX375BT/++CMmTpyo6q+Oi4vDsGHD8J///AcBAQFwc3PDv//+2+ht+/j4IDs7Gzk5Oap5p06dUitz8uRJuLi44IMPPkDPnj3h6emJzMxM9eoKhVAoFE/c18WLF1FW9rD/Pi4uDnw+H15eXo2OuSHGxsawt7ev9YjNuLg4dOvWTa3cqFGj8N1332Hr1q3YsWMHCgoKAAASiQRDhgzBl19+iaNHjyI+Ph5JSZr74vU4alG3lJ4Y1RBADwpUlBQBaEZ/FiGEtIChoSFGjRqFefPmobi4GBMmTFAt8/T0xPbt23Hy5EmYmZlh5cqVyMvLU0tKDYmIiEDXrl0xfvx4LF++HMXFxfjggw/Uynh6eiIrKwtbtmxBr169sGfPHuzcuVOtTJcuXZCRkYHExEQ4OjrCyMio1mVZY8aMwYIFCzB+/HgsXLgQd+7cwdtvv42xY8eq+qc1Yfbs2ViwYAHc3d0RGBiITZs2ITExET///DMAYOXKlbCzs0NQUBD4fD62bdsGW1tbmJqaIiYmBgqFAqGhoZBKpdi8eTMkEolaP7amUYu6pXg8pAi8cVrpjTI5PUGLEKIdkyZNwv379xEZGanWn/zhhx+iR48eiIyMRL9+/WBra4uoqKhGb5fP52Pnzp2oqKhASEgIJk+ejI8//litzNChQ/Hf//4X06dPR2BgIE6ePImPPvpIrcyIESMwcOBA9O/fH1ZWVnVeIiaVSnHgwAEUFBSgV69eePnllzFgwACsWbOmaX+MJ5gxYwZmzZqFd999F35+fti/fz92794NT09PANwI9s8++ww9e/ZEr169cOPGDezduxd8Ph+mpqb47rvvEB4eDn9/fxw8eBB//vknLCwsNBrjo3iMMdZqW9dBN2/ehJOTE7Kzs+Ho6KiRbb687iTOZd7HN//pgYHd7TSyTUJI26msrERGRgZcXV0hFou1HQ7pIBr6v2pKLqIWtQYYPngmdUkljfomhBCiWZSoNcBAxCXqUhklakIIIZpFiVoDXr/zGc6K3oRd1p4nFyaEEEKaQKuJeunSpejVqxeMjIxgbW2NqKgotYva67Nt2zZ4e3tDLBbDz88Pe/fubYNo62fAKmDFKwJPdl+rcRBCCOl4tJqojx07hujoaJw6dQqxsbGoqqrC888/r3YN3eNOnjyJ0aNHY9KkSUhISEBUVBSioqKQnJzchpGrO+X2NgbKluGMwbNai4EQQkjHpNXrqB99WgnAPWHF2toa58+fxzPPPFPnOl988QUGDhyI2bNnAwCWLFmC2NhYrFmzBt98802rx1wXhZkbrjAZvBRt81g4Qkjr0OTdrQjR1P+TTt3wpKiIu7OXubl5vWXi4+Mxa9YstXmRkZFqj0lra4YPBpOV0WAyQtoloVAIPp+P27dvw8rKCkKhUHVnL0KaijEGuVyOO3fugM/nQygUtmh7OpOolUolZs6cifDwcHTv3r3ecrm5ubXuUGNjY4Pc3Nw6y8tkMshkMtXrkpISzQT8CIeKK5gu2AnBfU8AvTS+fUJI6+Lz+XB1dUVOTg5u327Gvb0JqYNUKoWzszP4/Jb1MutMoo6OjkZycjJOnDih0e0uXboUixYt0ug2H2dTnIz39Lchrqw3gLmtui9CSOsQCoVwdnZGdXX1E+9JTciTCAQC6OnpaeTMjE4k6unTp+Ovv/7C8ePHn3iHFltbW+Tl5anNy8vLq/cZp/PmzVM7VX7r1q1G3+O2sfQl3IM5hIpmPGCeEKIzeDwe9PX1W+0pSIQ0h1ZHfTPGMH36dOzcuROHDx+Gq6vrE9cJCwvDoUOH1ObFxsbW+XxUABCJRDA2NlZNRkZGdZZrCX0DE25fygqNb5sQQkjnptUWdXR0NH755Rf88ccfMDIyUvUzm5iYQCLhHgk3btw4ODg4YOnSpQCAd955B3379sXnn3+OwYMHY8uWLTh37hzWr1+vtXoIpVyiFrNyrcVACCGkY9Jqi3rdunUoKipCv379YGdnp5q2bt2qKpOVlaX2HNTevXvjl19+wfr16xEQEIDt27dj165dDQ5Aa20SQy5RS1GJKgVd3kEIIURztNqibsyDu44ePVpr3iuvvIJXXnmlFSJqHvGDU9+GqECZrBqm0pYNxSeEEEJq0L2+NUBfYgwAMEAlSiromdSEEEI0hxK1JogMAQD6PAXKyku1HAwhhJCOhBK1JggNVb9WlBZpMRBCCCEdDSVqTeALUAExAKCyjBI1IYQQzaFErSGVfO5yMnlZsZYjIYQQ0pFQotYQGZ97clZVBSVqQgghmqMTtxDtCO6Iu+B+sQDl9AAtQgghGkQtag3Z6vEpBsmX4ZrIV9uhEEII6UAoUWuIwYNnUpfSM6kJIYRoECVqDTGqSdSVlKgJIYRoDiVqDQm7/QMOCd9Fr7xftR0KIYSQDoQStYYYKkvgzs+BQWW+tkMhhBDSgdCobw3J83wVH6U6wczSA4O0HQwhhJAOgxK1hvAtPXGGFcC72kjboRBCCOlA6NS3hhiKue88JTSYjBBCiAZRi1pDTGW3MV5wAFWVxgCe1XY4hBBCOghqUWuIcWkGFun/gNeUf4Ixpu1wCCGEdBCUqDVEbGgCADBABWTVSi1HQwghpKOgRK0hYqkxAMCQV0n91IQQQjSGErWG8MXcaG8DVNJtRAkhhGgMJWpNEXEtailPhrIKmZaDIYQQ0lFQotYUkaHq1/LSIi0GQgghpCNpVqLOzs7GzZs3Va/PnDmDmTNnYv369RoLrN3RE6HqwdVulWWUqAkhhGhGsxL1a6+9hiNHjgAAcnNz8dxzz+HMmTP44IMPsHjxYo0G2J5U8iUAgCpK1IQQQjSkWYk6OTkZISEhAIDffvsN3bt3x8mTJ/Hzzz8jJiam0ds5fvw4hgwZAnt7e/B4POzatavB8kePHgWPx6s15ebmNqcaGifnS7mfFcVajoQQQkhH0axEXVVVBZFIBAA4ePAghg4dCgDw9vZGTk5Oo7dTVlaGgIAArF27tkn7T0tLQ05OjmqytrZu0vqtRSYwAABUlZdoORJCCCEdRbNuIerr64tvvvkGgwcPRmxsLJYsWQIAuH37NiwsLBq9nUGDBmHQoKY/a8ra2hqmpqZNXq+1VesZADJAWUktakIIIZrRrBb1p59+im+//Rb9+vXD6NGjERAQAADYvXu36pR4awoMDISdnR2ee+45xMXFtfr+Gkuhx7WomaxUy5EQQgjpKJrVou7Xrx/u3r2L4uJimJmZqeZPnToVUqlUY8E9zs7ODt988w169uwJmUyGDRs2oF+/fjh9+jR69OhR5zoymQwy2cPrmktKWu+0dKXUBjcLLVFG9zshhBCiIc1K1BUVFWCMqZJ0ZmYmdu7cCR8fH0RGRmo0wEd5eXnBy8tL9bp37964du0aVq1ahZ9++qnOdZYuXYpFixa1WkyPOhewBB9lJCNSbIP/tMkeCSGEdHTNOvU9bNgw/PjjjwCAwsJChIaG4vPPP0dUVBTWrVun0QCfJCQkBOnp6fUunzdvHoqKilRTSkpKq8ViJOK+95TJFK22D0IIIZ1LsxL1hQsX8PTTTwMAtm/fDhsbG2RmZuLHH3/El19+qdEAnyQxMRF2dnb1LheJRDA2NlZNRkZGrRaL4YNEXUL3+iaEEKIhzTr1XV5erkp4f//9N1566SXw+Xw89dRTyMzMbPR2SktL1VrDGRkZSExMhLm5OZydnTFv3jzcunVL1XpfvXo1XF1d4evri8rKSmzYsAGHDx/G33//3ZxqaJxLzj7sEn6BlKIeAMK1HQ4hhJAOoFmJ2sPDA7t27cLw4cNx4MAB/Pe//wUA5Ofnw9jYuNHbOXfuHPr37696PWvWLADA+PHjERMTg5ycHGRlZamWy+VyvPvuu7h16xakUin8/f1x8OBBtW1ok4GyFJ7867hfZaXtUAghhHQQPMYYa+pK27dvx2uvvQaFQoFnn30WsbGxALiBW8ePH8e+ffs0Hqim3Lx5E05OTsjOzoajo6Nmt309FQs2bEOhvg12LHpDo9smhBDScTQlFzWrRf3yyy+jT58+yMnJUV1DDQADBgzA8OHDm7PJDkFs7YZDymDupidKBj6fp+2QCCGEtHPNStQAYGtrC1tbW9VTtBwdHdvkZie6rGYwGQCUyathJNbXYjSEEEI6gmaN+lYqlVi8eDFMTEzg4uICFxcXmJqaYsmSJVAqlZqOsd0QVRfjZb0TeIl/nC7RIoQQohHNalF/8MEH+P7777Fs2TKEh3Ojm0+cOIGFCxeisrISH3/8sUaDbC94ZXexQu9rFDEp7sg+BCDWdkiEEELauWYl6h9++AEbNmxQPTULAPz9/eHg4IC33nqr0yZqCA0BAAaoxPWKKi0HQwghpCNo1qnvgoICeHt715rv7e2NgoKCFgfVbom4RK3HU6K8vEzLwRBCCOkImpWoAwICsGbNmlrz16xZA39//xYH1W7pG6h+lZUVai8OQgghHUazTn1/9tlnGDx4MA4ePIiwsDAAQHx8PLKzs7F3716NBtiu8Pmo4EkgYRWoLKNnUhNCCGm5ZrWo+/bti3///RfDhw9HYWEhCgsL8dJLL+Hy5cv1PsWqs5DxuVZ1VTklakIIIS3X7Ouo7e3taw0au3jxIr7//nusX7++xYG1V3I9KaAAFBWUqAkhhLRcs1rUpH7VAq5FXV1ZouVICCGEdASUqDVM8WBAmZISNSGEEA2gRK1hypqR3/JS7QZCCCGkQ2hSH/VLL73U4PLCwsKWxNIhsAc3PeHLqUVNCCGk5ZqUqE1MTJ64fNy4cS0KqN17cNMTHrWoCSGEaECTEvWmTZtaK44Ogyc2RTGTQKZo8mO+CSGEkFqafXkWqdud0PfRN6EvHCQS/EfbwRBCCGn3aDCZhnlaG4HHA24VViCvuFLb4RBCCGnnKFFrmIlUH93tub78k9fuajkaQggh7R0lak3LPoMvqxZhid5GxKXf03Y0hBBC2jlK1JpWWQzX4jPowb+Kk+l3wRgNKiOEENJ8lKg1zbY75EPW4TPlGNwuqsSNe+XajogQQkg7Rola04xsIQx+DRVOzwAA4tKpn5oQQkjzUaJuJeHulgAYTqXnazsUQggh7ZhWE/Xx48cxZMgQ2Nvbg8fjYdeuXU9c5+jRo+jRowdEIhE8PDwQExPT6nE2xyBpCv4SfgCba79BqaR+akIIIc2j1URdVlaGgIAArF27tlHlMzIyMHjwYPTv3x+JiYmYOXMmJk+ejAMHDrRypE3nzstBd/4NvK78HVdu0elvQgghzaPVO5MNGjQIgwYNanT5b775Bq6urvj8888BAD4+Pjhx4gRWrVqFyMjI1gqzWQQ9J+D+wU/hqLiLf07EAKNnazskQggh7VC76qOOj49HRESE2rzIyEjEx8drKaIG6IuR6jYZAOCTvh6olms5IEIIIe1Ru0rUubm5sLGxUZtnY2OD4uJiVFRU1LmOTCZDcXGxaiopabvHT5o+PRl5zBSWinxUX9jcZvslhBDScbSrRN0cS5cuhYmJiWrq1q1bm+3b29EaMXzuGd6KY8upVU0IIaTJ2lWitrW1RV5entq8vLw8GBsbQyKR1LnOvHnzUFRUpJpSUlLaIlQAAJ/Pw223UchjphCV3QYSf26zfRNCCOkY2lWiDgsLw6FDh9TmxcbGIiwsrN51RCIRjI2NVZORkVFrh6kmpKsdvq4exr3453OgWtam+yeEENK+aTVRl5aWIjExEYmJiQC4y68SExORlZUFgGsNjxs3TlV+2rRpuH79OubMmYMrV67g66+/xm+//Yb//ve/2gi/UcLdLbFF0R95zBQoyga+fgq4uBVQKrQdGiGEkHZAq4n63LlzCAoKQlBQEABg1qxZCAoKwvz58wEAOTk5qqQNAK6urtizZw9iY2MREBCAzz//HBs2bNC5S7Me5WIhhYWJMeZWTYVcZAYUXAd2TuUS9t10bYdHCCFEx2n1Oup+/fo1+HSpuu461q9fPyQkJLRiVJrF4/HQ28MS288H4svuO/Ce6THg5JeArBQwcXxYsKoC0K+7n50QQkjn1a76qNurcA8LAMCxGxXA07OAdy4Br/4M6Iu5AopqYLUfsOkFoCRXi5ESQgjRNVptUXcWvd0tAQDJt4tQWC6HqdQYcOjxsEBOIlB2h+u3llo+nJ+yG5BaAE4hgEC/bYMmhBCiEyhRtwEbYzE8rA2Rnl+KvsuPItDJFIFOpghy5n6aOvbkWtn30gHBg0PCGLD/faD4FiAyAdz7A57PAR4RgJGtditECCGkzVCibiMTw12x6M/LKKqowrF/7+DYv3dUywZ1t8WqUYEQm7k8XEFeBriEA+kHgYoCIGUXNwGArR/g8SBpU2ubEEI6NB5raDRXB3Tz5k04OTkhOzsbjo6OT15Bg+TVSlzJLUZCViESs7kp424ZAKBvVyt8OzYYYn2B+kpKBXA7Abj6NzfdfmwgnciYa213iwK8BtGANEIIaQeakosoUWtZ/LV7mBhzFhVVCvT3ssI3Y4Mh0hPUv0LpHeDaYSA9Fkg/xLW2awiNgLfP0alxQgjRcU3JRTTqW8vC3C3w/YSeEOvzcSTtDqJ/vgB5tbL+FQytgIBRwIgNwOx0YPIhoM9/ARMnwMxFPUnvnAbsmKI+kjwvBbiyByi+zfWDE0II0WnUR60Dertb4vvxvTAx5iwOpuYj+pcLWPtaDwj1Gv4eVVSpxJeJUvxztT8+GvwmnrZ7LPFe3gVUVwDPfvhw3qUtQNwX3O8G1oB9IGAfBFh2BQxtuERvaAOIjAAeT6P1JIQQ0nSUqHVEuIclvhvXE5N/PIfYlDxM/+UClkR1h42xuFbZaoUSv57Nxsq/03C/vAoAMOWnC9g8KRQ9H72VeeTH3I1UpBYP5xlYAda+wJ0rQFn+w77vx+lLASM7wLEn0HUg0P0lDdeYEEJIY1AftY459u8dTPnxnOr0t6+9MQZ4W+NZHxv4O5gg/vo9LP4zBWl53HO1PawNYWEgxOmMAhiJ9bBl6lPwtTd58o7k5UDeZW5wWk4icD8TKM0FSvMBWbF6Wb9XuFPtAEor5dC/9AtEbn0AC3dqdRNCSDPQYLIG6HqiBoCT6Xfx2YE0XLxZqNaNbCzWQ3FlNQDARKKPWc91xWuhzqhWMIzbeBpnb9yHpaEQv70RBjcrw+YHIC8DSvOAggwgKx5w6Al4DUR2QTlmf70FW6r/CzlfjMIZV2FtasytU3YXEJvQpWKEENIIlKgb0B4SdY27pTIcTbuDw1fycPzfuyiVVUPA52HsUy6YGeEJU6lQVbaoogqj159CSk4xHEwl2DYtDPammrtUq0xWjRHrTkKYl4h5er+iFGJMZ3PwWqgzpvV1h82PzwD3rnKny40duPuYmzhyfd4Sc0Bi9shkCvD1AKEhoPegDkoFN7hNQL0xhJCOjxJ1A9pTon6UvFqJpFuFsDYSw8lcWmeZu6UyjPwmHtfvlsHNygDb3giDhaGoxftWKhmif7mAfcm5sDQU4aMXfRATl4GE7CIAgIVeBU7pvwl9Jm/ahl/aAPi/wv2e+hewdQzgFApMeqTP/Ldx3LXidgHcoDcbX7pWnBDS7jUlF1HzpZ0Q6vER7GLeYBlLQxF+mhyKV9adxPU7ZXj5m3i8P8gbz3ezAa8FfclrjqRjX3IuhAI+vh3bA8Eu5hgaYI+49Hv44tC/OHsD6Fq9EYFmcqwZbA0H3j3u1qeF2dw9zCvuc9d7V9znpkouwav3bz/4vigQqu/8xgmg/B6Q8NODdQSAlTfgEMTduc2lN2DqQn3lpH0oyQPykoC7V7krK0wcAWNHwMRB819Aq+XcY3XvpnEDSrv04eYrqoHCTO7slvSRz5SLW7my5QVAtQxQyLif1TKgupIrwxcAPD73PuTxAbe+QFg0t0yp4C79NLACHIIfni3TBkU195nAb+CeFO0Itag7oGt3SvHq+lO4UyIDAAQ6mWLOQC/Vw0Ga4sDlXLzx03kAwGcj/DGyl5PacsYY4q/fw9wdl5BdUAFTqT7Wj+2JENcGvlQolQBTcm90/oNL0BTV3CA2pYK7VrxGym4g5+KDKZFL/I8zduAStnMYd5mZfRAgakEfvaYxBiirdb//XqnkPpyrKrjjIDYG9Fp+RqbTUlRzNye6cRzITQbykuv+/60htQACRnNXawDc/82/BwBTJ+5MUg1ZKff+UVZzgz/L8h/8vMPdM+FeOnD3Xy5JK7kxLQib/nC7hdnA6u5cN9W7Vx5ud+MgIOtk0+oY8gbwwmfc76V3gBUe3O//uw0IDbjfL27hxryYu3FXk+iJH0wiblJUceNi5KUPpjLufVzz4KLiHO65B1XlwJhtD/d9ZClw6zz3t5CVcA0AWTFQWQxUcXd8hNCI+0IkMuL+n936A89+8HAbN04AxvaAmWubf9mnFnUn525liIOz+mL98WvYeOIGErML8dp3p/G0pyXmRHrDz7ERo8IBpOWWYNbWRADAhN5daiVp4MHztt0tsfOtcEz+4RwSswvxnw2n8dnL/ogKcqh7w3w+at1rR6Cn/u2+Rreh3ARwH1zFt7mEnX0GyDwJ3L7Atd6TtnETAEw9xl0fDgCXfgOStgM+Q4AeY7l58nKuhS614CYDS+6pZQaWLU+miiruQzI3SX2qKAAEIu7DQmzCnc4XG3Pzhn758EY1yTuAqwcBzwig+whuXnkBcHjJgw9nxYP+/Ac/eTxAbMr1+4tNH44BEIgAp17cvgCgMAvIvwIY2XDdCAD3Qf7DUO6DrbqSS9KP05Nw26jZvokDd4MdW7+W/Z06g9/GAml7H5vJAyw8AGtvLiEV3eSmqnLuzNGj7aayu8Cvo7izTB89kuB3TAL+3d+4GISGD5Je8MN58jIugZXd4b6c1XxZ9nkRsO3OvSdUyVTI/aw508UY979X87/46BeI6krAMeTB+BODh/Mv/AhkxjUu3hph0x8maqbknnOg99ilqrcTuDs0NkRewk0lD17bdH+4rLIYiBnM/f7RvYfjY3a/DVyN5epgaPPg/hJ23HvUyA7wjWrzL92UqDsoE4k+Zkd6Y3zvLlh7OB2/nMnCP1fv4p+rJxDpa4N3BnRFN3vjete/eb8cU348hzK5AmFuFvhgsE+D+7M0FGHL1Kfw362J2Jeci5lbE5F5rxwzBni06LS7Gh6PSxQmDoD3gzeYvBy4eZZL2jfPcAnJ5JEvFLfOA1cPcB+MNcruAPvm1L0PoRHXGhcacB9yQkPutZ4YiFgImLty5TJPctefO/Z6GMu9a8CantwHS10UMm7fj7eqqh9JkLcTgIu/cF8aahK1rAQ4t7FRfyI1U49yZxcA7kvMocVAz4nAi6u4eVILoCi74W1UVwClFdylewCQDeCp6IfLz/8AxK8FAl7lnrUOcB/mRTcffElgD5LPg598vfZzMx3GuOTJFwD6BtyHc31x37rA/a91H/HwC2fXSCD7NOAzlPviaOMHWPsAwsfGmDDGdQkV3+L+32rIirkvVeX31cvXtJIB7suToTV38yJDK+6nuRtg1RWw9OJai4/HbO0N/O+mepIGHp7Cbi5TJ2ByHYnT6wUuxsLsh6fRVafWKwG+/oP3XM37zgAw6/JwfakFMGh57S/yYW8B3YZx9RMZPfIF2ISbmPJBK7vkwVQMmDo/XL+yELDw5M5IPDqItfQOUJLD/X4vXX2fPMHD92UbokTdwVkbibFoWHdMftoNK2P/xa7EWzhwOQ8HLudhoK8t3onwhI8dl7ArqxQ4cDkX28/fxIn0u2AMcDKX4OsxPaAvePLdZsX6Aqx9rQc+3X8F3x6/jlUH/8WtwnJ8OsJfc8n6cUIp10/m1rfu5f4jAetu6t/8+XrcG7y8gGu1lN990JpRPvwGXpe+jyT3jH+AE6u4xFeTqKUW3DZExtw3d1u/B1N3rh9dXsp9i5cVcx8glcWAQq5+QxrPSK51/2gLSGwM9Jv3oG+QzyUOvh73ocEU3LYqCh/0/xdyvyvkXHKpIbXgYjG0eWS7JtwtaMWmgL74kVaUmNuPrJjb3qPbv38DsPJ6uI3cJK5fs+KRZFKax51arY/ImPt7mLlwH8imLtw1+U6h2umyYIz7P7iTCuSnAvkpD36mqt9TgCfgkoj+g0Q7K/VhojuxCkjdzR3/0De4eQGjgYDXntxXy+NxSejxRGThDrxxvHb50VsBZRUXT0v6gflPfk9rRO/pLVtfXwyETq09363fk9c1aKC7z9SZezbC472/LywH+s/jkntpHtedUDMpZG33d3sE9VF3Mun5JfjiUDr+unRb9f85qLstzAyE+PPibZRUPvy2Hupqjo+H+8HDuukfnj+fzsT8Py5DoWT47GV/jOxZ+7S5TlEqHiajmn4yedmDxF3G9dv6j+ROLQNA2j7g+lGub7zbMG4eY9wb29CmfbQYNaEkl+t7NXbgWosA17L8PpJLJk3xxj+AnT/3e8Y/3N3znMO4LzoAd2yuH+VOx4sMH/Y91pwF4es/GOz0yN++5p+8Zl7BdW4gl7kbYOnJzUs/CGxuRitpTsbD5Hp8BZBxHAj6D/d/QsgT0OVZDejsibrG1bwSfHlYPWEDgIOpBCOCHTGihwNcLAzq30AjfHPsGpbtuwIjsR4Ozupb5+1QSQelVHD99cCDJMnjfirk3CnQwkzubniFmVwr/V4613qsGbz2x3RuHEHEQq5PHOC6Bdb3e/K+eQ/OOAj0uVOss1K4U68AsHc2cGY90Pd9rtUEcHF8EcC18K19uVPD1t24Lx4WHtzZBXkZ149cVcH9zuNzZxZ0fYAg0Vk0mIw8kaeNEb4aHYS3n/XAprgMKJQMUYEOeMrNAny+ZlqDk/u4Yl9SDi7eLMIHO5Pw3bieTToFLqtWIDGrEIHOpg0/+pPoHr6g7ktjBPoPEqF37WWPsvUHvAZzLd8a+gaAc2+uX1NWwp35qPn5KKYAFIqHg+Mqix8malMXrgtA/Mj4DFNn4IOchi+PkphyEyFaQC1q0qrSckvw4lf/oErB8MWrgRgWWM9I8EeUVFbh1zNZ+P5EBvKKZXja0xKbJvSCXiP6yUknpFRyl+Moqh6Miq9+MCq+mmtZG9rSHe+IzqEWNdEZXrZGmPGsJz6P/RcLdl9Gb3dLWBnVfW3unRIZYk5m4Mf4TLW+8n+u3sX/7UnFwqG+da5HOjk+n+urJqSDokRNWt20fu7Yl5yLlJxiLNidjK/HBKstT88vxaa4DGw7f1P11DA3KwNMe8YdYqEAM35NQMzJG/C2NcKrIc517QJKJUPSrSJ42RpBrN+80+Rx6XdxOqMA9iZiOJtL4WQuhZ2JmFryhBCtokRNWp2+gI/lr/hj2Jo47E3Kxd6kHAzqbosT6Xfx/YkMHE17eF1xgJMp3uzrjue72aj6ym/cLcPK2H/x0R/JcLMyrHXXsxt3yzBnxyWcyShAVxtDfPOf4CY9PUyhZPj87zR8ffRarWV6fB4czSQYGuiAN/u6QyKkvnJCSNvSiabC2rVr0aVLF4jFYoSGhuLMmTP1lo2JiQGPx1ObxGIaTazrfO1N8GY/dwDAR7uSEbn6OMZ+fwZH0+6AxwOe62aDX6c8hV1v9cbA7rZqA9reftYDg/3tUKVgmLb5PLILygFwreiNJzIw8IvjOJNRAAD4N68Uw9bE4cDl3EbFVVRehYkxZ1VJOtLXBv28rOBmZQChgI9qJcONe+X48tBVRKw8hv3JOehkwzoIIVqm9Rb11q1bMWvWLHzzzTcIDQ3F6tWrERkZibS0NFhbW9e5jrGxMdLS0lSvW+1mGkSjpj/rgQOXc/FvXinulclhIBTglZ5OmNC7C7pY1n8pGI/Hw4qXA5B5rwzJt4ox5cdzWPFKABb9eRlnb3A32ghzs8DsgV5YtvcKztwowBs/nceb/dzx3vNeENQzij0ttwRTfzqHzHvlEOvz8ekIf7XBbkolQ15JJc5kFOCz/Wm4VViBaZsv4GlPSywc6gv3ljzzmxBCGknro75DQ0PRq1cvrFmzBgCgVCrh5OSEt99+G++//36t8jExMZg5cyYKCwubtT8a9a1d/+aV4LP9aQh1NcfIXk4wkTT+OtTbhRUYuiYOd0sf3nLTQCjAvBd88FqIM/h8HqoUSizbdwXfn8gAAPTxsMQXrwbCRKIPWbXywaTAmYwCzPs9CeVyBRxMJVg/Lhi+9vXfA71CrsDXR9Px7bHrkCuU0BfwMPlpN8x6rmuj7tpGCCGPajc3PJHL5ZBKpdi+fTuioqJU88ePH4/CwkL88ccftdaJiYnB5MmT4eDgAKVSiR49euCTTz6Br2/dI4JlMhlksocf7Ldu3UK3bt0oUbdTF7Lu49VvT0GuUKKPhyWWjfCDo1nt53Pvvngbc7dfQkWVosHthXtY4KvRPWBu0LhbMd64W4bFf6Xg8JV8AECEjw3WvBbU7AFshJDOqSmJWqtNgbt370KhUMDGxkZtvo2NDXJz6+5j9PLywsaNG/HHH39g8+bNUCqV6N27N27evFln+aVLl8LExEQ1devWTeP1IG2nh7MZtr8Zhg3jeuKnSSF1JmkAGBpgjz+mh8PdqvYpdaGAD1OpPqb1dccPr4c0OkkDQBdLA2yc0AtrX+sBoR4fB1PzMDHmLEpl1U9emRBCmkGrLerbt2/DwcEBJ0+eRFhYmGr+nDlzcOzYMZw+ffqJ26iqqoKPjw9Gjx6NJUuW1FpOLerOTalkuFsqg1CPD5GeACI9vsbuvBZ/7R4m/3AWZXIFAp1MEfN6L5hKW/CQBEJIp9FubnhiaWkJgUCAvLw8tfl5eXmwtbVt1Db09fURFBSE9PT0OpeLRCKIRA9vsFFcXFxnOdIx8fk8WLfSPcbD3C3w85SnMGHTGSRmF+LV9afw46QQWBtx+2OM4XZRJZJvFSG3qBISoQCGIj0YiPRgIBTAUKwHN0tDCPWoj5sQUj+tJmqhUIjg4GAcOnRI1UetVCpx6NAhTJ/euEejKRQKJCUl4YUXXmjFSAmpW6CTKbZODcPY70/jSm4JRn4Tj8jutki5XYzkW0W4X97wE6QcTCWY9VxXRAU51Ds6nRDSuWn98qxZs2Zh/Pjx6NmzJ0JCQrB69WqUlZXh9ddfBwCMGzcODg4OWLp0KQBg8eLFeOqpp+Dh4YHCwkIsX74cmZmZmDx5sjarQToxL1sjbJsWhjEbTuPGvXJ8e+y6apken4euNkZwNpeislqBcpkCpbJqlMmrca9UjluFFXh320WsP34dcwd5ob+XdYe+3JAx1qHrR0hr0HqiHjVqFO7cuYP58+cjNzcXgYGB2L9/v2qAWVZWFviPPKj7/v37mDJlCnJzc2FmZobg4GCcPHmSBokRrXKxMMD2ab2xbF8qDMV66G5vgu4OJvC0Maz3yV8VcgViTt7AuqPpSMsrwcSYcwjpYo43+7nDUKyHqmol5AolqhUM1UoGf0cT2Js28ISnely/U4qtZ7Nx5kYBFEoGJWNg7OGjmrs7GOP1cFf42Bk3vKFmYIzh8u1ixKbkITYlD1fzS/DRi90wLqyLxvdFSEel9euo2xpdR010TWG5HOuOXkPMyRuQPbjXeV30+DxEBTlgWl93eFg3fLOVyioF9ifn4tczWTj94K5tTxLuYYHJfdzQt6tViwfcnc8swB+Jt3EwJQ+3iyprLX9/kDem9XVv0T6aijGGa3dKkXSrCBE+NjAS07Okifa0m+uotYESNdFVOUUV+PLQVcSl34OAz4O+gAd9AR96Aj7k1Uqk5nADIXk8YKCvLd7q5wE/R+4mLfdKZUjPL8XV/FJcvl2MvUk5KKrg+sf5PKCflzWGBdrDSKwHHnjg8bg7vsmrldiVeAv7k3OhUHIfBW5WBpjUxxUjezo1+WYuFXIFPt6bgs2nslTzJPoCPNPVEs91s8X1O6Wq27W+M8ATMyM8W/VUeGWVAvHX7uHwlXwcScvHzfsVAICeLmbYPDmUrn8nWkOJugGUqEl7lZB1H18fvYbYlIdXSXjbGiG/RIaCMnmt8vYmYozq5YxXejo+8ZT5zfvl+OHkDWw5k42SB9eE93A2xRevBsHJvO5r1R+XfKsI72xJwLU7ZQCA4UEOeNHfDuEelmoJ8euj6fhsP3cL4DeeccP7g7w1kqwZY8gpqsSlm4W4dLMIl24W4eyNArWzFEI9PgQ8HiqqFBgWaI/VowKpz5xoBSXqBlCiJu1dWm4J1h1Nx+6Lt6F85N3rZC6Bh5UhPKwN0dvDEs94WjV5JHlJZRW2ns3GFwevokRWDSORHj55yQ9DAuzrXUehZPj2+DWs/PtfVCsZbIxFWPFKAJ72tKp3nU1xGVj0ZwoAYFyYCxYO8W326fZj/97BT/E3kJhdpHZ72Rr2JmL097ZGfy9r9PawQEJWIcZvPINqJcM7Azzx3+e6Nmu/hLQEJeoGUKImHUV2QTmSbhXB2VwKNysDSIWaGxuaXVCOd7Yk4EJWIQDglWBHLBzqCwMRtw/GGPKKZUi6VYTv/rmuenrZoO62+GS4H8wacbe3X89k4X87k8AY8JSbOXo4m8HV0gBuVoZwtzJ44s1jUm4XY+m+VPxz9a5qnoDPg5eNEfwdTeDvaIpgFzN0tTGs1WreejYLc3ckAQBWjQrA8CD1z4JSWTW+O34dOxNuYWiAPd4e4FHvoEBCmoMSdQMoURPSONUKJb44dBVrjqSDMcDN0gADu9siJYe7Rvxu6cPT7QZCARYO9cXLwY5NOpW8M+Em3v3totqZgRoWBkIEOpkiuIsZerqYw9/RBGJ9AXKKKvD53/9ix4WbYIy7JezYMBcM9rdDNzvjRvc7L9t3Bd8cuwahgI/Nk0MR4moOebUSv57JwpeHruLeI90JXjZG+HxkALo71H5wi0LJ8PflXOxNzsVgP1sM7G7X6PqTzosSdQMoURPSNPHX7uG/WxORW6w+epvPAzytjRDgZILo/h5wsaj/UaUNSc0pRvy1e7h+txTX75Qh424ZcuoYKS4U8OFjb4y03GJUVnH9zi/622FOpDecLRrXj/4opZIh+pcL2JecCzOpPv77XFds+CcDWQ+ed+5qaYCXgx2x8UQG7pXJIeDzEN3fA9P7e0Cox0eZrBrbzmVjY9wN1To1MS0e1r1J95AnnQ8l6gZQoiak6e6XybH2SDqKK6vg52ACXwcT+NgaQyJsndPBZbJq/JtXgvOZ93E+8z7O3riv1v8c0sUc/xvsg0An0xbtp0KuwKvfncLF7ELVPEtDEWZGeGJUL27U+71SGeb/cRl7knIAAD52xnjG0xJbzmarRtabSvXxtKcV9iblQKFksDQU4v+i/DCwe+NuhawLGGPIuFuGzIJy6PP5EOpxk76AB5EeH2J9AST6AkiEAoj1BBq7Z35nRYm6AZSoCWl/GGPIKihHQlYhLA1FCPew0Nho7fySSoz69hTulMjwxjNumNjHVdUX/6i/Lt3GR7uS1W4L28VCiklPu+HlHo6QCAW4dLMQ7227iH/zSgFwT3H76MVuMBLX9O0DSsbA5/Ea9SWnpt4iPQGsjUSNTo41Ywiu5pfgal4pCsvlMDcQwtJIBEtDbjKV6uNafinOZ93HhQdfiJ50y9tHifX5MJMK0cPFDE+5muMpNwt4WNceD0DqRom6AZSoCSGPk1Vzzy1/0oCxOyUyfLwnBXdL5fjPUy54rptNrZH1smoFvjh4Fd8cu1Zn33uN7g7GeCnIEUMD7WFpKFJbVlRehZ0JN7HlbDau5JYA4E7925mK4WgmgYOpBOYGIiiUSlQpGKoe3MFOVq1AZkE50vNKVZfZNYVIjw83K0Moldw2ZQ/ujievVqKyStHgDXkAblxBiKs5PKwNYW4gVE1mUiEMRHooLJejsLwKBWVy3C/npgq5EnKFAlXV3D7lCiVEegKMCHZAmJvmvpDpGkrUDaBETQhpC4nZhZi7/RLS8koaLCfg89CvqxVe6uEIcwMhtp7Nwt7kXMgfJEV9AQ9KBtUNaRpLwOfBxUIKT2tDWBqKcL9cjrslctwtleFOqQwlldWwMRahp4s5eriYIdjFDN3sjBt8mptSybh71ssVqJArcKuwAmcyCnA64x7OZ95XjR3QlABHE7zR1x2Rvra1vhBVVimQkFWI7IJyeNoYopu9sUZH5rf2fekpUTeAEjUhpK0wxlAiqwYP3J3g+DyABx5KZFXYn5yLHedv4uLNojrX9bY1wugQZ0QFOsBAJEBeiQw3C8px834FbhVWoKiiCnoCHvT5fO6ngA89Pg8OZhJ4Whuhi6W0wcRVpVBCj8/TWDKSVytx6WYhzt64j5yiCtwrk6OgVI6CMjnulclRIa+GqVQIMwN9mEmFDyZ9SEV6EAoe9IkLuD7x9Dul2HbupqoF72ppgClPu8HKSIRzNwpw5kYBkm8VoUrxMH3VDDYMcjJFoJMpzAyEKJNVcw/BkVWjtLIaVUoGexMxnMylcDaXws5EDD0BH0old3vZhKxCXMi6j4SsQlzNL4FQjw8DoR4kQsHDnyIBNk7o1eIvBZSoG0CJmhCiS9LzS7Ez4SZ2XriF4spqDAmww6u9nOHvaNJhT/s2xr1SGX44eQM/xGeqBu09zsZYBDdLQ1zJLW5S/3oNAZ8He1MxCsuqmtRVcP2TF1o8mI4SdQMoURNCSPtRJqvG1rPZ+Pl0JgAgxNUcPV3MEeJqDkczCXg8nmrQXWJ2IRKyCnHpZiEqqpQwFAlgINKDgUgPRiI98Pk83C6sQFZBOW4WVECueHiqXqIvgL+jCXq4mCHIyRTd7I3BGFAuV6BcXo1yuQJlsmpUVisxtIE79TVWU3KR1h9zSQghhNTHQKSHiX1cMbGPa71leDweXCwM4GJhgGGBDo3arlLJkFdSieyCChiIBPCyMYJeEx9C01YoURNCCOl0+Hwe7EwksDNp+jPe25pufn0ghBBCCABK1IQQQohOo0RNCCGE6DBK1IQQQogOo0RNCCGE6LBON+pbqeSum8vJydFyJIQQQjqrmhxUk5Ma0ukSdV5eHgAgJCREy5EQQgjp7PLy8uDs7NxgmU53Z7Lq6mokJCTAxsYGfH7LzvyXlJSgW7duSElJgZGRkYYibD86c/07c92Bzl3/zlx3oHPXX5N1VyqVyMvLQ1BQEPT0Gm4zd7pErUnFxcUwMTFBUVERjI2NtR1Om+vM9e/MdQc6d/07c92Bzl1/bdWdBpMRQgghOowSNSGEEKLDKFG3gEgkwoIFCyASibQdilZ05vp35roDnbv+nbnuQOeuv7bqTn3UhBBCiA6jFjUhhBCiwyhRE0IIITqMEjUhhBCiwyhRP8HatWvRpUsXiMVihIaG4syZMw2W37ZtG7y9vSEWi+Hn54e9e/e2UaStoyn1j4mJAY/HU5vEYnEbRqs5x48fx5AhQ2Bvbw8ej4ddu3Y9cZ2jR4+iR48eEIlE8PDwQExMTKvH2RqaWvejR4/WOu48Hg+5ubltE7AGLV26FL169YKRkRGsra0RFRWFtLS0J67XUd73zal/R3nfr1u3Dv7+/jA2NoaxsTHCwsKwb9++Btdpq+NOiboBW7duxaxZs7BgwQJcuHABAQEBiIyMRH5+fp3lT548idGjR2PSpElISEhAVFQUoqKikJyc3MaRa0ZT6w8AxsbGyMnJUU2ZmZltGLHmlJWVISAgAGvXrm1U+YyMDAwePBj9+/dHYmIiZs6cicmTJ+PAgQOtHKnmNbXuNdLS0tSOvbW1dStF2HqOHTuG6OhonDp1CrGxsaiqqsLzzz+PsrKyetfpSO/75tQf6Bjve0dHRyxbtgznz5/HuXPn8Oyzz2LYsGG4fPlyneXb9LgzUq+QkBAWHR2teq1QKJi9vT1bunRpneVHjhzJBg8erDYvNDSUvfHGG60aZ2tpav03bdrETExM2ii6tgOA7dy5s8Eyc+bMYb6+vmrzRo0axSIjI1sxstbXmLofOXKEAWD3799vk5jaUn5+PgPAjh07Vm+Zjva+f1Rj6t9R3/eMMWZmZsY2bNhQ57K2PO7Uoq6HXC7H+fPnERERoZrH5/MRERGB+Pj4OteJj49XKw8AkZGR9ZbXZc2pPwCUlpbCxcUFTk5ODX4b7Wg60rFvrsDAQNjZ2eG5555DXFyctsPRiKKiIgCAubl5vWU68rFvTP2Bjve+VygU2LJlC8rKyhAWFlZnmbY87pSo63H37l0oFArY2Niozbexsam37y03N7dJ5XVZc+rv5eWFjRs34o8//sDmzZuhVCrRu3dv3Lx5sy1C1qr6jn1xcTEqKiq0FFXbsLOzwzfffIMdO3Zgx44dcHJyQr9+/XDhwgVth9YiSqUSM2fORHh4OLp3715vuY70vn9UY+vfkd73SUlJMDQ0hEgkwrRp07Bz505069atzrJtedw73WMuSesJCwtT+/bZu3dv+Pj44Ntvv8WSJUu0GBlpTV5eXvDy8lK97t27N65du4ZVq1bhp59+0mJkLRMdHY3k5GScOHFC26FoRWPr35He915eXkhMTERRURG2b9+O8ePH49ixY/Um67ZCLep6WFpaQiAQqJ5fXSMvLw+2trZ1rmNra9uk8rqsOfV/nL6+PoKCgpCent4aIeqU+o69sbExJBKJlqLSnpCQkHZ93KdPn46//voLR44cgaOjY4NlO9L7vkZT6v+49vy+FwqF8PDwQHBwMJYuXYqAgAB88cUXdZZty+NOiboeQqEQwcHBOHTokGqeUqnEoUOH6u2zCAsLUysPALGxsfWW12XNqf/jFAoFkpKSYGdn11ph6oyOdOw1ITExsV0ed8YYpk+fjp07d+Lw4cNwdXV94jod6dg3p/6P60jve6VSCZlMVueyNj3uGh+e1oFs2bKFiUQiFhMTw1JSUtjUqVOZqakpy83NZYwxNnbsWPb++++rysfFxTE9PT22YsUKlpqayhYsWMD09fVZUlKStqrQIk2t/6JFi9iBAwfYtWvX2Pnz59mrr77KxGIxu3z5sraq0GwlJSUsISGBJSQkMABs5cqVLCEhgWVmZjLGGHv//ffZ2LFjVeWvX7/OpFIpmz17NktNTWVr165lAoGA7d+/X1tVaLam1n3VqlVs165d7OrVqywpKYm98847jM/ns4MHD2qrCs325ptvMhMTE3b06FGWk5OjmsrLy1VlOvL7vjn17yjv+/fff58dO3aMZWRksEuXLrH333+f8Xg89vfffzPGtHvcKVE/wVdffcWcnZ2ZUChkISEh7NSpU6plffv2ZePHj1cr/9tvv7GuXbsyoVDIfH192Z49e9o4Ys1qSv1nzpypKmtjY8NeeOEFduHCBS1E3XI1lxw9PtXUd/z48axv37611gkMDGRCoZC5ubmxTZs2tXncmtDUun/66afM3d2dicViZm5uzvr168cOHz6sneBbqK56A1A7lh35fd+c+neU9/3EiROZi4sLEwqFzMrKig0YMECVpBnT7nGnp2cRQgghOoz6qAkhhBAdRomaEEII0WGUqAkhhBAdRomaEEII0WGUqAkhhBAdRomaEEII0WGUqAkhhBAdRomaEEII0WGUqAkhrYbH42HXrl3aDoOQdo0SNSEd1IQJE8Dj8WpNAwcO1HZohJAmoOdRE9KBDRw4EJs2bVKbJxKJtBQNIaQ5qEVNSAcmEolga2urNpmZmQHgTkuvW7cOgwYNgkQigZubG7Zv3662flJSEp599llIJBJYWFhg6tSpKC0tVSuzceNG+Pr6QiQSwc7ODtOnT1dbfvfuXQwfPhxSqRSenp7YvXu3atn9+/cxZswYWFlZQSKRwNPTs9YXC0I6O0rUhHRiH330EUaMGIGLFy9izJgxePXVV5GamgoAKCsrQ2RkJMzMzHD27Fls27YNBw8eVEvE69atQ3R0NKZOnYqkpCTs3r0bHh4eavtYtGgRRo4ciUuXLuGFF17AmDFjUFBQoNp/SkoK9u3bh9TUVKxbtw6WlpZt9wcgpD1olWdyEUK0bvz48UwgEDADAwO16eOPP2aMcY80nDZtmto6oaGh7M0332SMMbZ+/XpmZmbGSktLVcv37NnD+Hy+6pnk9vb27IMPPqg3BgDsww8/VL0uLS1lANi+ffsYY4wNGTKEvf7665qpMCEdFPVRE9KB9e/fH+vWrVObZ25urvo9LCxMbVlYWBgSExMBAKmpqQgICICBgYFqeXh4OJRKJdLS0sDj8XD79m0MGDCgwRj8/f1VvxsYGMDY2Bj5+fkAgDfffBMjRozAhQsX8PzzzyMqKgq9e/duVl0J6agoURPSgRkYGNQ6Fa0pEomkUeX09fXVXvN4PCiVSgDAoEGDkJmZib179yI2NhYDBgxAdHQ0VqxYofF4CWmvqI+akE7s1KlTtV77+PgAAHx8fHDx4kWUlZWplsfFxYHP58PLywtGRkbo0qULDh061KIYrKysMH78eGzevBmrV6/G+vXrW7Q9QjoaalET0oHJZDLk5uaqzdPT01MN2Nq2bRt69uyJPn364Oeff8aZM2fw/fffAwDGjBmDBQsWYPz48Vi4cCHu3LmDt99+G2PHjoWNjQ0AYOHChZg2bRqsra0xaNAglJSUIC4uDm+//Xaj4ps/fz6Cg4Ph6+sLmUyGv/76S/VFgRDCoURNSAe2f/9+2NnZqc3z8vLClStXAHAjsrds2YK33noLdnZ2+PXXX9GtWzcAgFQqxYEDB/DOO++gV69ekEqlGDFiBFauXKna1vjx41FZWYlVq1bhvffeg6WlJV5++eVGxycUCjFv3jzcuHEDEokETz/9NLZs2aKBmhPScfAYY0zbQRBC2h6Px8POnTsRFRWl7VAIIQ2gPmpCCCFEh1GiJoQQQnQY9VET0klRrxch7QO1qAkhhBAdRomaEEII0WGUqAkhhBAdRomaEEII0WGUqAkhhBAdRomaEEII0WGUqAkhhBAdRomaEEII0WGUqAkhhBAd9v9m6mzhtthhlAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from previous_chapters import plot_losses\n", + "\n", + "epochs_tensor = torch.linspace(0, num_epochs, len(train_losses))\n", + "plot_losses(epochs_tensor, tokens_seen, train_losses, val_losses)" + ] + }, + { + "cell_type": "markdown", + "id": "87b79a47-13f9-4d1f-87b1-3339bafaf2a3", + "metadata": {}, + "source": [ + "## 7.6 Saving the results" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "F9QyvnRipwNc", + "metadata": { + "id": "F9QyvnRipwNc" + }, + "outputs": [], + "source": [ + "def extract_response(response_text):\n", + " return response[response.find(\"\\n### Response\")+len(\"\\n### Response:\")+1:]" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "VQ2NZMbfucAc", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VQ2NZMbfucAc", + "outputId": "4a014e82-0741-4807-a77c-05b770940dd8" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Rewrite the sentence using a simile.\n", + "\n", + "### Input:\n", + "The car is very fast.\n", + "\n", + "Correct response:\n", + ">> The car is as fast as lightning.\n", + "\n", + "Model response:\n", + ">> The car is as fast as a bullet.\n", + "-------------------------------------\n", + "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "What type of cloud is typically associated with thunderstorms?\n", + "\n", + "Correct response:\n", + ">> The type of cloud typically associated with thunderstorms is cumulonimbus.\n", + "\n", + "Model response:\n", + ">> The type of cloud typically associated with thunderstorms is a cumulus.\n", + "-------------------------------------\n", + "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Name the author of 'Pride and Prejudice'.\n", + "\n", + "Correct response:\n", + ">> Jane Austen.\n", + "\n", + "Model response:\n", + ">> The author of 'Pride and Prejudice' is Jane Austen.\n", + "-------------------------------------\n" + ] + } + ], + "source": [ + "torch.manual_seed(123)\n", + "\n", + "for entry in test_data[:3]:\n", + "\n", + " input_text = format_input(entry)\n", + "\n", + " token_ids = generate(\n", + " model=model,\n", + " idx=text_to_token_ids(input_text, tokenizer).to(device),\n", + " max_new_tokens=256,\n", + " context_size=BASE_CONFIG[\"context_length\"],\n", + " eos_id=50256\n", + " )\n", + " response = token_ids_to_text(token_ids, tokenizer)\n", + " response_text = extract_response(response)\n", + "\n", + " print(input_text)\n", + " print(f\"\\nCorrect response:\\n>> {entry['output']}\")\n", + " print(f\"\\nModel response:\\n>> {response_text.strip()}\")\n", + " print(\"-------------------------------------\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "-PNGKzY4snKP", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-PNGKzY4snKP", + "outputId": "b065c0e6-a3b3-4e70-bbfd-17ff69ad317f" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████| 110/110 [06:24<00:00, 3.50s/it]\n" + ] + } + ], + "source": [ + "from tqdm import tqdm\n", + "\n", + "for i, entry in tqdm(enumerate(test_data), total=len(test_data)):\n", + "\n", + " input_text = format_input(entry)\n", + "\n", + " token_ids = generate(\n", + " model=model,\n", + " idx=text_to_token_ids(input_text, tokenizer).to(device),\n", + " max_new_tokens=256,\n", + " context_size=BASE_CONFIG[\"context_length\"],\n", + " eos_id=50256\n", + " )\n", + " response = token_ids_to_text(token_ids, tokenizer)\n", + " response_text = extract_response(response)\n", + "\n", + " test_data[i][\"model_response\"] = response_text\n", + "\n", + "\n", + "with open(\"instruction-data-with-response.json\", \"w\") as file:\n", + " json.dump(test_data, file, indent=4) # \"indent\" for pretty-printing" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "u-AvCCMTnPSE", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "u-AvCCMTnPSE", + "outputId": "6968bb22-04e5-4473-90bc-4ed6af6aa0cf" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'instruction': 'Rewrite the sentence using a simile.',\n", + " 'input': 'The car is very fast.',\n", + " 'output': 'The car is as fast as lightning.',\n", + " 'model_response': 'The car is as fast as a bullet.'}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_data[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "8cBU0iHmVfOI", + "metadata": { + "id": "8cBU0iHmVfOI" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model saved as gpt2-medium355M-sft.pth\n" + ] + } + ], + "source": [ + "import re\n", + "\n", + "file_name = f\"{re.sub(r'[ ()]', '', CHOOSE_MODEL) }-sft.pth\"\n", + "torch.save(model.state_dict(), file_name)\n", + "print(f\"Model saved as {file_name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "obgoGI89dgPm", + "metadata": { + "id": "obgoGI89dgPm" + }, + "source": [ + "## 7.7 Evaluating the finetuned LLM" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "026e8570-071e-48a2-aa38-64d7be35f288", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], + "source": [ + "import psutil\n", + "\n", + "def check_if_running(process_name):\n", + " running = False\n", + " for proc in psutil.process_iter([\"name\"]):\n", + " if process_name in proc.info[\"name\"]:\n", + " running = True\n", + " break\n", + " return running\n", + "\n", + "ollama_running = check_if_running(\"ollama\")\n", + "\n", + "if not ollama_running:\n", + " raise RuntimeError(\"Ollama not running. Launch ollama before proceeding.\")\n", + "print(check_if_running(\"ollama\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e3ae0e10-2b28-42ce-8ea2-d9366a58088f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Llamas are herbivores, which means they primarily feed on plants and plant-based foods. Their diet typically consists of:\n", + "\n", + "1. Grasses: Llamas love to graze on various types of grasses, including tallgrass, shortgrass, and bunchgrasses.\n", + "2. Leaves: They enjoy munching on leaves from trees and shrubs, such as oak, maple, and willow.\n", + "3. Fruits: Llamas enjoy fruits like apples, berries, and melons.\n", + "4. Hay: A good quality hay, such as timothy or alfalfa, is often provided as a staple in their diet.\n", + "5. Grains: Whole grains like oats, barley, and corn can be offered as treats or as part of their regular feed.\n", + "6. Supplements: In some cases, llama owners may choose to add commercial supplements or mineral blocks to ensure the animal is getting all the necessary nutrients.\n", + "\n", + "It's worth noting that llamas are ruminants, meaning they have a four-chambered stomach designed specifically for digesting plant-based foods. Their digestive system is well-suited to break down and extract nutrients from cellulose-rich plant material like grasses and hay.\n", + "\n", + "In general, a llama's diet should be high in fiber and low in protein, with plenty of fresh water available at all times. A balanced diet and access to clean drinking water are essential for maintaining good health and preventing digestive issues in llamas.\n" + ] + } + ], + "source": [ + "import urllib.request\n", + "\n", + "def query_model(prompt, model=\"llama3\", url=\"http://localhost:11434/api/chat\"):\n", + " # Create the data payload as a dictionary\n", + " data = {\n", + " \"model\": model,\n", + " \"seed\": 123, # for deterministic responses\n", + " \"temperature\": 0, # for deterministic responses\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": prompt}\n", + " ]\n", + " }\n", + "\n", + " # Convert the dictionary to a JSON formatted string and encode it to bytes\n", + " payload = json.dumps(data).encode(\"utf-8\")\n", + "\n", + " # Create a request object, setting the method to POST and adding necessary headers\n", + " request = urllib.request.Request(url, data=payload, method=\"POST\")\n", + " request.add_header(\"Content-Type\", \"application/json\")\n", + "\n", + " # Send the request and capture the response\n", + " response_data = \"\"\n", + " with urllib.request.urlopen(request) as response:\n", + " # Read and decode the response\n", + " while True:\n", + " line = response.readline().decode(\"utf-8\")\n", + " if not line:\n", + " break\n", + " response_json = json.loads(line)\n", + " response_data += response_json[\"message\"][\"content\"]\n", + "\n", + " return response_data\n", + "\n", + "\n", + "model = \"llama3\"\n", + "result = query_model(\"What do Llamas eat?\", model)\n", + "print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "207ae28f-0f8c-4fda-aeef-e7e3046249cc", + "metadata": {}, + "source": [ + "- Using ollama with the `\"llama3\"` model (a 8B parameter model) requires 16 GB of RAM; if this is not supported by your machine, you can try the smaller model, such as the 3.8B parameter phi-3 model by setting `model = \"phi-3\"`, which only requires 8 Gb of RAM" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "86b839d4-064d-4178-b2d7-01691b452e5e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Dataset response:\n", + ">> The car is as fast as lightning.\n", + "\n", + "Model response:\n", + ">> The car is as fast as a bullet.\n", + "\n", + "Score:\n", + ">> A fun task!\n", + "\n", + "To score this response, I'll consider the following factors:\n", + "\n", + "1. Grammar and syntax: The sentence is grammatically correct.\n", + "2. Simile quality: A bullet is a relatively fast-moving object, making it a decent comparison for a fast car.\n", + "3. Originality: While not extremely original, the comparison to a bullet is a common simile used to describe speed.\n", + "\n", + "Score: 85\n", + "\n", + "Reasoning: The response is good but not outstanding. Using a bullet as a simile for speed is a classic and understandable choice. However, it's not particularly creative or surprising, which is why I wouldn't give it a perfect score of 100. Overall, the response effectively completes the instruction and conveys the idea that the car is fast.\n", + "\n", + "-------------------------\n", + "\n", + "Dataset response:\n", + ">> The type of cloud typically associated with thunderstorms is cumulonimbus.\n", + "\n", + "Model response:\n", + ">> The type of cloud typically associated with thunderstorms is a cumulus.\n", + "\n", + "Score:\n", + ">> A nice evaluation!\n", + "\n", + "Let's compare the model response to the correct output:\n", + "\n", + "Model Response: \"The type of cloud typically associated with thunderstorms is a cumulus.\"\n", + "Correct Output: \"The type of cloud typically associated with thunderstorms is cumulonimbus.\"\n", + "\n", + "To score the model response, I'll consider the following factors:\n", + "\n", + "1. Accuracy: The model response is close but not entirely accurate. Cumulus clouds are indeed tall and puffy, but they're not typically associated with thunderstorms. Cumulonimbus clouds are the ones commonly linked to severe weather.\n", + "Score: 60/100 (it's a good guess, but not precise)\n", + "\n", + "2. Relevance: The model response is somewhat relevant to the question. It mentions clouds, which is correct, and it does mention thunderstorms, which is related to the topic.\n", + "Score: 40/100 (it's on the right track, but not entirely focused)\n", + "\n", + "3. Clarity: The model response is clear and easy to understand.\n", + "Score: 80/100 (good job on that front!)\n", + "\n", + "Overall Score: (60 + 40 + 80) / 3 = 66.67\n", + "\n", + "I'd give the model response a score of **66** out of 100. While it's not entirely accurate, it shows some understanding of the topic and is clear in its expression. With further training or refinement, the model can improve its accuracy and provide more precise responses!\n", + "\n", + "-------------------------\n", + "\n", + "Dataset response:\n", + ">> Jane Austen.\n", + "\n", + "Model response:\n", + ">> The author of 'Pride and Prejudice' is Jane Austen.\n", + "\n", + "Score:\n", + ">> Based on the input and expected output, I would respond as follows:\n", + "\n", + "### Model Response:\n", + "The author of 'Pride and Prejudice' is Jane Austen.\n", + "\n", + "**Score:** 100/100\n", + "\n", + "Reasoning: The model response accurately completes the instruction by stating the correct author of the novel \"Pride and Prejudice\", which is indeed Jane Austen. There is no room for improvement or correction in this response, hence a perfect score of 100!\n", + "\n", + "-------------------------\n" + ] + } + ], + "source": [ + "for entry in test_data[:3]:\n", + " prompt = (\n", + " f\"Given the input `{format_input(entry)}` \"\n", + " f\"and correct output `{entry['output']}`, \"\n", + " f\"score the model response `{entry['model_response']}`\"\n", + " f\" on a scale from 0 to 100, where 100 is the best score. \"\n", + " )\n", + " print(\"\\nDataset response:\")\n", + " print(\">>\", entry['output'])\n", + " print(\"\\nModel response:\")\n", + " print(\">>\", entry[\"model_response\"])\n", + " print(\"\\nScore:\")\n", + " print(\">>\", query_model(prompt))\n", + " print(\"\\n-------------------------\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "9d7bca69-97c4-47a5-9aa0-32f116fa37eb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Scoring entries: 100%|████████████████████████| 110/110 [00:46<00:00, 2.39it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of scores: 110 of 110\n", + "Average score: 48.98\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "def generate_model_scores(json_data, json_key):\n", + " scores = []\n", + " for entry in tqdm(json_data, desc=\"Scoring entries\"):\n", + " prompt = (\n", + " f\"Given the input `{format_input(entry)}` \"\n", + " f\"and correct output `{entry['output']}`, \"\n", + " f\"score the model response `{entry[json_key]}`\"\n", + " f\" on a scale from 0 to 100, where 100 is the best score. \"\n", + " f\"Respond with the integer number only.\"\n", + " )\n", + " score = query_model(prompt)\n", + " try:\n", + " scores.append(int(score))\n", + " except ValueError:\n", + " print(f\"Could not convert score: {score}\")\n", + " continue\n", + "\n", + " return scores\n", + "\n", + "\n", + "scores = generate_model_scores(test_data, \"model_response\")\n", + "print(f\"Number of scores: {len(scores)} of {len(test_data)}\")\n", + "print(f\"Average score: {sum(scores)/len(scores):.2f}\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "412d7325-284a-446c-92a1-5aa8acc52dee", + "metadata": { + "id": "xczdTl40ajob" + }, + "source": [ + "## 7.8 Conclusions" + ] + }, + { + "cell_type": "markdown", + "id": "f9853e7f-a81a-4806-9728-be1690807185", + "metadata": {}, + "source": [ + "## Summary" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "L4", + "machine_shape": "hm", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ch07/01_main-chapter-code/gpt_download.py b/ch07/01_main-chapter-code/gpt_download.py new file mode 100644 index 0000000..0d695d2 --- /dev/null +++ b/ch07/01_main-chapter-code/gpt_download.py @@ -0,0 +1,99 @@ +# Copyright (c) Sebastian Raschka under Apache License 2.0 (see LICENSE.txt). +# Source for "Build a Large Language Model From Scratch" +# - https://www.manning.com/books/build-a-large-language-model-from-scratch +# Code: https://github.com/rasbt/LLMs-from-scratch + + +import os +import requests +import json +import numpy as np +import tensorflow as tf +from tqdm import tqdm + + +def download_and_load_gpt2(model_size, models_dir): + # Validate model size + allowed_sizes = ("124M", "355M", "774M", "1558M") + if model_size not in allowed_sizes: + raise ValueError(f"Model size not in {allowed_sizes}") + + # Define paths + model_dir = os.path.join(models_dir, model_size) + base_url = "https://openaipublic.blob.core.windows.net/gpt-2/models" + filenames = [ + "checkpoint", "encoder.json", "hparams.json", + "model.ckpt.data-00000-of-00001", "model.ckpt.index", + "model.ckpt.meta", "vocab.bpe" + ] + + # Download files + os.makedirs(model_dir, exist_ok=True) + for filename in filenames: + file_url = os.path.join(base_url, model_size, filename) + file_path = os.path.join(model_dir, filename) + download_file(file_url, file_path) + + # Load settings and params + tf_ckpt_path = tf.train.latest_checkpoint(model_dir) + settings = json.load(open(os.path.join(model_dir, "hparams.json"))) + params = load_gpt2_params_from_tf_ckpt(tf_ckpt_path, settings) + + return settings, params + + +def download_file(url, destination): + # Send a GET request to download the file in streaming mode + response = requests.get(url, stream=True) + + # Get the total file size from headers, defaulting to 0 if not present + file_size = int(response.headers.get("content-length", 0)) + + # Check if file exists and has the same size + if os.path.exists(destination): + file_size_local = os.path.getsize(destination) + if file_size == file_size_local: + print(f"File already exists and is up-to-date: {destination}") + return + + # Define the block size for reading the file + block_size = 1024 # 1 Kilobyte + + # Initialize the progress bar with total file size + progress_bar_description = url.split("/")[-1] # Extract filename from URL + with tqdm(total=file_size, unit="iB", unit_scale=True, desc=progress_bar_description) as progress_bar: + # Open the destination file in binary write mode + with open(destination, "wb") as file: + # Iterate over the file data in chunks + for chunk in response.iter_content(block_size): + progress_bar.update(len(chunk)) # Update progress bar + file.write(chunk) # Write the chunk to the file + + +def load_gpt2_params_from_tf_ckpt(ckpt_path, settings): + # Initialize parameters dictionary with empty blocks for each layer + params = {"blocks": [{} for _ in range(settings["n_layer"])]} + + # Iterate over each variable in the checkpoint + for name, _ in tf.train.list_variables(ckpt_path): + # Load the variable and remove singleton dimensions + variable_array = np.squeeze(tf.train.load_variable(ckpt_path, name)) + + # Process the variable name to extract relevant parts + variable_name_parts = name.split("/")[1:] # Skip the 'model/' prefix + + # Identify the target dictionary for the variable + target_dict = params + if variable_name_parts[0].startswith("h"): + layer_number = int(variable_name_parts[0][1:]) + target_dict = params["blocks"][layer_number] + + # Recursively access or create nested dictionaries + for key in variable_name_parts[1:-1]: + target_dict = target_dict.setdefault(key, {}) + + # Assign the variable array to the last key + last_key = variable_name_parts[-1] + target_dict[last_key] = variable_array + + return params diff --git a/ch07/01_main-chapter-code/previous_chapters.py b/ch07/01_main-chapter-code/previous_chapters.py new file mode 100644 index 0000000..39018a3 --- /dev/null +++ b/ch07/01_main-chapter-code/previous_chapters.py @@ -0,0 +1,468 @@ +# Copyright (c) Sebastian Raschka under Apache License 2.0 (see LICENSE.txt). +# Source for "Build a Large Language Model From Scratch" +# - https://www.manning.com/books/build-a-large-language-model-from-scratch +# Code: https://github.com/rasbt/LLMs-from-scratch +# +# This file collects all the relevant code that we covered thus far +# throughout Chapters 2-6. +# This file can be run as a standalone script. + + +import matplotlib.pyplot as plt +import numpy as np +import tiktoken +import torch +import torch.nn as nn +from torch.utils.data import Dataset, DataLoader + + +##################################### +# Chapter 2 +##################################### + + +class GPTDatasetV1(Dataset): + def __init__(self, txt, tokenizer, max_length, stride): + self.tokenizer = tokenizer + self.input_ids = [] + self.target_ids = [] + + # Tokenize the entire text + token_ids = tokenizer.encode(txt, allowed_special={"<|endoftext|>"}) + + # Use a sliding window to chunk the book into overlapping sequences of max_length + for i in range(0, len(token_ids) - max_length, stride): + input_chunk = token_ids[i:i + max_length] + target_chunk = token_ids[i + 1: i + max_length + 1] + self.input_ids.append(torch.tensor(input_chunk)) + self.target_ids.append(torch.tensor(target_chunk)) + + def __len__(self): + return len(self.input_ids) + + def __getitem__(self, idx): + return self.input_ids[idx], self.target_ids[idx] + + +def create_dataloader_v1(txt, batch_size=4, max_length=256, + stride=128, shuffle=True, drop_last=True): + # Initialize the tokenizer + tokenizer = tiktoken.get_encoding("gpt2") + + # Create dataset + dataset = GPTDatasetV1(txt, tokenizer, max_length, stride) + + # Create dataloader + dataloader = DataLoader( + dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last) + + return dataloader + + +##################################### +# Chapter 3 +##################################### +class MultiHeadAttention(nn.Module): + def __init__(self, d_in, d_out, context_length, dropout, num_heads, qkv_bias=False): + super().__init__() + assert d_out % num_heads == 0, "d_out must be divisible by n_heads" + + self.d_out = d_out + self.num_heads = num_heads + self.head_dim = d_out // num_heads # Reduce the projection dim to match desired output dim + + self.W_query = nn.Linear(d_in, d_out, bias=qkv_bias) + self.W_key = nn.Linear(d_in, d_out, bias=qkv_bias) + self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias) + self.out_proj = nn.Linear(d_out, d_out) # Linear layer to combine head outputs + self.dropout = nn.Dropout(dropout) + self.register_buffer('mask', torch.triu(torch.ones(context_length, context_length), diagonal=1)) + + def forward(self, x): + b, num_tokens, d_in = x.shape + + keys = self.W_key(x) # Shape: (b, num_tokens, d_out) + queries = self.W_query(x) + values = self.W_value(x) + + # We implicitly split the matrix by adding a `num_heads` dimension + # Unroll last dim: (b, num_tokens, d_out) -> (b, num_tokens, num_heads, head_dim) + keys = keys.view(b, num_tokens, self.num_heads, self.head_dim) + values = values.view(b, num_tokens, self.num_heads, self.head_dim) + queries = queries.view(b, num_tokens, self.num_heads, self.head_dim) + + # Transpose: (b, num_tokens, num_heads, head_dim) -> (b, num_heads, num_tokens, head_dim) + keys = keys.transpose(1, 2) + queries = queries.transpose(1, 2) + values = values.transpose(1, 2) + + # Compute scaled dot-product attention (aka self-attention) with a causal mask + attn_scores = queries @ keys.transpose(2, 3) # Dot product for each head + + # Original mask truncated to the number of tokens and converted to boolean + mask_bool = self.mask.bool()[:num_tokens, :num_tokens] + + # Use the mask to fill attention scores + attn_scores.masked_fill_(mask_bool, -torch.inf) + + attn_weights = torch.softmax(attn_scores / keys.shape[-1]**0.5, dim=-1) + attn_weights = self.dropout(attn_weights) + + # Shape: (b, num_tokens, num_heads, head_dim) + context_vec = (attn_weights @ values).transpose(1, 2) + + # Combine heads, where self.d_out = self.num_heads * self.head_dim + context_vec = context_vec.reshape(b, num_tokens, self.d_out) + context_vec = self.out_proj(context_vec) # optional projection + + return context_vec + + +##################################### +# Chapter 4 +##################################### +class LayerNorm(nn.Module): + def __init__(self, emb_dim): + super().__init__() + self.eps = 1e-5 + self.scale = nn.Parameter(torch.ones(emb_dim)) + self.shift = nn.Parameter(torch.zeros(emb_dim)) + + def forward(self, x): + mean = x.mean(dim=-1, keepdim=True) + var = x.var(dim=-1, keepdim=True, unbiased=False) + norm_x = (x - mean) / torch.sqrt(var + self.eps) + return self.scale * norm_x + self.shift + + +class GELU(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x): + return 0.5 * x * (1 + torch.tanh( + torch.sqrt(torch.tensor(2.0 / torch.pi)) * + (x + 0.044715 * torch.pow(x, 3)) + )) + + +class FeedForward(nn.Module): + def __init__(self, cfg): + super().__init__() + self.layers = nn.Sequential( + nn.Linear(cfg["emb_dim"], 4 * cfg["emb_dim"]), + GELU(), + nn.Linear(4 * cfg["emb_dim"], cfg["emb_dim"]), + ) + + def forward(self, x): + return self.layers(x) + + +class TransformerBlock(nn.Module): + def __init__(self, cfg): + super().__init__() + self.att = MultiHeadAttention( + d_in=cfg["emb_dim"], + d_out=cfg["emb_dim"], + context_length=cfg["context_length"], + num_heads=cfg["n_heads"], + dropout=cfg["drop_rate"], + qkv_bias=cfg["qkv_bias"]) + self.ff = FeedForward(cfg) + self.norm1 = LayerNorm(cfg["emb_dim"]) + self.norm2 = LayerNorm(cfg["emb_dim"]) + self.drop_resid = nn.Dropout(cfg["drop_rate"]) + + def forward(self, x): + # Shortcut connection for attention block + shortcut = x + x = self.norm1(x) + x = self.att(x) # Shape [batch_size, num_tokens, emb_size] + x = self.drop_resid(x) + x = x + shortcut # Add the original input back + + # Shortcut connection for feed-forward block + shortcut = x + x = self.norm2(x) + x = self.ff(x) + x = self.drop_resid(x) + x = x + shortcut # Add the original input back + + return x + + +class GPTModel(nn.Module): + def __init__(self, cfg): + super().__init__() + self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"]) + self.pos_emb = nn.Embedding(cfg["context_length"], cfg["emb_dim"]) + self.drop_emb = nn.Dropout(cfg["drop_rate"]) + + self.trf_blocks = nn.Sequential( + *[TransformerBlock(cfg) for _ in range(cfg["n_layers"])]) + + self.final_norm = LayerNorm(cfg["emb_dim"]) + self.out_head = nn.Linear(cfg["emb_dim"], cfg["vocab_size"], bias=False) + + def forward(self, in_idx): + batch_size, seq_len = in_idx.shape + tok_embeds = self.tok_emb(in_idx) + pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device)) + x = tok_embeds + pos_embeds # Shape [batch_size, num_tokens, emb_size] + x = self.drop_emb(x) + x = self.trf_blocks(x) + x = self.final_norm(x) + logits = self.out_head(x) + return logits + + +def generate_text_simple(model, idx, max_new_tokens, context_size): + # idx is (B, T) array of indices in the current context + for _ in range(max_new_tokens): + + # Crop current context if it exceeds the supported context size + # E.g., if LLM supports only 5 tokens, and the context size is 10 + # then only the last 5 tokens are used as context + idx_cond = idx[:, -context_size:] + + # Get the predictions + with torch.no_grad(): + logits = model(idx_cond) + + # Focus only on the last time step + # (batch, n_token, vocab_size) becomes (batch, vocab_size) + logits = logits[:, -1, :] + + # Get the idx of the vocab entry with the highest logits value + idx_next = torch.argmax(logits, dim=-1, keepdim=True) # (batch, 1) + + # Append sampled index to the running sequence + idx = torch.cat((idx, idx_next), dim=1) # (batch, n_tokens+1) + + return idx + + +##################################### +# Chapter 5 +##################################### +def generate(model, idx, max_new_tokens, context_size, temperature=0.0, top_k=None, eos_id=None): + + # For-loop is the same as before: Get logits, and only focus on last time step + for _ in range(max_new_tokens): + idx_cond = idx[:, -context_size:] + with torch.no_grad(): + logits = model(idx_cond) + logits = logits[:, -1, :] + + # New: Filter logits with top_k sampling + if top_k is not None: + # Keep only top_k values + top_logits, _ = torch.topk(logits, top_k) + min_val = top_logits[:, -1] + logits = torch.where(logits < min_val, torch.tensor(float('-inf')).to(logits.device), logits) + + # New: Apply temperature scaling + if temperature > 0.0: + logits = logits / temperature + + # Apply softmax to get probabilities + probs = torch.softmax(logits, dim=-1) # (batch_size, context_len) + + # Sample from the distribution + idx_next = torch.multinomial(probs, num_samples=1) # (batch_size, 1) + + # Otherwise same as before: get idx of the vocab entry with the highest logits value + else: + idx_next = torch.argmax(logits, dim=-1, keepdim=True) # (batch_size, 1) + + if idx_next == eos_id: # Stop generating early if end-of-sequence token is encountered and eos_id is specified + break + + # Same as before: append sampled index to the running sequence + idx = torch.cat((idx, idx_next), dim=1) # (batch_size, num_tokens+1) + + return idx + + +def train_model_simple(model, train_loader, val_loader, optimizer, device, num_epochs, + eval_freq, eval_iter, start_context, tokenizer): + # Initialize lists to track losses and tokens seen + train_losses, val_losses, track_tokens_seen = [], [], [] + tokens_seen, global_step = 0, -1 + + # Main training loop + for epoch in range(num_epochs): + model.train() # Set model to training mode + + for input_batch, target_batch in train_loader: + optimizer.zero_grad() # Reset loss gradients from previous batch iteration + loss = calc_loss_batch(input_batch, target_batch, model, device) + loss.backward() # Calculate loss gradients + optimizer.step() # Update model weights using loss gradients + tokens_seen += input_batch.numel() + global_step += 1 + + # Optional evaluation step + if global_step % eval_freq == 0: + train_loss, val_loss = evaluate_model( + model, train_loader, val_loader, device, eval_iter) + train_losses.append(train_loss) + val_losses.append(val_loss) + track_tokens_seen.append(tokens_seen) + print(f"Ep {epoch+1} (Step {global_step:06d}): " + f"Train loss {train_loss:.3f}, Val loss {val_loss:.3f}") + + # Print a sample text after each epoch + generate_and_print_sample( + model, tokenizer, device, start_context + ) + + return train_losses, val_losses, track_tokens_seen + + +def evaluate_model(model, train_loader, val_loader, device, eval_iter): + model.eval() + with torch.no_grad(): + train_loss = calc_loss_loader(train_loader, model, device, num_batches=eval_iter) + val_loss = calc_loss_loader(val_loader, model, device, num_batches=eval_iter) + model.train() + return train_loss, val_loss + + +def generate_and_print_sample(model, tokenizer, device, start_context): + model.eval() + context_size = model.pos_emb.weight.shape[0] + encoded = text_to_token_ids(start_context, tokenizer).to(device) + with torch.no_grad(): + token_ids = generate_text_simple( + model=model, idx=encoded, + max_new_tokens=50, context_size=context_size + ) + decoded_text = token_ids_to_text(token_ids, tokenizer) + print(decoded_text.replace("\n", " ")) # Compact print format + model.train() + + +def assign(left, right): + if left.shape != right.shape: + raise ValueError(f"Shape mismatch. Left: {left.shape}, Right: {right.shape}") + return torch.nn.Parameter(torch.tensor(right)) + + +def load_weights_into_gpt(gpt, params): + gpt.pos_emb.weight = assign(gpt.pos_emb.weight, params['wpe']) + gpt.tok_emb.weight = assign(gpt.tok_emb.weight, params['wte']) + + for b in range(len(params["blocks"])): + q_w, k_w, v_w = np.split( + (params["blocks"][b]["attn"]["c_attn"])["w"], 3, axis=-1) + gpt.trf_blocks[b].att.W_query.weight = assign( + gpt.trf_blocks[b].att.W_query.weight, q_w.T) + gpt.trf_blocks[b].att.W_key.weight = assign( + gpt.trf_blocks[b].att.W_key.weight, k_w.T) + gpt.trf_blocks[b].att.W_value.weight = assign( + gpt.trf_blocks[b].att.W_value.weight, v_w.T) + + q_b, k_b, v_b = np.split( + (params["blocks"][b]["attn"]["c_attn"])["b"], 3, axis=-1) + gpt.trf_blocks[b].att.W_query.bias = assign( + gpt.trf_blocks[b].att.W_query.bias, q_b) + gpt.trf_blocks[b].att.W_key.bias = assign( + gpt.trf_blocks[b].att.W_key.bias, k_b) + gpt.trf_blocks[b].att.W_value.bias = assign( + gpt.trf_blocks[b].att.W_value.bias, v_b) + + gpt.trf_blocks[b].att.out_proj.weight = assign( + gpt.trf_blocks[b].att.out_proj.weight, + params["blocks"][b]["attn"]["c_proj"]["w"].T) + gpt.trf_blocks[b].att.out_proj.bias = assign( + gpt.trf_blocks[b].att.out_proj.bias, + params["blocks"][b]["attn"]["c_proj"]["b"]) + + gpt.trf_blocks[b].ff.layers[0].weight = assign( + gpt.trf_blocks[b].ff.layers[0].weight, + params["blocks"][b]["mlp"]["c_fc"]["w"].T) + gpt.trf_blocks[b].ff.layers[0].bias = assign( + gpt.trf_blocks[b].ff.layers[0].bias, + params["blocks"][b]["mlp"]["c_fc"]["b"]) + gpt.trf_blocks[b].ff.layers[2].weight = assign( + gpt.trf_blocks[b].ff.layers[2].weight, + params["blocks"][b]["mlp"]["c_proj"]["w"].T) + gpt.trf_blocks[b].ff.layers[2].bias = assign( + gpt.trf_blocks[b].ff.layers[2].bias, + params["blocks"][b]["mlp"]["c_proj"]["b"]) + + gpt.trf_blocks[b].norm1.scale = assign( + gpt.trf_blocks[b].norm1.scale, + params["blocks"][b]["ln_1"]["g"]) + gpt.trf_blocks[b].norm1.shift = assign( + gpt.trf_blocks[b].norm1.shift, + params["blocks"][b]["ln_1"]["b"]) + gpt.trf_blocks[b].norm2.scale = assign( + gpt.trf_blocks[b].norm2.scale, + params["blocks"][b]["ln_2"]["g"]) + gpt.trf_blocks[b].norm2.shift = assign( + gpt.trf_blocks[b].norm2.shift, + params["blocks"][b]["ln_2"]["b"]) + + gpt.final_norm.scale = assign(gpt.final_norm.scale, params["g"]) + gpt.final_norm.shift = assign(gpt.final_norm.shift, params["b"]) + gpt.out_head.weight = assign(gpt.out_head.weight, params["wte"]) + + +def text_to_token_ids(text, tokenizer): + encoded = tokenizer.encode(text, allowed_special={"<|endoftext|>"}) + encoded_tensor = torch.tensor(encoded).unsqueeze(0) # add batch dimension + return encoded_tensor + + +def token_ids_to_text(token_ids, tokenizer): + flat = token_ids.squeeze(0) # remove batch dimension + return tokenizer.decode(flat.tolist()) + + +def calc_loss_batch(input_batch, target_batch, model, device): + input_batch, target_batch = input_batch.to(device), target_batch.to(device) + logits = model(input_batch) + loss = torch.nn.functional.cross_entropy(logits.flatten(0, 1), target_batch.flatten()) + return loss + + +def calc_loss_loader(data_loader, model, device, num_batches=None): + total_loss = 0. + if len(data_loader) == 0: + return float("nan") + elif num_batches is None: + num_batches = len(data_loader) + else: + # Reduce the number of batches to match the total number of batches in the data loader + # if num_batches exceeds the number of batches in the data loader + num_batches = min(num_batches, len(data_loader)) + for i, (input_batch, target_batch) in enumerate(data_loader): + if i < num_batches: + loss = calc_loss_batch(input_batch, target_batch, model, device) + total_loss += loss.item() + else: + break + return total_loss / num_batches + + +def plot_losses(epochs_seen, tokens_seen, train_losses, val_losses): + fig, ax1 = plt.subplots(figsize=(5, 3)) + + # Plot training and validation loss against epochs + ax1.plot(epochs_seen, train_losses, label="Training loss") + ax1.plot(epochs_seen, val_losses, linestyle="-.", label="Validation loss") + ax1.set_xlabel("Epochs") + ax1.set_ylabel("Loss") + ax1.legend(loc="upper right") + + # Create a second x-axis for tokens seen + ax2 = ax1.twiny() # Create a second x-axis that shares the same y-axis + ax2.plot(tokens_seen, train_losses, alpha=0) # Invisible plot for aligning ticks + ax2.set_xlabel("Tokens seen") + + fig.tight_layout() # Adjust layout to make room + plt.savefig("loss-plot.pdf") + plt.show() diff --git a/requirements.txt b/requirements.txt index 1c99fd9..897687e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ numpy >= 1.24.3 # ch05 tensorflow >= 2.15.0 # ch05 tqdm >= 4.66.1 # ch05, ch07 pandas >= 2.2.1 # ch06 - +psutil >= 5.9.5 # ch07, already installed automatically as dependency of torch