diff --git a/ch03/02_bonus_efficient-multihead-attention/mha-implementations.ipynb b/ch03/02_bonus_efficient-multihead-attention/mha-implementations.ipynb index b7b27df..d3500e1 100644 --- a/ch03/02_bonus_efficient-multihead-attention/mha-implementations.ipynb +++ b/ch03/02_bonus_efficient-multihead-attention/mha-implementations.ipynb @@ -1,1000 +1,1000 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "6f678e62-7bcb-4405-86ae-dce94f494303", - "metadata": { - "id": "6f678e62-7bcb-4405-86ae-dce94f494303" - }, - "source": [ - "# Efficient Multi-Head Attention Implementations" - ] - }, - { - "cell_type": "markdown", - "id": "b742938a-4bfc-4527-a1f1-d5963508967d", - "metadata": { - "id": "b742938a-4bfc-4527-a1f1-d5963508967d" - }, - "source": [ - "This code notebook compares different ways to implement causal multi-head attention used in decoder-style LLMs like GPT, Llama, etc." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "7898551e-f582-48ac-9f66-3632abe2a93f", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "7898551e-f582-48ac-9f66-3632abe2a93f", - "outputId": "2ddf0145-94d3-4490-8087-d1ffeb6f30ab" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "PyTorch version: 2.2.1+cu121\n", - "Running on cuda\n" - ] - } - ], - "source": [ - "import torch\n", - "\n", - "torch.manual_seed(123)\n", - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "print(f\"PyTorch version: {torch.__version__}\")\n", - "print(f\"Running on {device}\")\n", - "\n", - "batch_size = 8\n", - "context_len = 1024\n", - "embed_dim = 768\n", - "embeddings = torch.randn((batch_size, context_len, embed_dim), device=device)" - ] - }, - { - "cell_type": "markdown", - "id": "2f9bb1b6-a1e5-4e0a-884d-0f31b374a8d6", - "metadata": { - "id": "2f9bb1b6-a1e5-4e0a-884d-0f31b374a8d6" - }, - "source": [ - "## 1) CausalAttention MHA wrapper class from chapter 3" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "297c93ed-aec0-4896-bb89-42c4b294d3d1", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "297c93ed-aec0-4896-bb89-42c4b294d3d1", - "outputId": "ae6d707f-eae8-467a-ed4d-a88051bf776f" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "torch.Size([8, 1024, 9216])\n" - ] - } - ], - "source": [ - "from ch03 import MultiHeadAttentionWrapper as Ch03_MHA_Wrapper\n", - "\n", - "mha_ch03_wrapper = Ch03_MHA_Wrapper(\n", - " d_in=embed_dim,\n", - " d_out=embed_dim//12,\n", - " block_size=context_len,\n", - " dropout=0.0,\n", - " num_heads=12,\n", - " qkv_bias=False\n", - ").to(device)\n", - "\n", - "out = mha_ch03_wrapper(embeddings)\n", - "print(out.shape)" - ] - }, - { - "cell_type": "markdown", - "id": "21930804-b327-40b1-8e63-94dcad39ce7b", - "metadata": { - "id": "21930804-b327-40b1-8e63-94dcad39ce7b" - }, - "source": [ - "## 2) The multi-head attention class from chapter 3" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "4ee6a61b-d25c-4a0c-8a59-f285544e3710", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "4ee6a61b-d25c-4a0c-8a59-f285544e3710", - "outputId": "5df88462-8b1a-4b1f-ce71-3909ad2ca9c2" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "torch.Size([8, 1024, 768])\n" - ] - } - ], - "source": [ - "from ch03 import MultiHeadAttention as Ch03_MHA\n", - "\n", - "mha_ch03 = Ch03_MHA(\n", - " d_in=embed_dim,\n", - " d_out=embed_dim,\n", - " block_size=context_len,\n", - " dropout=0.0,\n", - " num_heads=12,\n", - " qkv_bias=False\n", - ").to(device)\n", - "\n", - "out = mha_ch03(embeddings)\n", - "print(out.shape)" - ] - }, - { - "cell_type": "markdown", - "id": "73cd11da-ea3b-4081-b483-c4965dfefbc4", - "metadata": { - "id": "73cd11da-ea3b-4081-b483-c4965dfefbc4" - }, - "source": [ - "## 3) An alternative multi-head attention with combined weights" - ] - }, - { - "cell_type": "markdown", - "id": "1fa1a5ea-eaff-4d2d-aaf0-b34cdb6fd4dd", - "metadata": { - "id": "1fa1a5ea-eaff-4d2d-aaf0-b34cdb6fd4dd" - }, - "source": [ - "- The code for the `MultiHeadAttentionAlt` class below is based on code that was kindly shared by [Rayed Bin Wahed](https://github.com/rasbt/LLMs-from-scratch/discussions/51)\n", - "- The main difference between the `MultiHeadAttentionAlt` class and the `MultiHeadAttention` class used in chapter 3 is that `MultiHeadAttentionAlt` uses a single weight matrix, `self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)` instead of separate weight matrices:\n", - "\n", - " - `self.W_query = nn.Linear(d_in, d_out, bias=qkv_bias)`\n", - " - `self.W_key = nn.Linear(d_in, d_out, bias=qkv_bias)`\n", - " - `self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)`\n", - "\n", - "- Here, `self.qkv` combines all three weight matrices `self.W_query`, `self.W_key`, and `self.W_value` to carry out the query, key, and value computation in a single step\n", - "- Using `q, k, v = qkv.unbind(0)`, we obtain the individual query, key, and value tensors, which are then used similarly to the query, key, and value tensors in the `MultiHeadAttention` class in chapter 3" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "9a6bd0a2-f27c-4602-afa0-c96cd295c1a6", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "9a6bd0a2-f27c-4602-afa0-c96cd295c1a6", - "outputId": "1240afaf-139a-4d01-ddac-4a186ff4a4fd" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "torch.Size([8, 1024, 768])\n" - ] - } - ], - "source": [ - "import torch.nn as nn\n", - "\n", - "\n", - "class MultiHeadAttentionCombinedQKV(nn.Module):\n", - " def __init__(self, d_in, d_out, num_heads, block_size, dropout=0.0, qkv_bias=False):\n", - " super().__init__()\n", - "\n", - " assert d_out % num_heads == 0, \"embed_dim is indivisible by num_heads\"\n", - "\n", - " self.num_heads = num_heads\n", - " self.block_size = block_size\n", - " self.head_dim = d_out // num_heads\n", - "\n", - " self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)\n", - " self.proj = nn.Linear(d_in, d_out)\n", - " self.dropout = nn.Dropout(dropout)\n", - "\n", - " self.register_buffer(\n", - " \"mask\", torch.triu(torch.ones(block_size, block_size), diagonal=1)\n", - " )\n", - "\n", - " def forward(self, x):\n", - " batch_size, num_tokens, embed_dim = x.shape\n", - "\n", - " # (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)\n", - " qkv = self.qkv(x)\n", - "\n", - " # (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)\n", - " qkv = qkv.reshape(batch_size, num_tokens, 3, self.num_heads, self.head_dim)\n", - "\n", - " # (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)\n", - " qkv = qkv.permute(2, 0, 3, 1, 4)\n", - "\n", - " # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_head, num_tokens, head_dim)\n", - " queries, keys, values = qkv.unbind(0)\n", - "\n", - " # (b, num_heads, num_tokens, head_dim) --> (b, num_heads, num_tokens, num_tokens)\n", - " attn_scores = queries @ keys.transpose(-2, -1)\n", - " attn_scores = attn_scores.masked_fill(\n", - " self.mask.bool()[:num_tokens, :num_tokens], -torch.inf\n", - " )\n", - "\n", - " attn_weights = torch.softmax(attn_scores / keys.shape[-1]**-0.5, dim=-1)\n", - " attn_weights = self.dropout(attn_weights)\n", - "\n", - " # (b, num_heads, num_tokens, num_tokens) --> (b, num_heads, num_tokens, head_dim)\n", - " context_vec = attn_weights @ values\n", - "\n", - " # (b, num_heads, num_tokens, head_dim) --> (b, num_tokens, num_heads, head_dim)\n", - " context_vec = context_vec.transpose(1, 2)\n", - "\n", - " # (b, num_tokens, num_heads, head_dim) --> (b, num_tokens, embed_dim)\n", - " context_vec = context_vec.reshape(batch_size, num_tokens, embed_dim)\n", - "\n", - " context_vec = self.proj(context_vec)\n", - "\n", - " return context_vec\n", - "\n", - "\n", - "mha_combined_qkv = MultiHeadAttentionCombinedQKV(\n", - " d_in=embed_dim,\n", - " d_out=embed_dim,\n", - " block_size=context_len,\n", - " dropout=0.0,\n", - " num_heads=12,\n", - " qkv_bias=False\n", - ").to(device)\n", - "\n", - "out = mha_combined_qkv(embeddings)\n", - "print(out.shape)" - ] - }, - { - "cell_type": "markdown", - "id": "48a042d3-ee78-4c29-bf63-d92fe6706632", - "metadata": { - "id": "48a042d3-ee78-4c29-bf63-d92fe6706632" - }, - "source": [ - "## 4) Multihead attention with PyTorch's scaled dot product attention" - ] - }, - { - "cell_type": "markdown", - "id": "f78e346f-3b85-44e6-9feb-f01131381148", - "metadata": { - "id": "f78e346f-3b85-44e6-9feb-f01131381148" - }, - "source": [ - "- The implementation below uses PyTorch's [`scaled_dot_product_attention`](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html) function, which implements a memory-optimized version of self-attention calld [flash attention](https://arxiv.org/abs/2205.14135)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "1b8e5a0d-1f65-4a03-bf6e-723f0cc428f5", - "metadata": { - "id": "1b8e5a0d-1f65-4a03-bf6e-723f0cc428f5" - }, - "outputs": [], - "source": [ - "class MHAPyTorchScaledDotProduct(nn.Module):\n", - " def __init__(self, d_in, d_out, num_heads, block_size, dropout=0.0, qkv_bias=False):\n", - " super().__init__()\n", - "\n", - " assert d_out % num_heads == 0, \"embed_dim is indivisible by num_heads\"\n", - "\n", - " self.num_heads = num_heads\n", - " self.block_size = block_size\n", - " self.head_dim = d_out // num_heads\n", - " self.d_out = d_out\n", - "\n", - " self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)\n", - " self.proj = nn.Linear(d_in, d_out)\n", - " self.dropout = dropout\n", - "\n", - " self.register_buffer(\n", - " \"mask\", torch.triu(torch.ones(block_size, block_size), diagonal=1)\n", - " )\n", - "\n", - " def forward(self, x):\n", - " batch_size, num_tokens, embed_dim = x.shape\n", - "\n", - " # (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)\n", - " qkv = self.qkv(x)\n", - "\n", - " # (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)\n", - " qkv = qkv.reshape(batch_size, num_tokens, 3, self.num_heads, self.head_dim)\n", - "\n", - " # (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)\n", - " qkv = qkv.permute(2, 0, 3, 1, 4)\n", - "\n", - " # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)\n", - " queries, keys, values = qkv.unbind(0)\n", - "\n", - " use_dropout = 0. if not self.training else self.dropout\n", - " context_vec = nn.functional.scaled_dot_product_attention(\n", - " queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=True)\n", - "\n", - " # Combine heads, where self.d_out = self.num_heads * self.head_dim\n", - " context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, num_tokens, self.d_out)\n", - "\n", - " return context_vec" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "fbc8ba92-3471-41cb-b1b2-4c0ef5be392b", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "fbc8ba92-3471-41cb-b1b2-4c0ef5be392b", - "outputId": "83ef0a2f-3fe6-4123-c8de-f481f2a9e415" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "torch.Size([8, 1024, 768])\n" - ] - } - ], - "source": [ - "mha_pytorch_scaled = MHAPyTorchScaledDotProduct(\n", - " d_in=embed_dim,\n", - " d_out=embed_dim,\n", - " block_size=context_len,\n", - " dropout=0.0,\n", - " num_heads=12,\n", - " qkv_bias=False\n", - ").to(device)\n", - "\n", - "out = mha_pytorch_scaled(embeddings)\n", - "print(out.shape)" - ] - }, - { - "cell_type": "markdown", - "id": "351c318f-4835-4d74-8d58-a070222447c4", - "metadata": { - "id": "351c318f-4835-4d74-8d58-a070222447c4" - }, - "source": [ - "## 5) Using PyTorch's torch.nn.MultiheadAttention" - ] - }, - { - "cell_type": "markdown", - "id": "74a6d060-6324-48fa-a35c-cb09f2a48965", - "metadata": { - "id": "74a6d060-6324-48fa-a35c-cb09f2a48965" - }, - "source": [ - "- Below, we use PyTorch's [torch.nn.MultiheadAttention](https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html) implementation" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "3799c7ef-3155-42c6-a829-f95656453ae0", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "3799c7ef-3155-42c6-a829-f95656453ae0", - "outputId": "aabf134e-c9bc-474b-ee57-0c24b5fb604c" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "torch.Size([8, 1024, 768])\n" - ] - } - ], - "source": [ - "import torch.nn as nn\n", - "\n", - "\n", - "class MHAPyTorchClass(nn.Module):\n", - " def __init__(self, d_in, d_out, num_heads, block_size, dropout=0.0, qkv_bias=False, need_weights=True):\n", - " super().__init__()\n", - "\n", - " self.block_size = block_size\n", - " self.multihead_attn = nn.MultiheadAttention(\n", - " embed_dim=d_out,\n", - " num_heads=num_heads,\n", - " dropout=dropout,\n", - " bias=qkv_bias,\n", - " add_bias_kv=qkv_bias,\n", - " batch_first=True,\n", - " )\n", - "\n", - " self.need_weights = need_weights\n", - " self.proj = nn.Linear(d_out, d_out)\n", - " self.register_buffer(\"mask\", torch.triu(torch.ones(block_size, block_size), diagonal=1).bool())\n", - "\n", - " def forward(self, x):\n", - " batch_size, num_tokens, _ = x.shape\n", - "\n", - " # Ensure attn_mask is compatible with expected shape and `batch_first=True`\n", - " # No need to manually adjust for num_heads; ensure it's right for the sequence\n", - " if self.block_size >= num_tokens:\n", - " attn_mask = self.mask[:num_tokens, :num_tokens]\n", - " else:\n", - " attn_mask = self.mask[:self.block_size, :self.block_size]\n", - "\n", - " # attn_mask broadcasting will handle batch_size dimension implicitly\n", - " attn_output, _ = self.multihead_attn(\n", - " x, x, x, attn_mask=attn_mask, need_weights=self.need_weights\n", - " )\n", - "\n", - " output = self.proj(attn_output)\n", - "\n", - " return output\n", - "\n", - "\n", - "mha_pytorch_class_default = MHAPyTorchClass(\n", - " d_in=embed_dim,\n", - " d_out=embed_dim,\n", - " block_size=context_len,\n", - " dropout=0.0,\n", - " num_heads=12,\n", - " qkv_bias=False\n", - ").to(device)\n", - "\n", - "out = mha_pytorch_class_default(embeddings)\n", - "print(out.shape)" - ] - }, - { - "cell_type": "markdown", - "id": "a3953bff-1056-4de2-bfd1-dfccf659eee4", - "metadata": { - "id": "a3953bff-1056-4de2-bfd1-dfccf659eee4" - }, - "source": [ - "## 6) Using PyTorch's torch.nn.MultiheadAttention with `scaled_dot_product_attention`" - ] - }, - { - "cell_type": "markdown", - "id": "d2164859-31a0-4537-b4fb-27d57675ba77", - "metadata": { - "id": "d2164859-31a0-4537-b4fb-27d57675ba77" - }, - "source": [ - "- Set `need_weights` (default `True`) to need_weights=False so that MultiheadAttention uses `scaled_dot_product_attention` [according to the documentation](https://github.com/pytorch/pytorch/blob/71d020262793542974cf13b30f2a9099773f015c/torch/nn/modules/activation.py#L1096)\n", - "\n", - "> need_weights: If specified, returns ``attn_output_weights`` in addition to ``attn_outputs``.\n", - " Set ``need_weights=False`` to use the optimized ``scaled_dot_product_attention``\n", - " and achieve the best performance for MHA.\n", - " Default: ``True``." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "4a4c2afe-5e1f-4bd7-a118-67031176f147", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "4a4c2afe-5e1f-4bd7-a118-67031176f147", - "outputId": "5b577a7c-4199-4e52-8d08-a0974a5a3685" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "torch.Size([8, 1024, 768])\n" - ] - } - ], - "source": [ - "mha_pytorch_class_noweights = MHAPyTorchClass(\n", - " d_in=embed_dim,\n", - " d_out=embed_dim,\n", - " block_size=context_len,\n", - " dropout=0.0,\n", - " num_heads=12,\n", - " qkv_bias=False,\n", - " need_weights=False # NEW!\n", - ").to(device)\n", - "\n", - "out = mha_pytorch_class_noweights(embeddings)\n", - "print(out.shape)" - ] - }, - { - "cell_type": "markdown", - "id": "8877de71-f84f-4f6d-bc87-7552013b6301", - "metadata": { - "id": "8877de71-f84f-4f6d-bc87-7552013b6301" - }, - "source": [ - "## Quick speed comparison (M1 Macbook Air CPU)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a97c0b2e-6593-49d8-98bc-2267b3aa610f", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "a97c0b2e-6593-49d8-98bc-2267b3aa610f", - "outputId": "ebe635b2-5c03-4e9b-da3a-951d308acf7b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.15 s ± 86.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "## 1) CausalAttention MHA wrapper class from chapter 3\n", - "%timeit mha_ch03_wrapper(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19db9c2c-8e75-431a-8eef-0b4d8284e6e6", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "19db9c2c-8e75-431a-8eef-0b4d8284e6e6", - "outputId": "c6e7bcff-661c-45a6-da82-b1e3f89cf761" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "273 ms ± 3.63 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "## 2) The multi-head attention class from chapter 3\n", - "%timeit mha_ch03(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa526ee0-7a88-4f34-a49a-f8f97da83779", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "aa526ee0-7a88-4f34-a49a-f8f97da83779", - "outputId": "92b634f8-43f8-468f-87a1-bb774b64c212" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "324 ms ± 17.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "## 3) An alternative multi-head attention with combined weights\n", - "%timeit mha_combined_qkv(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cc2b4256-16d8-4c34-9fd0-d4b4af0e60fa", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "cc2b4256-16d8-4c34-9fd0-d4b4af0e60fa", - "outputId": "80c6e314-0771-470e-b090-628984ce2d85" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "106 ms ± 598 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "## 4) Multihead attention with PyTorch's scaled dot product attention\n", - "%timeit mha_pytorch_scaled(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f209e70-ebb6-4a1a-b608-1ff42e41c01d", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0f209e70-ebb6-4a1a-b608-1ff42e41c01d", - "outputId": "3cd37b53-04d4-4dd0-9450-6fc8ebaac083" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "351 ms ± 7.88 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "## 5) Using PyTorch's torch.nn.MultiheadAttention\n", - "%timeit mha_pytorch_class_default(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3f4968c2-8d40-4ab9-8dba-052b4f77d756", - "metadata": { - "tags": [], - "id": "3f4968c2-8d40-4ab9-8dba-052b4f77d756", - "outputId": "2e86bdb4-7fa0-4051-b000-4a2b591060a2" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "333 ms ± 14.4 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "## 6) Using PyTorch's torch.nn.MultiheadAttention disabling `need_weights`\n", - "%timeit mha_pytorch_class_noweights(embeddings)" - ] - }, - { - "cell_type": "markdown", - "id": "a78ff594-6cc2-496d-a302-789fa104c3c9", - "metadata": { - "id": "a78ff594-6cc2-496d-a302-789fa104c3c9" - }, - "source": [ - "## Quick speed comparison (Nvidia A100 GPU)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "707a2a14-a089-48a8-88aa-d328e1e0a9d0", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "707a2a14-a089-48a8-88aa-d328e1e0a9d0", - "outputId": "07a711f6-f7ff-496c-ce16-be67308aeadf" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "41.1 ms ± 5.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "## 1) CausalAttention MHA wrapper class from chapter 3\n", - "%timeit mha_ch03_wrapper(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "8686dd69-3655-40e4-a57b-a2c55532a010", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "8686dd69-3655-40e4-a57b-a2c55532a010", - "outputId": "b0c29336-55e8-4194-89e4-9201f77e5375" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "6.58 ms ± 256 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "## 2) The multi-head attention class from chapter 3\n", - "%timeit mha_ch03(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "2209d7df-e54b-4910-ae2b-c78cf684d9bf", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "2209d7df-e54b-4910-ae2b-c78cf684d9bf", - "outputId": "ba357440-47d4-450d-b859-08031056ccf8" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "7.19 ms ± 590 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "## 3) An alternative multi-head attention with combined weights\n", - "%timeit mha_combined_qkv(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "1075abe2-4839-4fd6-af3e-c09bb3651e26", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "1075abe2-4839-4fd6-af3e-c09bb3651e26", - "outputId": "b2126630-7fae-4c44-8180-226ff5509d78" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "2.37 ms ± 569 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" - ] - } - ], - "source": [ - "## 4) Multihead attention with PyTorch's scaled dot product attention\n", - "%timeit mha_pytorch_scaled(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "868e3670-8edc-47bc-9e06-eb505e44dc9d", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "868e3670-8edc-47bc-9e06-eb505e44dc9d", - "outputId": "453d9b7b-3f45-4907-b4fd-77d395534d6b" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "6.66 ms ± 301 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "## 5) Using PyTorch's torch.nn.MultiheadAttention\n", - "%timeit mha_pytorch_class_default(embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "944870e6-de54-4e3b-a455-b8f21f6f92c8", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "944870e6-de54-4e3b-a455-b8f21f6f92c8", - "outputId": "ccfe127c-c069-4dcd-f37d-ea6a40406955" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "4.52 ms ± 317 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "## 6) Using PyTorch's torch.nn.MultiheadAttention disabling `need_weights`\n", - "%timeit mha_pytorch_class_noweights(embeddings)" - ] - }, - { - "cell_type": "markdown", - "id": "dabc6575-0316-4640-a729-e616d5c17b73", - "metadata": { - "id": "dabc6575-0316-4640-a729-e616d5c17b73" - }, - "source": [ - "## Speed comparison (Nvidia A100 GPU) with warmup" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "29b63d3d-6d0b-43bb-9c68-d5514dc81000", - "metadata": { - "id": "29b63d3d-6d0b-43bb-9c68-d5514dc81000" - }, - "outputs": [], - "source": [ - "# CUDA benchmark code shared by Andrei Aksionov\n", - "# and based on code from\n", - "# https://github.com/cuda-mode/lectures/blob/main/lecture1/pytorch_square.py\n", - "\n", - "def time_pytorch_function(func, *input, num_repeats = 1_000):\n", - " # CUDA IS ASYNC so can't use python time module\n", - " start = torch.cuda.Event(enable_timing=True)\n", - " end = torch.cuda.Event(enable_timing=True)\n", - "\n", - " # Warmup\n", - " for _ in range(5):\n", - " func(*input)\n", - " torch.cuda.synchronize()\n", - "\n", - " start.record()\n", - " for _ in range(num_repeats):\n", - " func(*input)\n", - " torch.cuda.synchronize()\n", - " end.record()\n", - " torch.cuda.synchronize()\n", - " return start.elapsed_time(end) / num_repeats" - ] - }, - { - "cell_type": "code", - "source": [ - "\n", - "import matplotlib.pyplot as plt\n", - "\n", - "\n", - "embeddings_cuda = embeddings.to(torch.device(\"cuda\"))\n", - "\n", - "functions = {\n", - " \"1) MHA wrapper class\": mha_ch03_wrapper,\n", - " \"2) MHA Ch03\": mha_ch03,\n", - " \"3) MHA with combined QKV weights\": mha_combined_qkv,\n", - " \"4) MHA with PyTorch scaled_dot_product_attention\": mha_pytorch_scaled,\n", - " \"5) PyTorch MHA class defaults\": mha_pytorch_class_default,\n", - " \"6) PyTorch MHA with need_weights=False\": mha_pytorch_class_noweights\n", - "}\n", - "execution_times = [time_pytorch_function(fn, embeddings_cuda) for name,fn in functions.items()]\n", - "\n", - "\n", - "# Plotting\n", - "\n", - "# Customize further for dark mode aesthetics\n", - "plt.rcParams['figure.facecolor'] = '#121212' # Dark figure background\n", - "plt.rcParams['axes.facecolor'] = '#121212' # Dark axes background\n", - "plt.rcParams['axes.edgecolor'] = 'white' # White axes border\n", - "plt.rcParams['axes.labelcolor'] = 'white' # White labels\n", - "plt.rcParams['text.color'] = 'white' # White text\n", - "plt.rcParams['xtick.color'] = 'white' # White x ticks\n", - "plt.rcParams['ytick.color'] = 'white' # White y ticks\n", - "plt.rcParams['grid.color'] = '#444444' # Lighter grid lines for contrast\n", - "plt.rcParams['lines.linewidth'] = 2 # Thicker plot lines for visibility\n", - "plt.rcParams['lines.markersize'] = 8 # Larger markers for visibility\n", - "\n", - "fig, ax = plt.subplots()\n", - "bars = plt.bar(functions.keys(), execution_times)\n", - "\n", - "plt.ylabel('Execution time (ms)')\n", - "plt.xticks(rotation=45, ha=\"right\")\n", - "\n", - "# Calculate new ylim with a margin\n", - "max_execution_time = max(execution_times)\n", - "upper_ylim = max_execution_time + 0.2 * max_execution_time # Adding a 20% margin\n", - "\n", - "plt.ylim(0, upper_ylim) # Setting new ylim\n", - "\n", - "# Annotate bars with execution times\n", - "for bar in bars:\n", - " yval = bar.get_height()\n", - " plt.text(bar.get_x() + bar.get_width()/2, yval + (0.05 * upper_ylim), round(yval, 2), ha='center', va='bottom')\n", - "\n", - "\n", - "plt.tight_layout()\n", - "plt.savefig(\"1.pdf\")\n", - "plt.show()\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 488 - }, - "id": "CDJAPZaszaqx", - "outputId": "47c9ef93-438e-4455-faef-7e253eaaaa8d" - }, - "id": "CDJAPZaszaqx", - "execution_count": 11, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnAAAAHWCAYAAAD3vrTNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAADOcklEQVR4nOzddVxU2fvA8c/QIiiiImJ3166urt1YoIiYKFhgEKJiY2FhoCgqa3d369q6dneLiihigoEIzO8Pf9wvI5jrLOHzfr18fXfu3Ln3zPlezjz33HOeozI3N1cjhBBCCCFSDZ3kLoAQQgghhPg+EsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyesldgJQme/bsvH79OrmLIYQQQohflImJCY8ePfriPhLAJZA9e3YuXbqU3MUQQgghxC+uZMmSXwziJIBLIL7nrWTJktILJ4QQQoj/nImJCZcuXfpqHCIBXBJev35NZGRkchdDCCGEECJJMolBCCGEECKVkQBOCCGEECKVkQBOCCGEECKVkQBOCCGEECKVkQBOCCGEECKVkQAujfH09OTZs2eMHj1a2dahQwc2btxIcHAwz549I0OGDF89Tq9evdi9ezf37t3j2rVrLF68mIIFC2qz6EIIIYT4RhLApSHlypXDyckpUTLidOnSsXfvXiZPnvzNx6pcuTJz586lfv362Nvbo6enx5o1azA2Nv7ZxRZCCCHEd5I8cGlE+vTpCQoKwsvLi969e2u899dffwFQpUqVbz5ey5YtNV67ublx48YNypQpw9GjR/99gYUQQgjxw6QHLo0YP348f//9NwcOHNDK8eMfu7548UIrxxdCCCHEt5MeuDTAzs6O0qVLU7duXa0cX6VSMXr0aI4dO8a1a9e0cg4hhBBCfDsJ4FI5KysrxowZg729Pe/fv9fKOSZMmECxYsVo3LixVo4vhBBCiO8jAVwqV7ZsWSwsLNi3b5+yTU9Pj8qVK9OlSxeyZ89OXFzcDx/fz8+P+vXr06RJE0JDQ39GkYUQQgjxL0kAl8odPHgw0eSEwMBAbt68SUBAwL8O3ho3boytrS3379//t0UVQgghxE8iAVwq9/r160Tj0t68ecPz58+V7RYWFlhYWJAvXz4AihcvzuvXrwkJCeHly5cArF+/nq1btzJnzhzg42NTe3t7HB0def36NRYWFgBEREQQFRX1H307IYQQQiRFArhfgLOzM/3791deb926FfiYGmT58uUA5M2bF3Nzc2WfTp06AbB582aNYyX8jBBCCCGSh8rc3Fyd3IVIKUxNTQkODiZv3rxERkYmd3GEEEII8Yv51lhE8sAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyWp2Fmjt3bv78809y5syJsbExT58+5eLFi5w8eVJrqwYIIYQQQqR1WgngWrRogaurK2XLluXJkyc8fvyYqKgoMmXKRN68eXn//j1r1qwhICCAkJAQbRRBCCGEECLN+ukB3L59+/jw4QPLly/Hyckp0fJLBgYGVKhQATs7O/bs2YO3tzebNm362cUQQgghhEizfnoeuFq1ammsy/klmTJlInfu3Jw/f/5nFuGHSR44IYQQQiSnb41FtNID961evHjBixcvfnYRhBBCCCHSNK3OQi1dujTFihVTXjds2JDFixczZMgQ9PX1tXlqIYQQQog0S6sBnL+/PwULFgQgT548zJ49m7dv32Jra8vw4cO1eWohhBBCiDRLq2lEChQowMWLFwFo2rQpR48exdXVlT/++IM5c+YwePDgHzqup6cnQ4cOJSgoSDmGoaEhvr6+2NnZYWBgwL59+/D29iY8PPynfZ+fxbTrouQuQooSObtDchdBCCGESFW02gOnUqnQ0fl4iho1avD3338D8PDhQ8zNzX/omOXKlcPJyYlLly5pbB89ejTW1tZ06tQJW1tbLC0tWbhw4b/7AkIIIYQQKZBWA7hz587Rp08fWrZsSeXKlZUALk+ePD/UM5Y+fXqCgoLw8vLi5cuXynZTU1PatWvHkCFDOHToEOfPn8fd3Z2KFStSvnz5n/V1hBBCCCFSBK0GcIMGDaJ06dL4+fnh7+/P3bt3AbC1teXEiRPffbzx48fz999/c+DAAY3tZcuWxcDAQGP7zZs3efDggQRwQgghhEhztDoG7sqVK1SrVi3R9mHDhhEbG/tdx7Kzs6N06dLUrVs30XsWFha8f/+eiIgIje3h4eFky5bts8c0MDDA0NBQeW1iYvJdZRJCCCGESA5aDeASSp8+vTIeLt63Jsu1srJizJgx2Nvb/9Q1VHv16kX//v1/2vGEEEIIIf4LWl/M3s/PjypVqmBkZKRsV6lUqNVqLCwsvuk4ZcuWxcLCQiNJsJ6eHpUrV6ZLly44ODhgaGhIhgwZNHrhsmbNSlhY2GePO2XKFGbOnKm8NjExSTQ5QgghhBAipdFqABcUFIRKpcLDw4Pw8HDU6h9btevgwYNUqVJFY1tgYCA3b94kICCAhw8fEh0dTY0aNdi8eTMABQsWJFeuXJw6deqzx42OjiY6OvqHyiSEEEIIkVy0GsCVKFGCOnXqcOvWrX91nNevX3Pt2jWNbW/evOH58+fK9qVLl+Lr68uLFy+IjIxk3LhxnDhx4osBnBBCCCFEaqTVAO7s2bPkyJHjXwdw32Lw4MHExcWxYMECjUS+QgghhBBpjcrc3PzHnmt+g7x58zJp0iRWr17N1atX+fDhg8b7V65c0dapf4ipqSnBwcHkzZv3mydY/NB5ZCUGDbISgxBCCPHRt8YiWu2By5IlC3nz5mXatGnKNrVa/d2TGIQQQgghxP9oNYCbOnUqFy9exMXFhSdPnvzwJAYhhBBCCPE/Wg3gcubMSbt27ZQVGIQQQgghxL+n1aW0Dh06RMmSJbV5CiGEEEKIX45We+B27tzJqFGjKFasWJKTGHbs2KHN0wshhBBCpElaDeAmTZoEkGQ6D5nEIIQQQgjxY7QawGXNmlWbhxdCCCGE+CVpdQycEEIIIYT4+X56AGdnZ/fN+1pZWfHHH3/87CIIIYQQQqRpPz2A69ixI0ePHsXd3Z3ChQsnet/U1JS6devy119/sW/fPszNzX92EYQQQggh0rSfPgbO1taWBg0a0LVrV3x8fHj79i1Pnjzh/fv3mJmZYWFhwbNnz1ixYgVVq1YlPDz8ZxdBCCGEECJN08okhh07drBjxw7Mzc2pVKkSOXPmJF26dDx79oyLFy9y4cIFWZVBCCGEEOIHaXUW6vPnz9m2bZs2TyGEEEII8cuRWahCCCGEEKmMBHBCCCGEEKmMBHBCCCGEEKmMBHBCCCGEEKnMfxLA6evrU7BgQXR1df+L0wkhhBBCpGlaDeDSpUtHQEAAISEh/PPPP+TMmROAcePG4enpqc1TCyGEEEKkWVoN4Hx8fChZsiS2trZERUUp2w8cOECzZs20eWohhBBCiDRLq3ngGjVqRJcuXTh16pTG9mvXrpEvXz5tnloIIYQQIs3Sag9c5syZk1wqy9jYWFZiEEIIIYT4QVoN4M6dO0f9+vWV1/FBW/v27Tl58qQ2Ty2EEEIIkWZp9RHqqFGjWLVqFUWKFEFXVxdXV1eKFClChQoVsLW11eaphRBCCCHSLK32wB0/fpwaNWqgq6vL1atXqVWrFk+fPqVBgwacP39em6cWQgghhEiztNoDBxAcHIyXl5e2TyOEEEII8cv4TxL5ZsmShaJFi1K8eHGNf9+jY8eOHDx4kODgYIKDg9mxYwd16tRR3jc0NGT8+PHcvHmTe/fusWDBArJmzfqzv4oQQgghRLLTag9cmTJlmD59OoULF0alUmm8p1arsbCw+OZjhYaGMnLkSO7cuYNKpaJ169YsWbKEmjVrcv36dUaPHk29evXo1KkTERER+Pn5sXDhQho1avSzv5YQQgghRLLSagA3depUbt++jaenJ0+ePPlXqUN27typ8Xr06NF07NiR8uXLExoaSrt27XBxceHQoUMAuLu7c+zYMcqXL58oD50QQgghRGqm1QAub968ODs7c/fu3Z96XB0dHZo2bYqxsTGnTp2ibNmyGBgYcODAAWWfmzdv8uDBgy8GcAYGBhgaGiqvTUxMfmo5hRBCCCG0QasB3MGDBylZsuRPC+CKFSvGjh07MDIy4s2bN3To0IHr169TsmRJ3r9/T0REhMb+4eHhZMuW7bPH69WrF/379/8pZRNCCCGE+K9oNYDz9PRk+vTpFC1alGvXrvHhwweN93fs2PFdx7t16xY1a9YkQ4YM2NraMn369H+VT27KlCnMnDlTeW1iYsKlS5d++HhCCCGEEP8FrQZwFSpUoGLFitStWzfRe987iQHgw4cPSm/e+fPnKVeuHC4uLmzYsAFDQ0MyZMig0QuXNWtWwsLCPnu86OhooqOjv6sMQgghhBDJTatpRMaNG8fq1aspXrw4WbNm1fj3vcFbUnR0dDA0NOTcuXNER0dTo0YN5b2CBQuSK1cumcAghBBCiDRHqz1w5ubmzJw5M8kF7b+Xj48Pu3fvJiQkBBMTE1q0aEGVKlVwcHAgMjKSpUuX4uvry4sXL4iMjGTcuHGcOHFCAjghhBBCpDlaDeC2bNlC1apVCQ4O/tfHypIlCzNmzCBbtmxERERw5coVHBwc2L9/PwCDBw8mLi6OBQsWYGBgwL59+/D29v7X5xVCCCGESGm0GsDdvn0bHx8fKlWqxJUrV4iJidF4f9asWd98LE9Pzy++//79e/r160e/fv1+qKxCCCGEEKmFVgM4R0dH3rx5Q+XKlalcubLGe2q1+rsCOCGEEEII8ZFWA7jffvtNm4cXQgghhPgl/SeL2QshhBBCiJ/np/fA+fr6MnbsWN6+fYuvr+8X9/Xx8fnZpxdCCCGESPN+egBXqlQp9PT0lP8WQgghhBA/108P4Jo1a5bkfwshhBBCiJ9Dq2Pgpk6diomJSaLtxsbGTJ06VZunFkIIIYRIs7QawLVu3RojI6NE242MjGjVqpU2Ty2EEEIIkWZpJY2IqakpACqVChMTE96/f6+8p6OjQ7169Xj69Kk2Ti2EEEIIkeZpJYC7c+cOarUatVrNiRMnEr2vVqvx8/PTxqmFEEIIIdI8rQRwTZs2RaVSsWHDBpydnXnx4oXyXnR0NCEhITx+/FgbpxZCCCGESPO0EsAdOXIEgHLlyhESEqKNUwghhBBC/LK0OolBgjchhBBCiJ9PltISQgghhEhlJIATQgghhEhlJIATQgghhEhlJIATQgghhEhltDILNV7WrFkZOXIk1atXJ0uWLKhUKo33LSwstHl6IYQQQog0SasBXGBgIDlz5mTixImEhYWhVqu1eTohhBBCiF+CVgO4SpUq0bhxYy5duqTN0wghhBBC/FK0Ogbu4cOHiR6bCiGEEEKIf0erAdygQYMYOnQouXLl0uZphBBCCCF+KVp9hDp37lzSpUvH6dOneffuHR8+fNB4v2DBgto8vRBCCCFEmqTVAG7w4MHaPLwQQgghxC9JqwHcihUrtHl4IYQQQohfklYDOAAdHR0aN25M4cKFAbh27Rrbt28nLi5O26cWQgghhEiTtDqJIV++fBw9epTp06fTuHFjGjduzMyZMzly5Ah58+b9rmP16tWL3bt3c+/ePa5du8bixYsTjaEzNDRk/Pjx3Lx5k3v37rFgwQKyZs36E7+REEIIIUTy02oAN3bsWIKDgyldujS1a9emdu3alClThnv37jF27NjvOlblypWZO3cu9evXx97eHj09PdasWYOxsbGyz+jRo7G2tqZTp07Y2tpiaWnJwoULf/bXEkIIIYRIVlp9hFq5cmWsra15+fKlsu3FixeMHDmSbdu2fdexWrZsqfHazc2NGzduUKZMGY4ePYqpqSnt2rXDxcWFQ4cOAeDu7s6xY8coX748p06d+tffRwghhBAiJdBqD1x0dDQmJiaJtqdPnz5RSpHvlSFDBuBjQAhQtmxZDAwMOHDggLLPzZs3efDgAeXLl0/yGAYGBpiamir/kiqrEEIIIURKo9UAbteuXUyePJnff/9d2Va+fHkmTZrEjh07fvi4KpWK0aNHc+zYMa5duwaAhYUF79+/JyIiQmPf8PBwsmXLluRxevXqRXBwsPJPlvwSQgghRGqg1UeoAwYMYMaMGezYsUPpcdPT02PHjh0MHDjwh487YcIEihUrRuPGjf9V+aZMmcLMmTOV1yYmJhLECSGEECLF02oAFxERgaOjI/nz56dQoUIA3Lhxg7t37/7wMf38/Khfvz5NmjQhNDRU2f7kyRMMDQ3JkCGDRi9c1qxZCQsLS/JY0dHRREdH/3BZhBBCCCGSg9bzwAHcuXOHO3fu/Ovj+Pn50bhxY2xtbbl//77Ge+fOnSM6OpoaNWqwefNm4ONSXbly5ZIJDEIIIYRIU356AOfr68vYsWN5+/Ytvr6+X9zXx8fnm487YcIE7O3tcXR05PXr11hYWAAfe/mioqKIjIxk6dKl+Pr68uLFCyIjIxk3bhwnTpyQAE4IIYQQacpPD+BKlSqFnp6e8t8/S6dOnQCU3rV4bm5uLF++HPi49mpcXBwLFizAwMCAffv24e3t/dPKIIQQQgiREqjMzc3VyV2IlMLU1JTg4GDy5s1LZGSk9s7TdZHWjp0aRc7ukNxFEEIIIVKEb41FtJpGZOrUqUnmVjM2Nmbq1KnaPLUQQgghRJql1QCudevWGBkZJdpuZGREq1attHlqIYQQQog0SyuzUE1NTYGPCXdNTEx4//698p6Ojg716tXj6dOn2ji1EEIIIUSap5UA7s6dO6jVatRqNSdOnEj0vlqtxs/PTxunFkIIIYRI87QSwDVt2hSVSsWGDRtwdnZW1iuFj8lzQ0JCePz4sTZOLYQQQgiR5mklgDty5AgA5cqVIyQkRBunEEIIIYT4ZWl1JYZcuXKRK1euz75/9OhRbZ5eCCGEECJN0moAt2nTpkTb1Or/pZ2LX01BCCGEEEJ8O60GcPnz59d4ra+vT+nSpRk4cCCjR4/W5qmFEEIIIdIsrQZwSWUQ3r9/P9HR0fj6+lKnTh1tnl4IIYQQIk3SaiLfzwkPD6dgwYLJcWohhBBCiFRPqz1wxYsX13itUqnIli0bnp6eXLp0SZunFkIIIYRIs7QawB04cAC1Wo1KpdLYfurUKTw8PLR5aiGEEEKINEurAVy5cuU0XsfFxfHs2TONpbWEEEIIIcT30WoAJ0l8hRBCCCF+Pq1OYhg7diwuLi6Jtnfp0kXSiAghhBBC/CCtBnA2NjYcP3480fYTJ05ga2urzVMLIYQQQqRZWg3gMmXKRERERKLtkZGRmJuba/PUQgghhBBpllYDuLt37yaZrLdu3brcu3dPm6cWQqRAZ8+e5dmzZ4n+jR8/Psn927dvz5YtW7h9+za3b99m3bp1/Pbbb4n2K1y4MEuWLOHu3bvcv3+f3bt3kyNHDm1/HSGESDZaDeBmzJjBsGHD6N+/P5UrV6Zy5coMGDAAHx8fZs6cqc1TCyFSoLp161KsWDHlX/PmzQHYuHFjkvtXqVKFdevW0bRpUxo0aMDDhw9Zs2YN2bNnV/bJmzcvW7du5ebNm9ja2lK9enUmTpwos93Fv5Y9e3aCgoK4efMmISEhHDp0iLJly37xMy1atODAgQM8ePCAy5cvM3XqVDJlyqS8/603JUJ8jcrc3Fz99d1+XMeOHenduzeWlpYA3L9/n/Hjx7Ny5UptnvaHmJqaEhwcTN68eZNcBuynnafrIq0dOzWKnN0huYuQpOzZszNs2DDq1KlDunTpuHv3Lu7u7pw7d+6znzEwMMDb2xsHBwcsLCwICwtjwoQJLFu2DIA2bdoQGBio8ZmoqKhftrdo9OjR1K9fnwoVKnzT/jo6Oty5c4f+/fsrbcjs2bOJiYmhe/fu2iyq+MVkzJiR/fv3c/jwYebPn8/Tp0/Jnz8/wcHBBAcHJ/mZP/74gy1btjBkyBB27NhB9uzZmTRpErdv38bJyQmAoKAgTpw4wYkTJ3j//j0eHh40btyYKlWq8OjRo//wG4qU6ltjEa2mEQGYP38+8+fPJ3PmzERFRfHmzRttn1KIfy1jxoxs27aNw4cP06pVK6Xxfvny5Rc/N2/ePLJmzYqnpyd37twhW7Zs6OhodnRHRERQsWJF5bVardV7qBRLX18fBweH7+qNNzY2Rk9PjxcvXgAfV3epX78+U6dOZfXq1ZQqVYr79+8zZcoUtm3bpq2ii1+Ap6cnDx8+xN3dXdl2//79L36mQoUK3L9/n1mzZin7L1y4UCNxfbdu3RKdx8bGhurVq6fIjg2Rcml9LVRdXV1q1KhBkyZNlBUZLC0tSZ8+vbZPLcQPS9h4nzlzhvv377N///7P3nkD1K5dm8qVK9OqVSvlEcqpU6c4ceKExn5qtZonT54o/8LDw7X8bVKmRo0akTFjRpYvX/7Nnxk2bBiPHz/mwIEDAGTNmhUTExM8PT3Zs2cPLVq0YOvWrSxcuJDKlStrq+jiF9CgQQPOnTvHvHnzuHbtGvv27aN9+/Zf/MzJkyfJkSMHdevWBT5enzY2Nvz999+f/cynNyVCfCutBnA5c+bk8OHDLF68mPHjx5M5c2YAPDw8GDFihDZPLcS/8iONd8OGDTl37hweHh5cunSJ48ePM2LECIyMjDT2S58+PefOnePChQssWbKEIkWKaPOrpFiOjo7s3r2bx48ff9P+np6e2NnZ0aFDB2V8W3zv5vbt2wkKCuLSpUsEBASwc+dOnJ2dtVV08QvIkycPHTt25M6dOzg4ODB//nzGjh1L69atP/uZEydO4Orqyty5c3n8+DHXrl0jIiKCfv36ffYzn96UCPGttJ7I99y5cxQoUICoqChl+9atW6levbo2Ty3Ev/IjjXeePHmoWLEiRYsWpUOHDgwePBhbW1smTJig7HPz5k08PDxwdHSkW7duqFQqduzYgZWV1X/xtVKMnDlzUqNGDZYsWfJN+/fs2RNPT09atGjBlStXlO3Pnj3jw4cP3LhxQ2P/mzdvkjNnzp9aZvFr0dHR4cKFC4waNYqLFy+yaNEiFi9e/MUbgyJFijB27FgmTJhA7dq1adGiBblz52bSpElJ7p/UTYkQ30qrAVylSpWYNGkSHz580Nh+//59jVlk3+LPP/9k6dKlXL58mWfPntGoUaNE+wwYMIDLly8TEhLCunXryJ8//78qv/h1/UjjraOjg1qtxtXVlTNnzrB79258fHxo3bq10gt36tQpVq5cyaVLlzhy5AhOTk48ffpUGeD8q2jbti3h4eHs2rXrq/u6u7vTt29fWrZsmWgCyYcPHzh79iwFCxbU2F6gQAEePHjwM4ssfjFhYWFcv35dY9uNGze+eGPQq1cvjh8/TmBgIFeuXGHfvn14e3vj6OhItmzZNPb93E2JEN9KqwGcjo4Ourq6ibZbWVnx+vXr7zqWsbExly9f/mxXtIeHBy4uLvTt25f69evz9u1bVq9ejaGh4Q+VXfzafqTxDgsL49GjRxqzhm7cuIGOjs5ne9hiYmK4ePEi+fLl+zkFTwVUKhVt27Zl5cqVxMbGarw3Y8YMfHx8lNceHh4MHDgQDw8P7t+/j4WFBRYWFhpjaAMDA2nWrBnt27cnX758dOnSBWtra+bNm/effSeR9hw/fvy7bwzSpUtHXFycxrb4azx+DDh8+aZEiG+l1QBu3759uLq6Kq/VajXp06dnwIAB7N69+7uOtWfPHsaMGcPWrVuTfN/V1ZVJkyaxfft2rly5Qvfu3bG0tEyyp06Ir/mRxvv48eOJJugUKFCA2NhYQkNDk/yMjo4OxYsXJyws7OcUPBWoUaMGuXLlYunSpYney5Ejh0ZPRceOHTE0NGTBggVcvXpV+dezZ09ln61bt9KnTx/c3d05dOgQjo6OODs7J7mMnxDfKigoiPLly+Pl5UW+fPmwt7enQ4cOzJ07V9nHx8eHGTNmKK937txJkyZN6NixI3ny5OGPP/5g3LhxnD59Whnr+S03Jb8ST09Pnj179tX10TNkyMD48eO5fPkyoaGhHD9+XJksAt+fJDwt0GoakaFDh7J69WqOHDmCoaEhs2bNIn/+/Dx//pyuXbv+tPPkyZMHS0tLjUGgkZGRnD59mgoVKrB+/fqfdi7xawgKCmL79u14eXmxYcMGfvvtNzp06EDv3r2VfXx8fMiePTs9evQAYO3atfTt25dp06bh5+eHubk5w4cPZ+nSpcoY0L59+3Lq1Cnu3r1LxowZcXNzI2fOnN88Fiwt2L9/vzKh6VNNmzbVeF2uXLlvOuayZcuUXHtC/Axnz56lQ4cO+Pj40LdvX+7fv8/gwYNZs2aNsk+2bNk0cjguX74cExMTunTpwsiRI4mIiODQoUMak/YS3pQk5Ofnl6aDjaSUK1cOJycnLl269MX99PX1WbduHeHh4XTs2JFHjx6RK1cuXr16pexTt25djSd+xYoVY926dZ9NEp4WaDWACw0NpXr16tjZ2VGiRAlMTExYsmQJa9as0ZjU8G9ZWFgAJErHEB4erryXFAMDA41HrCYmJj+tTCJ1+5HG+82bN9jb2zNu3Dh2797Nixcv2LBhA2PGjFH2MTMzY8qUKVhYWPDy5UvOnz9Pw4YNEz2uFUIkv127dn1xnKabm1uibbNnz2b27Nmf/cy33pSkdenTpycoKAgvLy+NG+OktGvXDjMzMxo0aEBMTAxAoqchz54903gdn4vzn3/++bkFT0G0GsBlzpyZZ8+esWbNGo0fPvgYHV+9elWbp/+qXr160b9//2Qtg0i5fqTxvnnzJvb29p/9zJAhQxgyZMhPKZ8QQqRW48eP5++//+bAgQNfDeAaNGjAqVOnGD9+PA0bNuTZs2esXbuWgICARGMO4ceShKdGWh0Dd+jQIerVq5doe8+ePb+Y2PB7PXnyBPiYNDGhrFmzKu8lZcqUKeTNm1f5V7JkyZ9WJiGEEEIkZmdnR+nSpfH19f2m/fPmzYuNjQ26urq0bt2aiRMn0qNHD/r06ZPk/j+SJDw10moAN3PmTBYsWMDEiRMxMjIie/bsrF+/Hnd3d43JDf/WvXv3ePz4sUZuOVNTU37//XdOnjz52c9FR0cTGRmp/PvembFCCCGE+HZWVlaMGTMGV1fXb859p1KpePr0KV5eXpw/f54NGzbg7+//2bRO35skPLXS6iPUadOmsX//fmbOnMnBgwfJlCkTp0+fpnr16l/sGUtK+vTpNVIt5M6dm5IlS/LixQsePnzIX3/9RZ8+fbhz5w737t1j0KBBPH78WNZDFEIIIVKIsmXLYmFhwb59+5Rtenp6VK5cmS5dupA9e/ZEj0XDwsKIiYnR2H7jxg0sLS3R19fXyDUbnyT8V8itqfXF7O/evcvVq1exsbEBYMOGDd8dvMHH/9M3bdqkvI6fcrx8+XLc3NyYOnUqxsbG+Pv7kzFjRo4fP07Lli0lu7UQQgiRQhw8eJAqVapobAsMDOTmzZufHdN24sQJ7O3tUalUqNVq4GOKpsePHydaKOB7koSndloN4P744w+CgoJ48eIF1atXV3Li1K1blz59+mhMAf6af/7557OpB+KNGzeOcePG/dtiCyGEEEILXr9+zbVr1zS2vXnzhufPnyvbZ8yYwaNHj5QxcvPmzaNLly6MHTuW2bNnkz9/fry8vBLN9v1SkvC0SKtj4DZs2MCGDRuwtrbmxo0bLFmyhJo1ayqL3AshhPj3evXqxe7du7l37x7Xrl1j8eLFiRJRf4mdnR3Pnj1j8eLFn91n4sSJPHv27KeOXxYiKZ8m9A4NDaVFixaUK1eOgwcPMnbsWGbNmsWUKVM0PvelJOFpkVZ74Fq0aMGRI0c0tgUHB9OwYcOvThsWQgjxbSpXrszcuXM5c+YMenp6DBkyhDVr1lC5cmXevn37xc/mypWLkSNHJmqrE2rcuDHly5fn0aNHP7voQiRK4P3pa/i4jrS1tfUXj/OlJOFpkVZ74D7XIKjVaiZNmqTNUwshxC+jZcuWLF++nOvXr3P58mXc3NzIlSsXZcqU+eLndHR0+Ouvvxg3bhz37t1Lcp/s2bMzbtw4XF1dE403EkIkH60EcCtWrMDU1FR57enpSYYMGZTXmTJl+uLdnhBCiB8X396+ePHii/t5e3vz9OnTzz5yUqlUzJw5k2nTpslqIUKkMFp5hFq7dm0MDQ2JjIwEUNaTjIiI+HhSPb3vGp8hhEgepl0XJXcRUpzI2R2SuwhfpFKpGD16NMeOHUs0WDyhihUr4ujoSI0aNT67j6enJzExMcyaNUsbRRVC/AtaCeBUKtUXXwshhNCOCRMmUKxYMRo3bvzZfUxMTJg5cya9evXi+fPnSe5TpkwZXFxcqF27traKKoT4F7SeB06I/4L0FCWW0nuKxM/n5+dH/fr1adKkCaGhoZ/dL2/evOTJk4dly5Yp23R0Po6oCQsLo2LFilSqVImsWbNy/vx5ZR89PT18fX3p1q1biluUXdqAxKQNSNu0EsCp1Wol2V7CbUIIIbTDz8+Pxo0bY2try/3797+4782bNxMlUx08eDAmJiYMHDiQhw8fsmrVKg4cOKCxz5o1a1i1apVG4CeESB5ae4QaGBhIdHQ0AIaGhkyaNEmZzm5gYKCN0wohxC9pwoQJ2Nvb4+joyOvXr7GwsAAgIiKCqKgoQDM56vv37xONj4tPrB6//cWLF4kmQXz48IGwsDBu3bql7a8khPgKrQRwK1as0Hi9evXqRPusXLlSG6cWQohfTqdOnQDYvHmzxnY3NzeWL18OfEyOmtQyRUKI1EkrAZy7u7s2DiuEECIJ35K8NKnkqAm5ubl99RgpbdybEL8yrSbyFUIIIYQQP58EcEIIIYQQqYykERFCCCFEkiQ9S2IpJT2L9MAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQyEsAJIYQQQqQykshXCCH+Y5IcVVNKSYwqRGoiPXBCCCGEEKmMBHBCCCGEEKmMBHBCCCGEEKlMmgvgOnfuzNmzZ3n48CG7du3it99+S+4iCSGEEEL8VGkqgGvWrBm+vr5MmDCB2rVrc+nSJVavXk2WLFmSu2hCCCGEED9NmgrgevToweLFi1m2bBnXr1+nT58+vHv3jnbt2iV30YQQQgghfpo0E8Dp6+tTpkwZDhw4oGxTq9UcOHCAChUqJGPJhBBCCCF+rjSTBy5z5szo6enx5MkTje1PnjyhUKFCSX7GwMAAQ0ND5bWJiYnG/2qLiWGaqfafw9T0Xx9C6jQJUq/aIfX680mdaofUq3b8hHr9km+NQX7p/2d69epF//79E22/dOlSMpTmFzYiOLlLkDZJvWqH1OvPJ3WqHVKv2vEf1auJiQmRkZGffT/NBHDPnj0jJiYGCwsLje0WFhaJeuXiTZkyhZkzZ2psy5QpEy9evNBaOVMKExMTLl26RMmSJXn9+nVyFyfNkHr9+aROtUPqVTukXn++X7FOTUxMePTo0Rf3STMB3IcPHzh//jzVq1dn27ZtAKhUKqpXr86cOXOS/Ex0dDTR0dEa274U7aZFr1+//uW+839B6vXnkzrVDqlX7ZB6/fl+pTr9lu+ZZgI4gBkzZjB9+nTOnTvHmTNncHV1xdjYmGXLliV30YQQQgghfpo0FcBt2LCBLFmyMGDAACwsLLh06RItW7YkPDw8uYsmhBBCCPHTpKkADmDOnDmffWQq/uf9+/f4+fnx/v375C5KmiL1+vNJnWqH1Kt2SL3+fFKnSVOZm5urk7sQQgghhBDi26WZRL5CCCGEEL8KCeCEEEIIIVIZCeCEEEIIIVIZCeCEEEIIIVIZCeDSEJVKldxFEOKbyLUqhBD/TppLI/KrUqlUqNUfJxTXqVOHkJAQbt26RWxsbDKXLHWzsbEhf/786OrqsnnzZm7evJncRUr1El6rHTp04OnTp+zdu5eoqKhkLlnqlrBexc+TsF5NTU1/mZUAtEmu1Z9DArg0Iv6PYciQITg4ODBy5EhCQ0OlsfkXhg4dioODA+fOnaNq1apUqFABR0dHCYr/pfhrddiwYbRs2ZKAgACMjIwkgPsXEv4g/vnnnxgbG3P16lUePXokP5T/QsJ69fLyIn/+/IwfP54HDx4kc8lSr4R12rRpU6ysrDA0NGTfvn2cP38+mUuXukgAl4b06dOHtm3b4uzszMWLF3n37l1yFynV6tOnD61ataJNmzZcuHCBokWLsmvXLiwtLXn48GFyFy/Vc3FxoU2bNtjb23P58mVA7sr/jfh6GzFiBPb29piamnL9+nXWrFnDvHnziImJSeYSpk6f3myMHTuWDx8+JHOpUrdP6/TAgQMUKFAAOzs7li9fTlBQUDKXMPWQAC6NyJgxIzVq1MDPz48TJ05gaWlJ6dKlcXBw4MaNG2zYsIEnT54kdzFTheLFi1OhQgW8vb25cOECAK9eveLGjRu4urqio6PD2bNnWbt2bTKXNPX4NDgrWbIkCxcu5PLly+TJk4dy5crh4uLCjRs32LVrF9u2bUvG0qZOlStXpnLlynTs2JEXL17g5uaGnZ0dJiYmTJ06VYK4H9SgQQNatmxJ27ZtlR4iExMTsmTJwosXL3j16lUylzD1sbW1pXnz5kqd2tvbExgYSEhISHIXLVWRAC6V+vQHUVdXF3Nzc8zNzbGxscHGxgYrKyuMjY0pV64cWbNmZcyYMdLD8Q0ePnzIokWLOHbsGPCxrteuXYtarUZHR4eiRYtSpUoVAAnivoGhoaGyBE6tWrXYt28fWbJkoXTp0jx48IAWLVoQHR3N7du3KVy4MGZmZuzZs0eWzfkOjRs3pl69ehw+fJiTJ08CMHjwYIYMGUL9+vVRq9VMmzZNgrhv8Gnbam5uzs2bNzl//jwlSpTA2tqa1q1bo6+vz549e/Dz85P1tr9Trly5OHv2LOfPn6dp06ZMnDiRgQMHsmXLFtKlS0fu3Lm5fv16chczxZNZqKlQwgamdu3aZM+enefPn7N+/XqcnZ0JDAzkwYMHjBs3jtq1a3Pnzh3MzMwkePtGr1694u+//+b58+cAtG7dmrCwMGxsbBgyZAgtWrQgJiaGGjVqJHNJU76GDRuyYMECAEaNGsXEiRMxMDDAw8ODN2/e4ObmxoEDBxg7dizu7u4sWbIEc3NzdHSkafpWxsbGdOzYEXt7e4oUKaJsf/PmDaNGjeLMmTPUrVuXQYMGSb1+Rd68eZV2skePHpQrV46QkBCqVKlCUFAQK1asoFChQkybNo2ZM2fSsGFDzM3Nk7nUKVtSM87Tp09PSEgI5cuXZ+rUqYwcOVJpJ5o2bYq1tTXGxsb/cUlTH+mBS4U+nbAwduxY1q1bx6RJk9iyZQsfPnzgzp07yv6ZMmWSrumvKFeuHJkyZeLhw4fK5A8dHR3i4uJYs2YNq1evJiYmRtl248YN3rx5k9zFTvFCQkL4888/OXz4MDly5KBRo0ZER0fz9OlTmjRpgpmZGS9evAA+9iI3bdqUhw8fyvjN7/D27Vu6devGqFGjKFeuHE5OTixcuBD4GMSNHj2aCRMmkDFjRuLi4pK5tClXiRIl2L9/P126dOGPP/6gVatWNGjQgLNnz9KlSxfq1q2Lr68vBw8e5PHjx5ibm9O6dWtMTEySu+gpVsLOhsqVK3Pt2jWeP3/OoUOH2LRpE66urnTu3JlNmzYBkC5dOpo3b87t27d5+/ZtchY9VZDF7FMpb29vOnXqRIcOHbh27Vqi2aYZM2akQIEC9O3bl1y5clGzZk2ZPfkZw4YNo1mzZhgbG/Py5UseP35M7969uX37dpID662srFi8eDGLFy9W7hrF582bNw8bGxsOHTpEixYtEgURJiYm1K9fnxYtWpArVy5q1aolj/o+I+H1aGlpyZs3b9DX1+f58+dYWFjg5+dHlixZWLFiBUuXLlU+Z2RkxPv376UXPgnZs2fn0aNHAHh4eODt7U1sbCyNGjXiypUrSp3H37zp6OhgaGjIggULMDY2xtbWVur1KwYPHky9evVYuHAhy5Yt4/379/Ts2ZNBgwYxZMgQ9u/fT4YMGRg8eDBZs2albt268nv1DaQHLhXKlCkTNWrUYNiwYZw8eRILCwsKFy5MixYtOHv2LPv37ydv3ryMHj2a58+fU6tWLWJjY5UGSPyPvb09jo6OtG/fnrt37/LHH3/Qrl07/v77b5o3b865c+eUeosfuLx48WJu3bolwds32rx5M9u2bcPX15eFCxfSs2dPIiIilPfNzMyoWLEi0dHRyo2Grq6uNOBJiA8UvL29qV+/PhkzZiQiIoIJEyawc+dO+vfvj5+fH61bt0atVrNs2TIAJUWLzPTVNHXqVMqWLUvnzp25efMmjx8/xsjIiLi4OIoWLcqVK1eU+oqLi8PAwIBOnTrRuHFjjI2Nsba2Rq1WS71+wcCBA3FycqJDhw5cuXJFGdu6cOFCDA0NGT58OK9fvyY8PJynT59Sr149+b36RtIDlwply5aN/fv34+/vz+3bt3FwcCB//vyYmJigo6PDvHnzmD17NuXLl+f06dOo1Wr5QfwMDw8PypcvT4cOHZRtuXLlYvjw4dSuXZv69etz8+ZNDA0N8fDwoEGDBty9e5cuXboA8oP4qYTXmZmZGa9fv1Z608qVK8eKFSs4ceIEPXr0UHqN7ezs2L17t/JaGu4v69u3L66urnh7e2NmZkapUqVo3749vXv3ZsmSJVhaWjJ69GiKFy/OsGHD2LVrV3IXOcXKmTMnO3fu5Pr163h6evLgwQMsLCxo06YNgwYNwsvLSwmCAQwMDKhUqRK1atVi1KhRcrPxiVq1anHmzBllZm6BAgWYPXs2w4YN49ChQ2TOnBkrKysaNWrEgQMHOHbsGPny5SNz5sxERkZy48YN+b36DhLApXCfCxD69++Pi4sLenp6zJs3j/3793PgwAGWLl3Kw4cP6dev31ePIT7+GDo7O1OmTBmNBiNHjhyMHz+erFmz0qpVK168eEHu3LmpWLEiq1evBqReE6pTpw5nz55VJn706dOHKlWqYGZmxpQpUzh27BhPnjyhbNmyrFixgnPnzjFjxgx69uxJpkyZlJ4MoenTBMcZMmRg+fLlLF++nCVLlgAfr0MvLy8GDhxI48aNlTRCnTp1Yty4cRIMf4aenh4xMTFkz56dvXv3cuvWLXr16sXt27eBj21s79698fDwYOXKlQCMHj2aNWvWcPbsWUBuNhLq0KEDvr6+DBs2jLVr1xIZGYmVlRW7du1izJgxXLhwga5du/L7778DUKRIEVq2bMm+ffs0jiPt6reTKUkpWMILuWTJklSqVImCBQsC4Ofnh729PfXq1WPEiBEcOHAA+NjgP3v2TOM48sfwefv27SM8PJxu3bphZGSkbH/48CHz5s3DxMSE/PnzA3D//n0J3pLg6OjI/PnzsbOzQ19fnw4dOtCtWzf27NlDaGgoI0eOpFOnTlhZWXHu3DmaNm1K0aJFGTVqFBkyZKBRo0ZSl0lYu3Ytffr00dhmbGxM0aJFiY6OVrap1WpmzJjB/v37sbW1RU9Pj8ePHzNmzBhlzJbQpFKplJ7hR48eUadOHQoWLMjEiRMpVKgQ8LGNnTRpEoGBgUyYMIHt27dTu3ZtJTckIMFbAosWLWLVqlV0796dFi1akDFjRp48ecKWLVvo378/O3fuVGZGV61alSNHjlC1atVEx5G24NtJD1wqMGTIEBo2bIilpSUXL17k0qVL+Pj4KBe6iYkJBQsWxNvbm9y5c8uEhS/4NPDS0dHB39+fUqVKMXPmTDZv3qyM0TAxMeHkyZOMGDGCFStWJFeRU4WxY8dSr149pkyZQokSJdizZw+7d+8GPi5B1LJlSzZv3szChQt5+PChkutJHpl8XunSpbl27RrR0dFKbxF8nBRiZGREr169NJJzL1iwgNevX+Pm5pZcRU51KlasyMOHDwkJCcHKyoo9e/Zw/fp1vL29lXWPO3bsSP369Xn69CleXl4as9HFRwnrY+LEidSsWZPp06ezZMkSjIyMlI6H+J5LPT09Nm3axLp165gzZ06ylTu1k1uzFM7Ly4u2bdvSv39/ypQpw927d2nfvj0BAQFKfp2KFSvi6+uLgYGBxoQFoUlPT08j6DU3NycuLg5vb2/CwsLo2bMnTk5OSt2ZmZnx6tUrSdL5Bbq6usDHgcr79+/Hy8sLGxsbjX0mT57MqlWrsLGxoX379uTNm5d3795x/fp1ZQC4BG+aVCoVFy5cIDo6Gjc3NxYsWKCkq9i9ezfm5uZ0794dMzMz4OPYrEyZMhEWFpaMpU5d/vzzT+bPn0/btm2xsrIiNDSUOnXqUKRIESZMmEDhwoUBmD9/Pl27dsXd3Z2YmBh0dXUlePtEwp7evn37sn//ftzc3HB0dESlUnH27FnOnj1LunTpKFq0KIsWLSJdunTMnz8/mUueusmvfApSvXp1jddFihTB2tqanj17cvjwYcqXL0/z5s3ZsWMHFStWZNKkSahUKvbs2cOoUaNo1aqVNDCfMDY2pmHDhgBKD8bUqVNZv34969atw8nJiQ8fPuDk5MSlS5do06YNhw8fZvLkyaxfv57bt2+zZ8+e5PwKKdangVffvn3ZtGkTWbJk4c8//1SCC/gYxK1YsYIuXbpQrVo1jePII5PE4utER0eHc+fOUaNGDcaPH4+Ojg7Lli1j586dVK1alZ07dxIUFMSWLVvIkiULY8aMSeaSpx5Hjx5l2bJl2NjY0KZNG40grnDhwowZM4bixYsD8Pr1a+VzcrPxPwmT9Cb8zYkP4nr27Im9vT0ZMmQAoEmTJvj4+JA+fXqN2abix8gj1BSiadOmzJkzB09PT41ZT23btmXXrl0ULFiQuXPnMnbsWJYsWcLChQuxtrZm9+7dtG/fXmnwZWyWpg4dOjBp0iSlXidMmEClSpVYvHgxBQoUoFOnTvj7+zN27Fh0dHSoXr06derUQVdXlydPnjBlyhRA6vVLGjVqRExMjDLbcezYsVhbWzN9+nRWr16tkTKkZcuWrFmzRm4wPqNy5cqoVCr++ecffH19efDgAbNmzaJSpUosX76cXbt20a1bN9RqNdWqVaNKlSrkzJmThw8fMn78eEm/8A309fU1FqQfOHAgNjY2rF27lmXLlvHo0SOsrKw4f/48c+bMYeDAgclY2pQrYZtYvXp1LCwsePz4MdeuXePp06cATJo0ierVqzNjxgyWLl2Kubk5JUqUYN++fcTFxcnQiX9J8sClEBs3bqRgwYJKr1p8Es74YK5Pnz5s375dmQ11/fp1MmTIwIMHDzSOI0GGprVr12JhYcGUKVNQq9WEhobSvXt3Ll26BMD58+fx9/cHPgYe+/fvZ//+/RrHkOBNU8L6KF26NMOGDePmzZtERkZy9OhRBg4ciJ6eHt27dwfQCOJWrVoFyOy9pGTLlk2ZtODk5ISNjQ116tQB4NixY7Rp04bly5cTFBSEm5sbhw4d4tChQxrHkB/EL3N2dkZfX5/ly5crvWpjx45FpVLRqVMn4GObGxoaSrFixZRZ1SKx+DZg6NChtGrVisePH5MjRw527NjBihUrOHbsGH369GHixIl069YNY2Nj5syZozzRkKET/54EcCnIpEmTlEH1gEYm9Vy5cmFoaMiHDx9QqVQULFiQ9evXs2jRIkCCjM95+/YtEydORFdXl4CAAN68eaMRoMUHyBMnTiQ2Npbx48cnOobUq6b4+hgwYABZsmQBPqYRMTAwQF9fn4MHD+Lt7c348eNxcXHB2NiYefPmaSw9JsFbYmFhYfj5+REUFETVqlXx8vLiypUrwMe/7/ggbtmyZUyZMoUBAwYkWoFFfhC/rHr16pQpU4a3b9+yceNGJYgbM2YMxYoVw9HREVNTU6ZPn66MfZWbDU158uTh3r17APTs2RMHBwecnZ05efIkffv2pVevXpiZmaGrq8s///xD3759mTNnDr///jvTp09XjiPt6r8nAVwy+zTwmjBhAiqVKlEQt3fvXtq3b8/GjRvR19cnY8aMSjJZkD+GTzVv3pxChQqRKVMmhg4dyvTp04mMjGTYsGFUqFBBmQ0F/wviAgICuHz5Mlu3bk2uYqcaLi4uuLq60rp1a6ZOnUrx4sUZPHgwXbp0IS4ujsOHD9OvXz+CgoIoW7asrBv7jV6/fs3Dhw959OgRtra2PHjwgEOHDilLOR07doy2bduyefNm7t69y8SJE5O7yKlKp06dCAgIwMPDAx0dHTZs2KAEwcHBwRQoUIDMmTNrTFyS4O1/nJycaNu2LU5OTsTExFCqVCnGjh3LyZMnadSoEd27d2fZsmXUqVMHNzc31Go1R44coUuXLkkuai/+HQngklHC4M3BwQE9PT1WrlzJ+PHjiYuLw9/fH5VKxZIlS1i/fj1qtZrffvuNt2/fMmjQIGXmjzQwmnx8fKhfvz6bN29m7969REdHEx0dzcKFC0mXLh2jR4/m9evXGmMNly1bxt27dzl69Ggyljxlatu2rUZdAZQvX57t27dz/Phx4GOOvKioKKZNm4aHhwcAhw8fplu3bjJI+Qs+vYG7evUqNjY2VK9ene7du+Pp6Ylarebw4cPK3/mxY8eoUaMG169fT65ip3gJ6zVjxozo6Ojw6tUr4uLi8PT0JDAwEDc3N3R0dNixYwdhYWFkzZqVfv36cfjw4WQufcrUoUMHJk6ciJOTE48fP0ZXV5f58+dz48YNSpcuzejRo/Hz82PWrFn06NEDb29vDA0N8fX15ezZs7LkmBZIAJeM4i/k4cOHY2dnx/Tp07G0tCQ0NFS5s44fE7d48WLmz5+vMe1axrsk5uXlhaOjI61bt+b8+fMawe2bN2+YMWMGKpWKgIAAAI3AJD54k0bmf+Izpy9fvlxjokxUVBSmpqbKa7Vazf79+5k5cyYDBgzg1atXREdHc+LECeLi4qROPyO+Tpo1a0aGDBl4/fo169at4+DBgxgaGtKpUycl0Dh48CBLlixhx44dyioM0gYklrBO+vTpQ9WqVSlevDjLly/nwIED7Nu3Dzc3N/z9/enSpQvdunXj3bt3pEuXjh49egDSBnzK0dGR8ePH06FDB7Zv3w58fFx/6dIl3rx5g5OTEzdu3FCG9MTExHD+/HmuX7/OuXPnlONInf5cEsAlszZt2tCyZUvat2/P6dOnNd6LD+L8/PxIly4ds2bN0nhfGm5NefLkoUmTJgwbNkzjEWlC7969Y8aMGQBMmTJFGVibkDQy/7NixQrmzp2LWq2mUqVKHDt2DLVazYkTJ/D396dGjRrKKiAAkZGRHDt2jAIFCtCsWTNOnDgBSJ1+yciRI2nTpg1Pnz7F2NgYOzs72rdvz99//w187PmYOnUqL1++JEOGDDg7OyuflTbgf8qUKcP58+eVOhk0aBBOTk4MGTKEuLg4XFxcKF++POnTp2fLli307t0be3t7cuXKhb6+Pv7+/vJUIwk1a9Zk8uTJ9OzZUwne4GNC6S1btrBu3TqMjY0xMTEhV65c3Lx5k2rVqrFq1SrlBlkCYu2QAC6Z/fbbb+zatUsjeEt4sU+cOJGMGTNiY2OTKIATmrJnz07evHkTBcKfev/+PRMnTiRDhgyUK1fuPypd6hQ/Pig+D9mGDRuUVDbly5dn0aJFdO/enXPnzhEREYG1tTWrV69GV1eXadOmERQUxP3795P5W6Rc5ubmFClSBBsbG54/f07ZsmXx9/dn7dq12Nvb8/fff/Ps2TMKFy5Mjhw5mDJliiygnoRNmzZx6dIlLl68SFxcHLVq1cLW1pZ27dpx6tQpKlWqROnSpbly5Qo9e/YkOjqaXbt2sXbtWo3jSPCW2KNHj3j+/DmNGzdmy5YtvH37ljlz5lCmTBmGDh0KwKlTp7Czs2POnDkYGRkRGxurZEwAuYHTFgngkpm5ublGTiL4eLHr6+tTpUoVDh48iI+PTzKVLnUxNjb+6nirUqVK4ejoyODBgxk1apTGmpLi8y5fvszu3bupXr26Mlu3V69eREVFERQUpAz6/vDhA+vXr6ds2bLcuXNHWZZMJObi4oKDgwP379/nwYMHvHnzhj179uDm5sb06dNZs2YNLVq04MyZM5w5c0b5nI6OjgRvCbi4uJAvXz7s7e2Ji4tDX1+fkJAQVq1axalTp6hbty4zZ86kT58+XLt2jeXLl9O7d29MTExYt26dxrEkeEvs+vXr2Nrasm7dOmbNmoVarSZPnjzY2try8OFDAHbt2kVsbCwFCxbEyMiIwMBAyUn4H5AALpndvXuXdu3akSNHDuWPASBTpky0bduWDx8+8M8//yRjCVOPFy9eYGJiQuXKlZV1DD/1xx9/EBMTo6zKIBL79HGHgYEBT58+ZcKECfTq1Yv69esTFxfHxIkTGTBgANu2bSNTpkzo6emxfv164uLiaNasGZGRkURFRSXjN0m59PT0iIqKwtzcHBMTE2WWbmxsLIcOHaJHjx5MnTqVPXv2KLng4skPoiZTU1Pu3bvHhw8f8PX15c6dOyxevJjZs2djZGSEq6srQUFByuO8a9euYWlpye+//54ogBOJqVQqrl+/jr29PQsXLiR//vzUqlVL+b2KD9L27NmjsWqNBG/aJ9PDkpmfnx8hISGsXLmS4sWLky1bNiwtLZk2bRo5c+aUWZHf4ezZs6xZs4ZRo0ZRpUqVRO9bWFjQrFkzQkNDk6F0qUPC4K1z585MnjyZNWvWKDcTEydO5J9//qF+/fr069cPgIMHD7Jx40bWrl1LgQIFCAgIoFWrVnh4ePDq1avk/DopxqcpFGJiYli/fj3jxo0je/bsBAYGKu/FB3He3t48evRI0i98xZYtWyhXrhy7d++mW7duHDt2jJiYGCIjIzEwMCB37ty8fPkSgAwZMvDo0SPGjRvHkCFDkrfgKVz804z49uDatWt06NCBR48eMWjQIMzNzYHP31BI8KZ9EsBpWcLGN1u2bIne//DhA506deLp06esX7+e3bt3s3z5cjJnzoytra0yg098mzlz5nDx4kVWrFhBy5YtsbKywszMjJo1a7Ju3TpevHihkUxSaIpvrIcNG4aXlxfPnj1jz549BAQEMGTIEF6/fs2UKVM4evQotWrVYuTIkcpnjY2NyZEjBxkyZKBp06ZKEtpfXcKguHz58jRs2JBy5cqhUqlYvXo1/fv3p3bt2srMaPj44/f333/j6OiopF8QicX3Dh04cIAyZcqwdetWbty4obxvaGjI7du3qV69Ot26dWPOnDnkzp2bDRs2SL1+QXzvWalSpVi3bh16eh8f1l2/fp2WLVtSunRppk+fTqZMmZK5pL82WQv1P+Lj44OVlRX9+/fXWBsyIWtra9KnT09UVBQ7duyQteJ+UIUKFXB1daVp06Y8e/YMPT09QkNDuXDhAm5uboDMivqSSpUqMX36dLp06cLZs2cpVaoUe/fupUePHqxevRoAMzMzhg8fjlqtxsvLS/msrq4u+vr68ug0CUOHDqVZs2ZERERgaGhIcHAw48eP5+LFizRv3pyhQ4eye/duevXqldxFTVUyZ87M0KFDOXv2LKNGjWLNmjUMHz5c6XVr2LAh7du3J2fOnISGhuLo6EhMTIy0AZ8RH7wVLVqUtWvX8vfffye6JosUKcKqVasICwvD3t4+0Yog4r8hY+D+A1WrVqVevXp4eHgkGbzFNyQ7d+7U2C6DlZP2tYb35MmTnDx5kgULFpAjRw5iY2O5efMm58+f/6bP/+rSpUvHgwcPOHv2LM2aNSMgIIB+/fqxevVqTE1NKVSoEGfOnMHHx0dpuOPrNDY2Vq7ZJDg7O9OqVSs6derE8ePHGTx4MC4uLpiZmRETE8PmzZtRq9XMmDGDe/fuMXny5OQucor16d/vs2fP8PLyIi4ujuDgYJYuXYpKpWLkyJE8e/aM7du3888//6BWq5XrVW6MkxYfvBUrVowNGzawcuVKhg4dio6ODtOnT8fDw4MPHz5w/fp12rVrx4ABA5TlyMR/TwI4LWvZsiW//fYbR44c4dy5c0kO7PxcMCFjCDQVL16c4OBg3r59+037fy6jugRvX2ZkZET27Nlp2bIl48aNY/jw4UoC6SpVqtCmTRsGDRqkDGKWgPjz4uvmt99+Y9myZRw/fpxGjRrRpUsXhg4dyr59+0iXLh16enps2rSJp0+fauTVE5oSXmvOzs4UKFCA3Llzs3LlSs6cOcP+/ftp3bq1knh65MiRPH/+XOPGWRZRT9qnwduqVasYOnQoKpWKHTt2oKOjg76+vpI14dKlSzg6OgLSBiQXGQOnZfb29nTu3JmSJUtiYGAgQdkP6tmzJ3v27GH79u3UqVOHggULarwvY1n+nbZt27JhwwYA9u3bx507d5g+fTrTp09XgjdDQ0McHR2JiorSmDEtDffnGRkZAR97Nc+ePUulSpWYOXMmw4cPZ+HChejq6uLg4ECtWrV4//49+/btU5LJisQSjtHs378/r1+/5s2bNwwfPpy+fftibGzMoUOHaNWqFXZ2dvj7+ysrhnx6DPE/KpVKCd7WrVvHqlWr8PHxQaVSsXv3bl68eEGzZs0+e/MsdZo8pAfuJ0rqLqRVq1YEBARQv3592rRpw+rVq7+5B0l8FB+crVy5krt379K9e3eMjY3Zu3cvK1asICQkRBqQfykyMhI9PT0aNmzI9u3bWblyJWZmZtSoUYMLFy5gbm6Ovb092bNnp2bNmoDcdSelWrVqHDp0CPi4jNPz58+ZP38+ISEhBAUFoVKp6NWrl5JA1tTUFDs7O/bt26dxHLnR+7waNWpgY2OjLJdXo0YNmjdvzpEjR3j79i06Ojr8888/dOrUCXd3d3nE9w3UajUFChRgy5YtLF++XCN4e/bsGZ07d5Z6TIFkEsNPkvDHrESJEqjVaoyMjJQEnLNnz6Z48eIEBASwefNm3r17l5zFTXUqVKjAokWLsLGx4enTp1StWhU3NzeioqK4desWAQEBPHv2TILj72RmZsbLly/JmDEjU6dORVdXV3ksYmdnR5MmTahTpw6XL1/m4cOH9OjRg5iYGMnxlIRs2bKxadMmnj9/ztmzZ3FycqJevXpcuXIFU1NTAgMDqVixIlWrVuX9+/ekT5+egIAAzMzMaNSokTzWS0Lnzp05deqUMn4VwMbGhm7dutG4cWOaNWvGlClTGDFiBPPnz8fY2JhSpUpx/vx5jYk0crOhKan66Ny5M6ampkyZMgWVSsXff//N8+fP6dSpkwRvKZQEcD/ZoEGDaNiwIQYGBqRLl46tW7cycOBA4GOKi6JFixIQEMDWrVsl2PgGCRuaYcOGYWFhweDBg3n58iXlypVj586dPHnyhHfv3nH69Gl27NihPAoUX9a7d29atWpFnz59OHz4MFZWVhw6dIiZM2cq6/ACWFlZER4erox9kQHgSdPR0eG3335jzZo16Ojo0KRJEy5cuICenh4xMTFUqFCB4cOHU7x4ccLCwnj9+jWxsbE0btxYguIk/PnnnwQFBbF//36CgoK4evUq8HH9aAcHByZPnszixYsZOXIk8+bNAz4Gd3/++SeTJ09WVgcRmhK2qdbW1oSGhnLx4kWN9/ft20d4eDgdO3aU4C0Fk4EWP5GHhwfOzs707t2bWrVqsXbtWrp06aKst9mlSxeuXr3KqFGjqFSpUjKXNmWrWLEimTJlQq1WK+OBjh8/TtGiRYmMjCRLliwsXbqUJUuWULJkSaZMmYKJiQnW1tbJXPLUo0iRIuTPn5/AwED69etHnjx56N27Nw0aNKB69erKfo8ePdJY7k2CN03xj/jj4uKIjIwkLCyM8PBwfH190dfXV1b9OHnyJE2aNKFv374EBAQwYcIEGjZsSExMDLq6uhK8feLo0aP4+vpSsmRJunXrRsmSJQHYunUrhQoVYt26dfTt21cJ3gwNDWnbti0ZM2aU4O0L4oO3oUOHMmLECKpWrUqGDBmU69jBwYELFy5Iz1sqID1wP4mOjg6zZs1i165drFq1isaNGzN16lRGjhzJwoULMTY2VnrcBg4ciJ+fnzTYn1GtWjWmTJnC6tWrmTlzpkY2/9WrV6Ovr0/hwoXZt28f/fr1U5YhSp8+vfLf4uuyZcvGgAEDMDAw4MWLF+TPnx8DAwNevXrFnTt38PPzkyXHvuLPP/8EPgYbkydPJioqirFjx1K8eHEmTJjAq1evaNasmUY9ftqDKT1viSWc7dixY0fatGnDtWvXmDFjBteuXcPa2hp/f3+OHz/OrFmzMDc3x9nZGUtLS2rVqiU3GV/Rp08fXF1dadu2LefPn0+0Hrf0sqcOEsD9JOnTp+fo0aN4e3vz+vVrli1bxrBhw1iwYAF6enp4e3tz9OhR9u/fr3xGGu7PGzlyJH/++Se7du1i9uzZSlLOWrVqMXfuXDZv3oy3t7csRv+devfuTXR0NDt27ODWrVv07NmTbNmysWjRIjJkyICfnx9ly5YFPg4Wl9UUPs/U1JQ9e/Zw9+5dXr16Rd26dbG1teXSpUvo6upSrVo1Ro4cycuXL2nevDkxMTH4+/tz8uRJli9fntzFTxV69eqFpaUljRo1wtLSkrVr1+Lv78/NmzepXbs2vr6+mJqaEh4ezr1793BxcZHH0UlI+Ng0S5YsLFy4kJkzZ7JlyxasrKzInz8/Dg4OXL58mfnz5ycK6ETKJLNQf0BSA0DfvHnDunXrcHZ2pkqVKgwaNIglS5YAYG5uTtmyZQkJCdH4jDQwicXf+Q0dOpT+/fvToEED1Go1s2fP5tWrV1y5coVHjx7x4sULCd5+wIcPH3BycqJSpUps2rSJ+fPns2vXLh49esTMmTOxtrbGy8uLYsWKce3ateQubooWGRlJw4YNOXjwIJkzZ6Z3795cunQJ+N96pkOHDmXkyJGcPXuWu3fvkjt3bry9vZO55KlDz5498fT0pGPHjixdupSqVavi7OyMp6cnU6ZMYe/evRw4cIA8efLw8uVLnj9/DkjvUVLif6+srKx4/vw5RkZG1KhRg6dPn+Li4kKuXLl4+fIlbdu2JV26dBrLuomUS3rgvlPC4C179uzo6OgoObEaNmzIxIkTuXjxIr179yY0NJQsWbIwbdo0MmTIgI2NjQRt38DQ0JD3798DcOvWLZ4/f87q1auZO3cuz58/p3nz5owdO5Y2bdoos3zFtytbtiyNGzfG2dmZ1atXc+/ePTw8POjcuTPHjh3T2Fd+DD9PT0+PvHnzMnv2bNKnT8+NGzeYOXMm//zzj7KPjo4OBQoUoG3btsTGxjJ27FhiY2Olh+grdHV1WbJkCbdv39ZYdL5t27YMGzaMPXv2EBgYmKiHWGabaqpTpw6//fYbEyZMYOzYsZibm+Pt7U3r1q1p164dBQsWZNasWezbt4+DBw8yYcIEDA0N8fDwSO6ii28gPXDfKb5xGDx4MHZ2dqRPn57Q0FCmTp3Kxo0byZQpE7169WLFihW8evUKAwMD9PT0sLa2VhJ0SsOtqVevXqjVagICAtDR0eH9+/cYGBiwefNmTp8+ze3bt2nQoAEqlYpZs2Zx+PBh1Go1BQsWlADuM2rUqIFKpdJ4ZB/v3LlzXLt2jfXr1xMUFESZMmUwNjamVatWXLlyRSNrvQRvmhIGCDExMdy6dYtatWphZWXFqlWrcHd3R61Wc+TIEeBjL/vNmzcZMWKEcgxpA74uNjaWqKgojI2Ngf/V2bJlyyhdujQtWrTA2NiYkSNHcufOHeVzErz9T7p06ahUqRLNmjWjatWqlClTBmtrayIiIliyZAnr1q3DzMyMW7duKZ8pUqRIops4kXLJLNRvlDDTf6tWrejQoQPjxo3D1dWVe/fu4e3tTbdu3Vi2bBnu7u7MnTuX48ePM2/ePOrVqyczzb5AV1eXIUOG4OrqSlxcHCqViu3btxMREUHr1q0ZNGgQBw8epH79+nTu3Jnw8HBcXV1ZtWpVchc9xVGpVJiamjJ9+nQl4W5SoqKiuHLlCtbW1mzdupUnT56QJ0+eJNfqFR8lDN4KFSpEhQoVSJ8+PUZGRoSGhtKxY0dy5MhBz549qVGjBgCbN2+mb9++GseRNuDbXLp0iaZNm1KoUCGNOgsPD+fu3buEhYVx9+7dZCxhyvbu3TumTp3K06dPqVy5MitWrOD69esAvH//nqdPn3Lr1i3SpUtH+fLlWblyJRkzZsTPzy+ZSy6+lTxC/U6NGjUiS5YsACxatEjZPmrUKKytrenevTunTp1K9Dm5604s4Q+iq6srvr6+DB06FDs7O16+fJko+/ewYcNo2bIlvXv3ZufOnYmOIf6nZ8+euLm50bRpU27cuJHkPvHXpEqlIkeOHDx8+FDq8hsMGjSIpk2bYm5uTkhICMuXL2f9+vWEh4dTqFAhgoKC0NXVxdDQkNjYWGrVqiWDwn/QqlWrKFSoEM7Ozjx48IDIyEjmzp3L1q1bWblyJSBtwJdkypSJvn37YmRkRKVKlVi/fr2S4zE+P2Hjxo2xsbEhS5YstG7dWiaBpCISwH2HHDlycOzYMYyMjBg/fjwTJkzQGCP0999/ExwcTNeuXZO5pCmfj48P5ubm9OvXT/lx6969OyNHjuTBgwfUqFGDyMhIQHMcVrNmzSRR7zcoXrw4M2fOZMmSJcyePfubG2T5MfyyPn360KlTJzw8PNizZw9LliyhePHiyhjNJ0+ekDt3bmrUqEG6dOmYO3cusbGxMpbwB1lYWODv70/lypUJCwtDpVKhUqmoXLkysbGxcr1+4nP1YWFhQefOnWnatCmrV69m0qRJynt//PEHsbGxnDlzBrVaLddqKiIB3HfQ1dWlcuXK+Pn58fz5c+zt7Xn//r3yRzN27FiyZctGp06dkruoKVqRIkU4fPgw8LEXs1+/fkqD4ezszIQJExg4cCBz5sxRPvNpoyIN99dNmzaNP/74g4oVKyZ3UdKEIkWK4O/vz9SpU9m5cyc1a9ZkwYIFnD59mgIFCrB8+XLmzZuXKIms9GYklvDvN0+ePISGhn6xl9LGxgYzMzMMDQ2ZP3++TAT5RMaMGTXyZXbt2pWCBQuiUqmYMGEC4eHhWFlZ0aFDB2xsbNiyZQsTJkxgxYoV3Lhxg0GDBgHSrqY2EsB9RsILWUdHB5VKpQQQVatWZc6cOZw+fRo3NzfevXvHhw8f2L59O1evXsXT0zM5i54qTJ06FVNTU6pWrcrBgwfp2rWr0hh3796dESNGMGTIEGbNmpXMJU358uTJw71795TX8UlQCxQowMqVK5kxY4aSrV78uAwZMlCzZk12795N6dKlmTdvHuPGjWPRokWsWbOGAgUKsGPHDsaNG6fxYyo0JWxbvb29KV68OIsWLWL//v2JgofPBRQSvP3P4MGDcXV1pUKFCoSFhTFkyBA6dOjAsWPHKFiwIFmyZKFly5acO3cOKysrWrVqhaurK2/fvuX169fUrl1bEnanUjKJ4TPiGw0PDw/mzJnDtm3bcHR0pFChQhw+fFhZImvXrl0sXryYmTNnkj59evr06ZPMJU8dQkJCyJw5M23atKFSpUr89ddfypJZM2fOZOjQofj6+uLl5ZXMJU3ZSpQowalTp1i8eDGdO3cGUHoynjx5ws2bN6lVq1ZyFjHNiIiIYO/evbx9+5bWrVuzdetWJdfj/fv3effuHbq6uhK8fUXCpZy6dOnCqlWruHDhgkagFt8WfK43SIK3/1m5ciXnz59ny5Yt5MyZE1NTU1q2bKn0th05coTVq1fz22+/ERoayty5c2natCkjRoygZs2aygQ7kfpIAPeJhLNN+/bti4eHB/fv3+fevXt4enoyaNAgfv/9dw4fPoyLiwvv37+naNGiTJw4kSpVqsgfwzeaNGkSGTJkoGzZsnTp0oXatWszc+ZMpeEOCgpiwoQJFCxYMJlLmnLZ2NhQpUoVHB0d0dPTw8PDg3/++YcuXbpQoEABIiMjmTRpEjVq1KBJkybJXdw0IX5STaZMmUiXLh16eh8zMZmamjJkyBD69euXnMVLNapXr46dnR0ODg7KjHNLS0tq165NxowZlZRL4utu3bqFu7s7T58+ZdeuXfz+++/Kdfrs2TPc3d05fPgwK1asoFy5ckRERHD9+nU2btyo1LOMeUud5C/kEwkzVmfPnp1OnToxfPhwXFxcGDp0KCYmJnTt2pUsWbJw9OhRBgwYgJ6enkaySflj0DR8+HAWLFhA8+bNyZQpE/CxjjZs2ECxYsU4evQozs7O1KtXjxkzZigN98SJE+nZs2dyFj1FUqlUZMqUibFjx/Lw4UN27tyJi4sLzZs35+LFizg6OrJr1y769OmDubk5GzZsoFq1anJj8RPdvXuX0qVLM2vWLHbs2EGJEiWUnHsJbwJF0tRqNW/evCEiIoIiRYowYMAAtm3bhr+/P3v37sXc3Fx62b4i4XUWHBxMjx49OH36NCVLlsTQ0FDZJzIyEg8PDw4ePMiuXbsoVKiQxnGknlMvGQOXBBsbG+bNm8eTJ0/o2rWrkpQz/j1/f39atmzJ2bNnUalUVKlShZkzZ3Lr1i3s7OySseQpT8IJCzt37qRYsWJMmDCBEydO8Pr1a44fP07Hjh3Zv38/VapUYcGCBZw9e5aWLVsmc8lTNn19fU6fPo2Hh0eiZL1FixalXr16ODo68u7dO0qUKEFUVBR//PEHjx49Sp4CpwLfO4Dbx8eHrFmzEhcXR58+fWRg/WckrFcrKyvCw8MpUaIEkydP5s2bNxQtWpQtW7Zw4sQJrl27xuzZs/Hx8WHbtm3JXPLU4Y8//uDEiRMA5M+fn4CAAHLkyEHDhg0JCwtT9suYMSM9e/Zk3Lhxco2mERLAJcHAwIDx48fTrl07vLy8lHEu8U6cOMHSpUs11ourWbMmfn5+2NnZERoa+l8XOUVr06YN/v7+BAYG8uzZM2rWrEmOHDnYuHEjZcqU4cWLF/Tv3593795Rs2ZNmjRpkij5qdBkYGDA8ePHcXV1VRrvTwOQQoUKUaRIEby8vNDT06NWrVrScH9GwlnOX0uj8LkgTdIvJPbphIVChQoxa9YsTp06Re3atSlQoAD379/nyJEjREZGYmZmxvr16xkxYkSSq4gIzTotVqwYBw8eZPDgwcqEr7x58zJjxgyyZctGo0aNlPQrCdsGuVbThl8+gPvcXbehoSGBgYHUqlWLTp06cejQIdRqNWZmZuzcuZNp06YlCuzSpUvHu3fv/quipyqdOnVi3Lhx9OrVi3379pE3b1769+9PyZIluXTpEi1atJCZUF/RqFEj7t27x+XLl8mZMyf79++nWbNmygLq8ZK6puO3SQ9RYvXq1ePVq1ecOHGCMWPGkCVLFlxcXL76OUm58O2GDh1KmzZt6N+/P8eOHePJkyca7+vr65MxY0amTp1KpkyZaNy4sVynX+Hu7o6enh7e3t4AjBkzhsDAQOBjEDd9+nSyZMkinQpp2C8dwCVsgMuXL4+BgQFv3rzh/PnzwMe7lLlz51KzZk1WrFhBcHAw1apVI3fu3NSsWVPuYL5T165dGT16NL6+vkybNg0DAwOKFi3K/fv3efnyZXIXL0UzNDRk0aJFVK1aldq1axMSEsKVK1eoX7++sjzO10jAkbS9e/eSNWtWjh07Rq1atWjSpAnXrl375s/nz5+fsLAw3rx5o8VSpl41a9Zk6tSptGvXjosXL6Kjo0OWLFnIkycPwcHBhIeH4+bmRvXq1TEzM6NRo0ayGsBXeHt706VLFzw8PEifPj2lSpWiZ8+ejB49WnkylCdPHlatWsXly5clN2ka9UsvZp9wYXoHBwfevXtHvnz5mDhxIkuWLOHx48d07tyZwMBAOnfuzNq1a9m7dy8LFiyQ7Oo/YPbs2cTFxTFu3Dh0dHQICAjgwoULgAQXX/P+/Xu8vLwYM2YMmzZtolu3bly9epWGDRuSNWtWTExM0NfXJzIyEpVKRdGiRdm4caPGnbfUb9Jq167N5cuXadKkCb179/6u4K1r1660adMGR0dHCeA+Q1dXl7CwMJ4+fUqRIkWwt7dXetxfvHhBy5YtuXDhAjExMcyaNYu4uDhpWxPInTs39+/fV16nT5+e2rVrKwmlAdatW8fjx4/x9fUlOjqaoKAg7t27R7NmzTTGwYm05ZcO4AC8vLxo27YtnTt35tixY/j4+NC/f38yZcrE1KlTCQsLw8PDA5VKRbVq1Zg3b54yWFkamI++J/iaO3cuAKNHjyY2Nlbp8pfg4utCQ0MZMGAAEydOZNWqVcDHdBZdunRBX18fXV1d3r17h0qlIiwsjL/++iuZS5xyxV+zKpUKExMTwsLCeP78OV5eXgQHB3Ps2DHl/fhr89Pr3MnJiQEDBtC3b195RPUFMTExWFpaMmXKFMqVK8eOHTuYOHEi4eHhjBkzhpIlS3Lw4EEOHjwIIG1rAosWLSIiIgI3Nzdlm4GBATlz5tTonVSpVMydO5dq1aoxYsQIYmNjmTVrljJpSXoz06Zf7hHqp0u4jBo1ihUrVrB161YaN25MQEAAGzdupH379syePZvp06cTGhqKrq4uc+bM4Y8//sDFxYV//vknmb9JyqCnp8fKlSu5du0a9+7dY86cOUpD8aVGo1OnTvj5+dGmTRt27979XxY51cifPz/Zs2cna9asPH36VJnNmyVLFoYMGUKbNm1o1qwZx44dI0OGDMqNhUqlUpLJSs9mYgnrxMbGhqtXr3Lr1i0Atm/fTtasWXF3d+f48ePK9Ru/ukU8Jycnhg8fjru7O1u2bPnvv0QKlLBeTUxMiI6OJjo6GoAmTZpQsGBBbt26xeHDh3n58iUZM2Zk48aNDB8+XCYsfEbGjBl5+/YtHz58wNzcnOfPnwPg5+dHlSpVcHZ2Vq5dAF9fX0qVKkWVKlXo2LGjXJtp3C8XwMWLHzxvb2/P9u3bKV68OHPnziUwMJDZs2fj6+tL165dWbVqFcOHD+f58+fo6emxYsUKcufOTfXq1YmKikrur5Ei2NjYkCFDBgYNGsTly5c5dOgQf/31F9HR0V8M4qpXr67cdQtNrVu3VgYpZ8mShQwZMnDw4EHmzp3Ltm3byJIlCwEBAfz222+0bNmSixcvAnyxx0hoGjZsGE2aNGHlypUsWrRIGVi/fft2zM3N6devH2fOnGHmzJmEhYUpq6zEB28eHh5s3rw5Ob9CipHwWuvevTv16tVDR0eHkJAQpfdIT0+PmJgY9PT0SJ8+PUFBQWTMmJEmTZpI71ASEradLi4udOjQARcXF65cuUKlSpXw9vbmzZs3DB06lODgYAwNDZkzZw6LFi3C2tqasmXL0rx5cyIiIpL5mwht+WUCuIQNzMiRI+nWrRsFChQgJiaGd+/eMWzYMHLlyoWbmxtRUVH069eP8uXLY2xsjI2NjfJZXV1dsmXLJo9MkpApUya8vLwoX748L1++pHPnzrx79+6r3fcSaGhq2bIl/v7+eHt7888//xAbG0uZMmWYOHEikZGRjBw5kq1bt2JhYcHEiRP5448/cHBwUII48XUuLi707duXVq1acenSJT58+KAx7mrjxo3kz5+fN2/e8OHDB2rVqkVMTAy2trbMmDGDbt26Se9GEnx8fGjdujWBgYG8evWKwYMHc/XqVdq3b8+7d+9Ily4dPXr0oHLlymTIkIGGDRvKhIVvkDVrVg4cOMDt27fp1asXt2/fxsbGBmdnZ8qUKcOpU6fIlSsXcXFxVKtWjd69e9OgQQPq16+f3EUXWvTLBHDxChYsSLdu3Vi7di1Hjx4FPgYQixYt4sOHD7i4uBAbG6usbxr/qFSCjC+Lb4ANDQ2xtrbG3d2d9+/f06JFC6KioqT+vlHOnDlZuHAhCxYsYPHixRrvFS5cmLVr1xIaGkrr1q158eIFlpaWzJo1i3fv3tGqVatkKnXqYmBgwPTp07l69Sr+/v4a4+ESXqOtW7dGrVazZs0aJbDLnz8/uXLl4sCBA8lV/BQja9ashIeHK6/r16/P0KFD6dWrF6dOncLa2lqZlHDr1i1sbW159+4d9erVo3Tp0kyZMkUmgyXhc21l1qxZ2bdvHyEhIfTo0YM7d+6QN29eateuTdGiRQkPD2fKlCl8+PABf39/zMzM6NGjhzwpSsN+qQCuWbNmDBs2jIiICFq2bMmTJ0+UP5QWLVowc+ZMDh06hIWFBQA1atSQhuUzqlWrhrm5OXp6emzevFkZ6wIfg7kaNWowYMAALl68SP/+/aUev1HZsmVZsmQJbdu2VWbowv8C5N9//50dO3bg5ubGypUrgY89ny9fvpQA+TM+/UHU09Nj586dHDlyBB8fH419DQ0NKViwIJcvX9bYrqurS1xcnNTx/5s8eTI6Ojr4+/tz79494GOewqJFi+Lv70/dunWZMWMG48aN49KlS8oNs7OzM2/fvlWOIz1vmhJeq02aNCF//vx8+PCBM2fOcPz4cbJmzcqePXt4+PAhHh4e3Lx5U+PzWbJkwdPTk7Zt29KoUaNvTjEkUqdfai3Ud+/eERwcTN68eTE1NUWtVivrQ65Zs4bOnTtz584ddu7cqQRvsqByYkOGDGHy5Mn07t2bmTNn8tdffymLesc3yIcOHWLt2rUULlyY8uXLA7JG5LewtLTEyMiIyMhIAOX6i0+tcPr0ac6cOUOBAgWUz7x48ULpQRKJxf8gxq/Da2BgwIMHDyhUqBCZMmXSqLdcuXLh6elJ4cKFNY4RGxsrwVsCly5dUpKc58uXD4Bt27axcuVKjI2N8fLyYtasWcybN487d+4QHBxMrVq1mDBhgsZxJHjTFH+NDRs2DF9fX/7880/Kli3Lli1bsLW1JTw8nJo1a5I9e3b8/f0pXbq08tnMmTPj7OxMuXLlaNq0qQRvv4BfIjqxt7fHxsaGnTt3EhgYyI0bN/jrr7/Ily+fRpC2adMm+vbti6+vr9K1Lw2MJnd3d9q2bUvXrl1p0qQJFStWpE6dOrRr1w74X4McExPDkiVL0NXVpU2bNoCkCvkWN2/exMTEhGbNmgEf6zM+wIjvxVSpVEkmPpb6/bymTZty+PBhihYtytu3b5k2bRrVqlVjyJAhWFlZoaurS6ZMmRg5ciSZMmVK1LMhNM2dO5cxY8bQvHlzOnXqRP78+QF4+PAhlpaWZM+enb179wIfr8vLly9Tr1493N3dk7PYqYKNjQ0ODg506dKFNm3asGvXLgCMjY0BeP78OXXq1KFChQp06NBB+dyzZ89YsmQJHTp0SLQ6i0ib0nweOCMjI2UA7ebNm9mzZw/6+vp07dqVqVOn4u7uTnBwcJLjMOSxn6YiRYpQr149Bg4cyNmzZ9HV1eXu3bvs3LmTQoUKaeyro6PD27dvGThwIIGBgRQuXJgbN24kU8lTj4cPH7J+/XpcXV0JDg5m/fr1GoFZpkyZMDAwoGLFihgbG7N7926uXr3K+/fvk7HUKd+zZ8+4dOkSs2bNwtXVldOnT9OuXTvmz59P2bJlMTQ05M2bNxgaGlK3bt0kx8QJzUd8K1asQF9fn379+gEfg7rg4GAl91jfvn0JCgqiV69e6OjocP78eVnO7Rvky5ePvXv3cvr0aZo0aYK/vz+9e/dmxYoVmJqaYmlpyc2bNylcuDCvX7/W+Ozjx4+TqdQiOaS5HriEj0P09PSIioqiV69eVKlSRbn727FjB7NnzyYqKoqAgAAKFiwowdo3ePbsGW/fvuX27dvA/wLcp0+fKnfgCR/5xb93//59DAwMkqHEKVtSjzyjoqJYvHgxDx8+ZPjw4coddrp06bCwsCAwMJCsWbOSLVs2MmfOTNasWSV4+0RS9Xr48GEmTZpESEgIc+fOpWjRouzfv5+6desyf/58Nm3axKJFi6hTpw4xMTHo6upK8PaJhMFbpUqVAFi8eDFjx46lWbNmdO7cmfz58/Pu3Tu8vb0pWLAgfn5+6Ojo0KJFCyUoluAtafFtZ8IchdOnT2fYsGHKhKZ69erRrl07zMzMiIiIIC4uTob5/MLS7CQGFxcXdHV12bFjB3fv3sXFxYWWLVsydOhQjhw5AnycNTVgwABOnz6tLAgsvszExES564u/kx40aBAFChSgc+fOwMeu/rx583LlyhXg41qI165dk7vD/+fj48OaNWu4evXqZ3t5atasSc+ePalZsyY3btxAT0+Pp0+foq+vL6kBvpG9vT1Hjx7VSPlTsWJFPD09yZMnD506deL69euJ/j+QHqIvGzRoEM2aNeOvv/5SVlZp27YtgwYNYsOGDcyYMYPQ0FCMjIywsrLizp07ADLb9AtatGiBsbExixYtwsbGBh8fHywtLfH19WX27NnAx7Z3zpw53Lhxg6FDhyZziUVKkCYfoWbLlg0PDw+MjY1p0aIFvr6+7N27l6pVq1K1alXOnDlDVFQUu3bt4uXLl5w8eTK5i5xqfNplDx/Hu8UzMzNj9+7dLF++XAngJMv6/5QqVYrq1avz559/4unpyc2bN5MM4vbv38/169cpUqQINWrU4N27d1y7dk3JPSY/hprmz59PcHAwI0aMAD6mXHF3d8fJyQkXFxfl5uH48ePMmjWLadOmMWPGDNzc3Lh69arGsSR4+7y+ffvi5ORE+/btNdbnXLZsGWq1mkGDBhEXF8fixYu5efOmErypVCq5Xj9DV1eXZs2akTFjRhYtWsTmzZupVq0a7dq14927d5QoUQIdHR18fHzIkiWLMt5YiDTZA2dkZISrqyt//PEHJ0+epEePHgQEBFC2bFkqV66Mvb19ogWrZbzLjxsyZAhFihShZ8+ebN++nbCwMJo3b57cxUqxatWqRbdu3TAzM8Pd3Z0bN2581/UnPUSadHV16d69O0OGDMHPz4/JkycDHycuODk5oVKp6NGjhzI2S1dXlw0bNlCgQAEOHTqEq6trchY/1cicObOSnzB+LV7QvJlo27YtkydPZsiQIUrPkdCU8G/dwMCA6OhozMzMOHHiBMuXL2fYsGEATJ8+nZIlS1K0aFHOnj3Lu3fvcHBwkMTHQpGmHp47ODhQpkwZoqKiWLlyJQUKFODRo0fUr1+fXLly8f79eywsLAgKCsLU1FTjsxK8/bg3b96QMWNGNm/ezOPHj5XgTdJaaIpPtbJv3z6WLVtGZGQk/v7+5MuX77vSgEjDrSk2NpaZM2cycOBA+vXrpyx5tXHjRubPn4+Ojg7Tp0/H3Nwc+PiI/8GDB3h6etKtW7fkLHqqYmJiQpkyZRL1wsfGxpIuXTrgY09chw4dlEerIrGES455enpSvHhxXr58ydChQ6latSrW1tYA9OzZEycnJ5o2bUqPHj1o3ry5Mj5T2gABaSiAy5kzJ3Z2duzYsQMXFxdev35Nt27d8PLyInPmzIwcOZJFixZx7do13r17l+SjQPFj9PX1+fPPP7l69Sr29vaA9GgmJf5Rs6enJ3Z2dlhYWFCxYkUCAwMpVKiQ5HL7AfEDuGNjYzl//jzz589nwIABdO/eHYDNmzczZ84cdHV12bNnD97e3qxYsYIcOXKwe/duqfPvEBERwbVr1yhSpAiGhobA/27SatasyaBBgwDYuXOnDK7/isyZM+Pk5ISbmxt//fUXdnZ2HD16lHv37lGpUiVMTEwACA4O5tixY9y5c0e5VuVRtIiXph6hGhkZ0aZNG9zc3Lhy5QqHDh1CV1eXzJkzExAQoCRH/dzSOeLHFClSBDc3N2WWr9Tr57m4uDB48GCcnJy4d+8etWrVws7ODj09Pdzd3bl165bU3w8YOnQodevW5eLFi1SoUIF8+fIxbtw4Jk2aBMDvv/+Oo6MjhQoVUhZYj4mJkbr+TpMmTcLa2pq+ffuye/duYmJiMDIyYs6cOXz48IGOHTsmdxFTBT09PTp27EidOnXYvXs3vXv3Zv78+eTJk4eGDRvSunVrTp48Kden+KI0FcDFK1++PI0aNcLW1hYzMzPCwsJwd3fnzJkzyj7yh6EdUq+fp6enx4wZM3j+/DkDBgxQtjdq1IgBAwbw6tUrJS+h+Hb169dn9uzZtGjRgpMnT2JpaYmDgwM+Pj6MGzcOf39/ZV9TU1PlRk4mgny7hH/XCxcupGTJkly6dIknT55QokQJTE1NqVWrlsaEJpFYmzZtCA0N5cCBA5iamrJp0yZWrVrF6tWr8fDwwNTUFEdHRx48eED9+vV5+vRpchdZpGBpso/71KlT+Pv706lTJ65cuULhwoVxcXHR2EeCDO2Qev28mJgYoqKiKFCggMbjpW3btnHgwAEqVarEypUryZ07dzKWMvXJmjUr9+7dU2aTP378mPnz5zNlyhQGDhyopLcBlOANJFH394hPwAvg5OTEX3/9xfPnz8mePTvHjx+nZs2ayvgskbTs2bNTu3Zt1qxZg7e3N4aGhnTu3Jk2bdpQsmRJfH19mTdvHseOHePJkyc8e/YsuYssUrhU1QP3I48+9fT0cHBwYNWqVdJgi//M567Rjh074urqyqBBgzh8+DDR0dEAtGvXDhsbG+XmQwYpf7uaNWsq+bPOnz+vbK9UqRIbN25ER0cHDw8Pli9fnoylTNlKly6tzMyPjo7+7PX76ezHhPtJj+bX6enp0bhxYwYNGsTdu3c5efIkkZGRWFlZERAQwIsXLwAZ5iO+TaoJ4Jo1a0bNmjUJCAjg8ePHvHv37quf+bSxkQZG/BcSNrrW1taYm5tjYGDAhg0bePXqFcuWLSNv3rz4+flx/Phx3rx5w4wZM7hw4YKy2LekCUjscz9m8TPLnzx5wtSpU5X8gwULFsTDw4Pt27eza9cu+dv/jNq1a7Ny5UoWLlyIjo4OAQEB3Lt3L7mLlaYVK1aMJk2aYGdnR86cOQkPD6dPnz4aOTMleBNfkyoCOFNTU/bv34+JiQmPHz/mzJkzHDlyhNWrVyv7yA+eSGmGDRuGg4MD58+fp0iRIrx69YpRo0axb98+lixZQu7cubG0tCQ8PBxdXV2qVKkiQcZnJPwxa9WqFbly5cLc3Jy1a9dy5swZ6tWrh5eXF69evWLFihU8fPiQvn37EhMToyQ+lRu4pFWuXJlly5YREBBA1qxZadq0KatWreLMmTNs3rxZ2U/a2J/LyMiIHDlyMGLECKytrdm2bRtOTk7JXSyRiqSKlRjevHnDhg0bCA4O5uLFi1SrVg0/Pz9q167N1atXCQwMlIZFpCitWrXCwcGBNm3acPHiRRwcHJgxY4aSfsHR0ZHy5ctTqFAhYmNjWbt2LbGxsfIj+RnxwduIESNo06YN//zzDyVKlKBOnTps27aNMWPGEB0dTatWrfjrr7+4ffs2kZGRNG7cWDmGBG9JO3bsmJKXcPLkyZw5cwZLS0umTp1Ko0aNOHLkCEuWLJHr8itsbW05dOiQ8hj0a6Kiorh9+zaOjo40bdpUI1gW4lukih44+DjT7K+//qJBgwZcv34dY2NjPD096d27N+fPn2f9+vXs2bMn0QoLQiSHAQMGYGFhQe/evbGzs2PSpEn4+voyf/58TExMMDQ0TDRIWYK3L6tduzZTpkzB0dGRCxcuANCnTx/q1KnDnj17lJQhOXPmRFdXl/v376NWq6Xn7TMS9mp6enri4OBA7dq1iY6ORk9Pj/Pnz/Py5UuioqIwNjZm+fLlrF69WlnRQvxP69atGThwIIsWLWL27NlERER80+dkmI/4N1LsLNT4nor4mU+7du1i7dq1Sp6ht2/f0qRJE7Zv387Ro0epVasWhw4dolWrVslWZiHir9ccOXLw+PFjSpUqxZQpUxg5ciTz589HpVLRunVrbG1tlZUZ4knwpunTRLCmpqZER0drLE7v7+/P8ePHadmyJcbGxgCEhIRw7949SXz6GZUrVwZQgluAgIAA3r59S4cOHQDYs2cP169fx8HBgfbt23PhwgXKlSunrCkrNK1YsYL169fTqFEjXF1dyZgx4zd9Lv5vPj4hslyr4nukyACuWrVqTJs2jezZs2tk9D5//rySc2jv3r28fPmSHj164OPjg4eHB926dWPNmjXJXHrxK/k0i398g7xjxw48PDzYu3cvXl5eLFiwAIB06dJhbW1N7ty5JWfWV8TXZbdu3fjtt9/Q19dHV1dXubnT09NDrVYzZcoUcubMqQQmCckgcE1mZmbMmzePbdu2AR8Dhvgbia1bt1K1alVOnjzJq1ev6NatG6GhoYSGhuLq6krHjh1l5YokGBgYADB8+HClM6FLly6Jlmv8kgIFCmireCINS5EBXPHixSlQoAADBgzA0tJSacgXL16MsbExd+7cITIyknbt2ilLYoWGhirjiCQXkfivxAcIderUoXXr1hQtWhRjY2N27NjB4sWLefLkCR8+fCB9+vQULlyY+fPnkzlzZkaNGpXMJU+5EgYIjo6OjBw5koiICLZu3QqAn58fKpVKCYAzZ85McHDwN489+pW9fPmSDh06kC1bNtavXw/8b4m3TZs2UblyZaKjo7Gzs+PJkydA4p5QCYo1xacCatOmDTExMRQsWJDu3bvTtWtXMmTI8NXPd+7cmaNHj5IzZ05tF1WkMSl2DFyXLl1o2rQp9+/fZ8SIEUpj0rp1a3r27EnPnj2VcTBCJKcRI0bQsmVL1Go1r1+/Zv369QQGBmJmZoanpyft2rXj6dOnPH/+nJcvX2Jvb09MTIyMefuKmjVrkiNHDqKjo5UZ5+XKlWPJkiXcuHGDefPmERERQbdu3cicOTMNGjSQ+vwGurq6FC1alEWLFnH58mWcnZ2VeuvZsye1a9fG3d1d41G1+DJvb2+6detGv379ePfuHa1ataJw4cKsWbOGWbNmaSSQTsjJyYnBgwfj7e3Nxo0b/+NSi9QuxfXAxd/tzZkzhxUrVlClShWGDBmCpaUlAP/88w+ZM2emWrVqyVlM8QtL2ENUvnx5ypQpg6OjIxUrVmTdunXUrl2b/v378+rVK/r27Uu9evXo27cvXl5eNGvWTMlYL8HG5xUsWJDVq1czZcoUjV6Ms2fPYmtrS7p06Rg6dCh+fn7o6+vTqFEjWUD9M+IXRoePj51jY2O5fPkyDx48oGHDhqxevVq5pq9evUrBggUpVqxYchU31cmUKRMNGzZk9OjRrF27VkkHcvDgQZydnTUepya8Pp2cnBg+fDh9+vSR4E38kBTR2pUoUYLMmTMDmt3zderUwdDQkKJFi+Lj44OVlRUPHjwgMDAQT09PChcunFxFFr+gkiVLAv+7Rps1a0bnzp25ffs2p0+fJjIyknHjxrFt2zYqVqxIv379yJYtG5cuXWLnzp2cOXNGBtZ/o3v37uHk5MTjx4+pWrWqsl1HR4fbt2/TqFEjmjZtioODAw4ODhIUf0aVKlWYNWsWRYoUAf73uHTevHlkypQJZ2dn8uXLpzxO3bt3LyEhIbRv3z7ZypzavH37ltjYWCVQjh/C079/fx4+fIijoyN9+/bF1NRUuT6dnZ2VsduSPkT8qGQN4FQqFVZWVuzfvx9vb28sLCyUH8cFCxZQoEAB6tSpo2SuHzRoEFmzZuXIkSPs3buXmzdvJmfxxS9k5MiRODs7A//rgbO2tqZBgwaULl1aY9zl5MmT2bZtG7///js+Pj6JxsHIGCJNSQ2K//DhA9u2bWPQoEHUqVNHSRESFxenBGqhoaE8ePBAguIvyJMnD5kzZ6Z///7kypUL+Ni2FipUiDZt2rB161ZcXFzInTs3GzZsAMDNzY1OnTolY6lTrqSu1ejoaJ48eYK1tbXSwxnf03bjxg1iYmIwMjJSHqPWqFGDCRMm4OXlJcGb+FdSxBg4e3t7AgMDmTFjBn5+fgQFBVGoUCHat29PcHAw8L8xcc+fP6dr167KwFEZRyT+CxUqVODs2bPExMSQK1cuHjx4gEqlYujQodjY2LB8+fJEY12GDBmCubk5ffr0kaDtG3Tv3p0SJUpgYWHB4sWLOX36NKGhodjY2DBjxgxWrlxJ3759k7uYqYKFhYUybtjBwYF27doRHh5O5syZMTMzw9nZmfv37yv7//7772zevJm5c+fi4+MDSNv6qYR588qVK4dKpUJXV5eTJ09iZWXFrl27OHXqFO7u7kRFRfHhwwdmz57N+vXr2b59u3KjER9Unz59Opm/kUjtki2A++2334iIiOD27duo1WqaNWvG7NmzefToES9fvqR169aEhoZqJDb08PAgT5489O3bV34QRbKws7Oje/fujB49mgMHDqBSqRg3bhy//fYbW7duZc6cOcrM6IRkXcPEEtZJ//79cXV1Zc2aNeTPn588efJw8uRJpkyZwo0bN7CxsWHq1Kns3r2brl27JnPJU7ZmzZrh5ubGlClT2LJlC/BxZRBnZ2eKFi1Kx44d2b9/f6JrskiRIty8eVOCtq8YPHgwtra2REdHkz17djZt2sSECRPImTMnCxYs4OXLl4SHh2NqaoqJiQl//vmnMj5T6lb8TMnyCNXGxoadO3cyaNAg8ubNC8CGDRvo0KED2bNn58yZM0pPRsLu6KlTpyq9GZKLSCSHd+/e8erVK7p370716tVRq9UMGDCAs2fP0rhxYzp16pRk/icJ3hKLr5Ns2bKRN29e2rVrR79+/WjRogWTJk3CysoKV1dXTE1N2bZtG97e3mTOnFn+9r8gS5Ys9OvXj0KFCtGiRQtsbW0BWLlyJXPmzOHChQt06NCBIkWKJLomr1+/LhNBvqJ79+506NCBHj16UK1aNYKCgmjfvj3ZsmXj5MmTVKxYkQ0bNnDu3Dn2799P5cqVJXgTWpMsf6n6+voANG7cmLFjx5InTx4Atm/fTufOnWnXrp3SWMPHcS+fNtrygyi0LalAYceOHQQFBQEfe4Tjg7j+/ftz+vRpOnbsSMOGDf/roqZarVq14ty5c/z++++8fftW2R6f2b5hw4ZkyZJFWS+2efPmcgP3BU+fPuWff/5BrVYTFxdHq1ataNKkCQBr165l2bJlmJubM2DAgM9OApNA4/NKlSrFhAkTOH36NDY2NnTv3p1+/fpx7tw50qVLx+vXr5kwYQLDhw/H19dXyUsqdSq0IVkCuCNHjrBs2TKGDh1KwYIFmTZtGrlz5wY+JpPs0qUL3bt3x83NjSxZsgASsIn/Xvw1Z21tTdOmTbGzswM+LjMUFBREbGws7u7uVKtWDbVazcCBA5k9e7asBvIdtm/fzp49e8iXL5/SBsQHZwsXLkStVlO7dm1Asw2Q9iCx+BUVpkyZwoEDB7h27Rq6urp07NiRxo0bAx974pYvX07GjBkZP368MrFBfJ2RkRHly5cnLCyMChUqEBgYqKxvrKenR//+/ZNMbyWTa4S2JEsA9/jxY+Li4qhWrRoNGzYkR44cTJ06VWnAN27cSOfOnXFzc6NZs2bJUUTxixo7dqzGKgljxoxh+vTp+Pj4KOPeSpQowf79+wkKCiImJgY3Nzfq1q2LWq1mxowZ8hjqO0RERNC9e3cOHTqEr68vZcqUUYKzLFmy8PbtW54/f57MpUzZ4p9UxKcIefPmDXFxcbx69Qpvb29iYmISBXGbNm3i2rVrhISEJFu5U7KkenijoqJYs2YN7u7ubNiwgYEDBypL5JmYmFCqVClKlCjxH5dU/Mr+k0kMv//+O5GRkTx8+JA3b94AHxem3rhxIyNHjuTmzZvs3LmTGzdu0KtXL2V2VLVq1Thy5IjcwYj/RIYMGejbty916tRh/fr1rFq1ijlz5tC7d2/Cw8PR09Njzpw5ZM2alWbNmhESEkLdunXp168fJ06cYMiQIcn9FVItU1NTlixZQsGCBVm6dCn379+nYcOG5M6dm5o1a0ob8BnNmjXDy8uLvXv3MnPmTN6+fcvr16+pWbMms2bNomHDhhgZGeHj44OOjg4LFixQ1kGNJxNsNCWsjyJFipA5c2ZCQkJ4/PgxJUuWxN/fn4iICDw8PAgODiZr1qxMnTqVjBkz0qRJE3lcKv4zWg/gmjZtypw5czh//jwvXrxgzJgxPHjwgKdPnzJp0iTev3/PoEGDyJMnD5s3b+bGjRv069ePO3fuKMdIOBNVCG3Kli0b7du3x9bWlvv37xMXF0fnzp358OGDss/+/ft5+vQpLVq0AD7eoMQn6RU/ztTUlNmzZ1OnTh2WL1/O7du3CQwMVMYRSRugydLSklWrVpEvXz7UajV79uwhOjqav/76iytXrjBixAiuX7/OvHnzqFixIh4eHlhaWjJkyBCOHj2a3MVP8Xx8fLC2tsbc3Jxbt24RHh6Om5sbNjY2dOzYkZw5cxIWFqYEfA0aNJAl8sR/SusBXM2aNVm9ejXnzp3j7t27lCpViosXL/L3338THBzMypUradGiBWfOnCFXrlycPHmSefPmMWjQIG0WS4jPsrS0pH379rRs2ZK3b99So0YNAAwNDXn//j2NGzdm5MiR2NvbK3kKQXoykvK9P2YZMmRg9uzZ5MmTBycnJ65fvy4/iF/QvHlzZXm2K1euoFKp6NKlC6tWraJu3bpERUVhbW1NdHQ0lSpVomHDhgwfPlyu06/o1q0bvXr1omPHjhw9epQJEybQtm1bmjdvzvHjxylZsiQlSpQge/bsBAcHs2nTJiXJtNxoiP+KnjYPrlKp2L9/Py1btmTVqlXs2LGDDRs2YGZmxsiRIzl+/DimpqbUqFGDCxcu8ODBA0qXLs3Tp0+1WSwhNHwaeD1+/JilS5cC4O7uzrBhwxgxYgTv378HPo6FgcSDk+VHUZNKpVICr2zZshEWFvbVz0RERNClSxeWLl3KwoUL6dy5M5cvX9Z2UVOd+Gt23bp16OrqYm9vT5kyZejduzc7d+6katWq6Orqkj9/fiwsLAgJCeHYsWMcO3ZM4/MiMUNDQypVqsT48eM5evQodevWpWXLlgwYMIDjx4+jr6/PrVu3uHTpksbndHR0JHgT/6n/LJFvo0aNWLhwIUFBQYwcOZL06dPTqFEj6tSpg7+/P5cvX9ZoVOSuW/wXEl5zxYoV4/3794SHhxMZGUn27NlxdHSkdevW7Ny5k2nTppEhQwZ8fX1Jly4dtra28iP4GTVq1ODPP/9k3LhxjB8/HgsLC1xcXJQVVD4n/v+P9OnTs3HjRgwMDKhTp47GI2yRWPPmzXFyciIiIoJhw4Zx584dMmTIgJmZGffv35eA7TutWLGCmTNnoq+vz9y5cxk2bBgLFixAT0+P1q1b8+TJE3bt2pXcxRS/uP90JYYGDRqwePFiFixYwIgRI5LMWC9EcvDx8cHR0ZGIiAgiIyNp3749Dx8+VII4T09PoqKi2L59O8bGxri6uhITEyM/jEkwMDBg+PDh/PHHH7x9+5YSJUrQoEGD71q7uHz58soMycePH2urqGmKnZ0dTk5OREZGMn78eC5evAhIb9uXJFU3Ojo6LFy4kDx58mBlZYWv7/+1d59xUdzr38c/dAEJAoqKigUVu/Fv71gxCmLEiggiQYyiYC+gSMCCYkFRiQ2xxEIsHCtRCbZYYktiLMRYQLEjClKk3Q+4mUhsKcYFud5PTrI7u68rc4bZ7/yqP+Hh4QCUL1+ekJAQdu7cyfr161VRshCKD7rWwf79+xk8eDBDhgzBx8dHWeNNCFVq1aoVNjY2uLm5ERAQwKNHj4iOjsbS0pK7d++ybt06Fi5cSGZmJpcuXcLV1ZWsrCw0NDTkh/E1Xrx4wfTp03nx4gUtW7Zk27ZtSnj7Kwvwuri4EB4eTpkyZSS8/Q07duwgPDwcfX19xo0bR/369QHp2n+Tl8NbgwYNqFq1KmZmZuTk5DB58mR0dHSIj49n06ZN6OrqYmRkxMKFCylRooQyxEIIVXovLXBt27YlOTmZCxcu/PHFb3nqs7a2Zu3atezYsYMpU6bw9OnTf1uCEH/Zn6/N5s2b07JlSxYtWgSAmZkZQUFBNG7cmJ49e3L16lUqVapE27Zt2bx5s3Ttv4OGhgalSpViwoQJGBgYULVqVWJiYpg7d67y/stjhV7+/8PZ2ZkZM2YwevRodu3apZL6C5u/24KWv7TIwYMH8ff3/w8r+zj4+vrSp08f1NTUiI2NZfny5Rw4cIAOHTqwatUq7t69S3Z2NikpKejq6tK1a1eZbSoKhX8d4Fq3bs3EiRMxMzPjp59+IjIykv3795OZmfnWGTl2dnYMGzYMGxsbeUIUKuHh4UGNGjVo0KABFy9exMvLS7lezczMmDdvHo0aNaJv374FBtLLjftVbwoZBgYGTJw4kRYtWnDgwAElxAFUqlSJO3fuKOcyP7yNGjVK2YRd/KFevXqUKFGCCxcuKIv2vkm7du04duyYXKfv0KxZM0JDQxkxYgSVK1emTZs2tG3blgkTJnDgwAFMTEwYMGAA6urq3L17l+3bt8tsU1FovJcWOG1tbUxNTfH398fY2Jj09HRcXFxITU197Y/dn2/2MkZDfAgvX2ejRo3Cy8uLAwcOYG5uTu3atRk8eDDHjh1Tji9fvjxhYWE8efKEgQMHqqrsQu/l89q3b19q1qyJuro6Bw8e5MSJExgaGjJ27FiaNWvG8ePHCQ4OJjw8nPj4eDw9PQFwdXVl6tSpeHl5ScsbMHnyZM6fP09UVBQAfn5+9OrVCxMTE86dO0doaChRUVGvhIg/30vlYePNBgwYQN26dXny5AkLFiwAoFatWri7u9OxY0emTp3Knj17XvmcnFNRWLyXAJd/09DV1aV9+/aMHTsWIyMjunXrxuPHj+WCF4VKlSpVGDlyJNu2bePkyZPo6emxZMkS2rRpg7Ozs7LUAuRtU5SYmCgPGH+Bn58fffv25fLly5QoUYJmzZoxa9YsFi5cSKlSpfDw8MDGxgY9PT0ePnxIt27dyMzMpHnz5qxevRpvb28iIyNV/Z+hcoaGhsTExHDr1i0WLFhAiRIlmD59Oj4+PiQlJTF9+nR0dXUJCwtj27Zt0hL0D1SoUIEFCxbQvHlzwsLC8PPzU96ztLTE3d0dKysrvvrqK3bu3Km6QoV4i/9kFqqlpaVy0+7YsaOybpYQqpa/nE1CQgLDhw9XVqTX1NTk66+/pnXr1jg7O3Pq1KkCn5NW4rfr0KEDy5YtY8CAAfz0008ADBkyhMDAQLy9vVm1ahUGBgaYm5tjbm5OVFSU8lBXtWpV9PT0ZL23l5QrV47w8HAePnzIpUuXeP78OcHBwUBet/SyZcsoU6YMq1evZvv27RLi/oFWrVoxYsQImjZtyuDBgzl9+rTynqWlJZMmTUJDQwNnZ2cVVinEm/2jWagtWrSgSZMmb5xRdvXqVSZOnEhycjIBAQFoaGj8qyKFeF/27t1LeHg4ZmZm1K1bFx0dHSBvI3B3d3eOHj3K7t27qVOnToHPSXj7w7hx46hZs2aB14yMjLh//z5Xr15V7gtr167F398fHx8fqlWrRnJyMr/++iv79u1TxhEB3LhxQ8LbS9TU1Lh37x4uLi6UL1+eMWPGUKNGDeX95ORkRowYwcOHDxkyZAiDBg1CXf2DLijwUfjhhx8IDg7m5MmTzJkzh6ZNmyrvXb16lRkzZjBkyBDVFSjEO/ztv/rPP/+cXbt2MW/ePBo0aPDG4y5fvsy3335LtWrVqFChwr8qUoh/4k0PGOPHj2fbtm34+PjQuXNntLW1gbwQ9+WXX7JgwQKuXLnyIUstMsqUKcPkyZP56quvqFq1qvJ6dnY2lpaWGBsbk5ubi6Zm3iYv3333HcnJyZiamr7yXdJqVFD+9Zqbm4uJiQkJCQk4ODhw+vRpGjZsSKdOnZRj80Ncbm4uDRs2lCEq/9CPP/5IaGgocXFxBAYG0qRJE+W9uLg4cnNz/9LSN0Kowt8KcLVq1cLDw4OgoCA0NTVZsmQJn3766WuPzc7OZtOmTZQtWxYXF5f3UasQf9nLXZ62traMGTOGoUOHKvuaDh8+nAMHDhASEkKXLl0KhLjZs2cXaCESedTU1Hj48CGNGzfm008/JTAwUGkZOnz4MKdOnSIwMJCKFSsqsyRTU1OVyUzizV6+Xr28vAgJCaF69ercv3+foUOHkp6ezqhRo7CyslI+k5ycTN++fRk/fryKqv44nDhxgq+//pobN24QFhZGrVq1Crwvre+isPpbd9WSJUty8uRJNmzYQLt27dDQ0CA4OPiNIS4lJQU/Pz8qV66MgYHB+6hXiL8k/6Y7Y8YMgoKCaN26NV9++SWBgYH4+voC4ObmxnfffceiRYuwtbVVWo3ySQtRQfktEXFxcdjZ2dG8eXPGjRtH9erVSUpKIjw8HAMDA5YvX46VlRVWVlbMmzePp0+fFpgYIl6Vf71Onz4dNzc3du3apbSq3b9/n8GDB1OyZEm8vLyUhxCAtLQ0aSV6DS0tLeWfS5UqVeC9152rEydOsH79ejZv3kxsbOx/XZ4Q78XfmsRQokQJTE1NiYuLA/I2/Y2OjiYrKwtPT09lIV9dXV3S0tIAqFOnDgMGDGDevHkkJye///8CId6gS5cuLFq0CBcXF06fPk25cuXo3bs3bm5ubNq0SVmTbNOmTWhqatK3b18VV1w0TJ8+HU1NTbp3707lypWJjo5m7Nix3LlzB2traxwcHOjatSuxsbE8fvyYfv36ycKnf0HTpk0JDQ3Fy8uLo0ePKq/nrzmWP7FBT0+P0aNHc/78eRVWWzj17t2bnTt3KtfZmDFj6NatGykpKURFRREeHk5GRsY7r0W5VkVR8I9noWppaZGZmYmWlhYxMTFkZWUxatQo7t+/z4wZM/j+++/ZunUrABUrVlT2NRTiQ3Fzc8PBwYFOnTopN2MTExPc3d1p1aoVbm5u3L17F5BZpn+Vu7s748ePx8HBgRcvXmBsbMzKlSu5cOECXl5eyt+5hYUFycnJPHz4kNzcXFn49DX+fM199tln+Pv706FDh1cedvPvtxUrVmT8+PGMHTtWAsaf9OvXj0mTJhEREcGcOXMYMGAA/v7+zJs3j3bt2mFiYsKVK1eYMmUK6enpEtJEkfevlhHJvylraWkRHR2tDF7W0NCgVatWcsMWH8zLP4b5/2xra4uPjw9OTk5cvXpVObZFixZERkbSrVu3Aq0YEuLebenSpWRnZzN69GjlNUtLS/bu3cvx48eZOXNmgXMNcl7fZciQIcTGxqKrq8uCBQtwcHBQZuXmnzsHBwcuXLjApUuXlM9JACnI0NAQT09PWrduzZEjR1BXV+fcuXPs2bMHDQ0N3Nzc+Pzzz7ly5QqTJk2SECeKvH81sjg7Oxt1dXUyMzPp378/tWvXJikpidatWyvvCfFfezkg2NnZ0bp1a3R1dbl27RpaWlr079+fsmXLKsc/ePCAK1euvLIdkYSMdzMyMiownlVbW5urV6+yZMkSPvvsMwIDAzEzMyvwGTmvBb08BsvNzY3JkyeTmJhIQkICampqDBw4kPLlywMorZf9+vXj888/L/A9Ejz+oKmpydOnTwkODubYsWO0adOGvn37kpiYCOT9VoWFhbFjxw4sLS2ZPXs2urq6cg5FkfavE1ZOTg4mJiaEh4cTGxuLra0tWVlZaGhoyB+H+CDyA4Kvry+zZs1SFoa9fPkyc+bMYejQoYwfP56ePXtSv359AgMDSU9P5+LFiyquvOjZtGkTnTp1olevXgC8ePECgCdPnrBt2zbS09OVbmnxevnXa7169ShXrhxTp07lypUrXL58GX9/fwYPHszkyZNxcHCgS5cuREREUKpUKebMmaPiygsnTU1N5WGsXLlyzJo1i2PHjlGiRAn69++vBOaMjAzWrFnD9u3badu2Le7u7qosW4h/TfPdh7xbqVKliI2NxdPTk+zsbBnvIj44Z2dn+vfvr3Q/ZWZmArB161YyMzMZMmQIvXv3JiEhgSdPntCjRw9l9p60EP11J06c4JtvvsHb2xstLS22b9/OJ598Qrdu3di1axfffPMNIN2m79KkSRP27dtHVlYWXl5eyusRERGkpaXh5OSEn58fcXFx3L9/n86dOyu9GvJg/AdbW1vatm3LxIkTCQgIoFOnTrRt25bFixejpqaGlZUVkydPZvbs2UDeA0d4eDj379+XPXdFkffet9KS8CZUITg4mMzMzAJrYr18LRoYGGBsbEyJEiWIjY2VgfX/Qs2aNRk4cCDDhw/n9u3baGpqkpycTMeOHV/plhZvNmTIEObNm8e6deuYOXOm0t0HoKenR8mSJYG8Ln+Qe+vL8h8QrK2t2bBhA+fPn6dGjRr06NFDGSdoYGDAmDFjaNOmDd9//z1z5sx55aFCArEoyt5LC9zL5AYjPjQtLS3q16//ylpj2dnZaGtrU6tWLa5du8atW7eU99TU1ORafUnnzp05e/YsT548eeexsbGxzJw5k4iICBo1akRGRgY7duyQ1vc3eNM5Wbt2Lbq6unz11VfcvHmTtWvXKrNP8xdAzifX6x/WrVvHwoULOX/+PFFRURw9epS2bduybdu2ApM8kpOTWbhwIQDt2rXjk08+YcqUKQW+S8KbKMree4AT4kPLzMzkwIED9OvXj40bNxbYV7NSpUq4urqybNmyArMjpXvvDw4ODgQGBuLn50dERARPnz5952eys7O5dOnSK7MiJWS8Kv+cODg4ULt2bdTU1Pjpp5+IiIhg+fLlaGlpMX36dHJzcwkPD3/teplyvf4hMTGxwPjV/fv3c/DgQby9vUlKSmL69Om8ePECDQ0NJcTp6emhp6enwqqFeP/eexeqEKrQrFkzJk+ezIsXLwgICODixYuULl2a4OBgDA0NsbW1lR/BtwgICKBbt258/fXXfPvtt3+pJe7PZNxbQT179kRPT4/NmzczY8YMBg0axP79+6lTpw46Ojpcv34dJycnADw8PPDx8WHRokUEBwcrC6GLP/y5u3P48OFcunSJI0eOANC9e3dWrlzJ+vXr8fHxUbrzmzRpwpkzZ1RSsxD/JQlw4qNhY2PDgAEDaNeuHXFxcaipqZGeno61tTVZWVkSMF5DW1tbmUk6Z84c2rVrR1hYGFu2bOHZs2d/6Ttq1arFlStX/ssyi5z88W12dnbK7Mdhw4Zx6tQpNDU16dmzJ6NHj+bKlSsMHz4cgPHjx9OhQwd69Oih4uoLp/y/3/z/PXz4MGXKlMHNzY2TJ0+SnZ3NZ599xsqVK9m6dSvr1q1jwoQJGBoaYmNjo+ryhXjvJMCJQu91wetNYax8+fI0bNiQypUr8+DBAyIjI5WN6aV7780cHBwoU6YMEyZM4Pnz5wQFBf2lEDdkyBCmTJmCtbU1N2/e/DDFFnIODg4EBQXh7u7Orl276NWrFwEBAbRp04akpCQgb7vBAQMGMHjwYIYPHy77b/4NVlZWxMTEALBjxw4sLCwYMWIEJ06cIDs7GysrK8LDw4mPjycjI0N5gBPiYyMr7YpCTUtLSwlqNWrUoGrVqmhqar5xA++7d++yf/9+vv76a3bs2EFOTo6MzXqHCRMm4O/vz61bt/D09OTo0aOMHz+e/v3788knn7zxc87OzkyfPp3x48dLePv/+vbtS3BwMPPnz1eWqbh9+zapqanUr19fOS4tLY2DBw9Sq1Ytqlevrqpyi5yqVasSERGBi4sLAJ9//jnXr19n2bJltGzZEk1NTWJiYmjevDkeHh507txZWZdUiI+NBDhRKM2cORMjIyNlPbdp06axc+dOtm/fTlRUFOXKlfvL3aEy0+zNSpUqhY2NDbNnz2bnzp1s27aNL774gt27dzN16lT69euHoaEhUHAHAWdnZ2bMmIGnp6esp/X/OTs7s3TpUs6cOcPIkSNp2bIlAHfu3CEtLY0hQ4YUCGsvXrzg6tWrpKSkqKrkQu/Pu/k8ePCAkJAQ2rZti6WlJQC9evXi+vXrhISE0KJFC7S0tLh37x4XLlxQHvTkAU58jCTAiULHzMyMnj17EhkZiYGBgbItjpeXF76+viQnJ3PgwAHlBi7+ufyupfwfOB0dHQDGjRvHpUuXcHNzw8XFBQMDAyUwu7i4MG3aNEaPHi3h7f9zcXEhMDCQIUOGYGtry759+9i6dSutWrXi7t27eHp60qpVK/z8/Pjyyy/p0KEDISEh5OTkcOzYMVWXX2jlP3xZW1ujpqbG8+fP2bNnD7Vq1aJVq1bKcb169eL3338nIiKC2rVrF/gOGfcqPlYS4EShk5CQgL29PZmZmezevZtKlSqxZMkSDhw4wP/+9z9cXV25dOkS27ZtkxD3N7yuyzklJYWEhAQcHByAvO2GNDXzVheKj49HV1eX2rVrK0tbtG/fnlmzZjFmzBgJb/+fnp4e9vb2uLm5sXfvXrKyspg2bRr/+9//2LJlC61bt+bChQv069ePzMxMXF1d8fX1JTs7G2tra6WbX7yera0tGzZs4Ntvv6Vr165cuHCBkJAQAgICqFatmnKcvb09a9eulS3yRLEhkxhEofLy5IQaNWqwdOlSGjVqxMKFC5k1a5ZynLGxMcuWLaNWrVoMGjSowNpv4lUvn9eGDRuipqaGjo4Op06dwsLCgm3btnH58mUGDhyoLNewcuVKVq1axenTpwt8VlNTk7Nnz6ryP6fQeNtK/sbGxvj7+9OzZ08GDBjA8ePH0dPTQ0tLC319fRISEgDZYeHP/jxBqWLFiuzbtw9tbW127drFJ598wrp16+jVqxfGxsZ4enq+snae7LAgigMJcKLQqFSpEvHx8UBel0hUVBTm5uYEBgZiZmbGZ599xuPHj5XjjYyM2Lp1K/fu3WPw4MGqKrtI8fb2pnv37mhqaqKrq8uhQ4eYMWMG//d//0dQUBA5OTnExsZSvnx59PX1admypdJCJD+Ib5Z//a1fv75AIMsPcba2tvTr1++V3UJkaZs3q1ChAk+fPiUlJQUbGxv69etHTEwMenp6jB8/nsuXL2NoaIifnx9RUVGqLleID07a7UWh0LJlS0JDQ+natSsBAQGsXLkSIyMjrl69ysSJE0lOTiYyMrLArMgnT57Qu3dvZTFU8XYeHh44Ozvj6elJmzZt2LhxI46OjlSqVInvv/+eLl26sGvXLm7cuMGRI0do1aqVhLe/6PPPP6d3795Awe0EExMTlQk4u3btok6dOgU+J+Ht9Xr27Ml3332Hp6cnVatW5fvvv+fx48fk5OQQEhKCs7MzKSkp1KhRg86dO6u6XCFUQlrghEoZGxuTmJiIubk5c+fOpVatWhgYGNCjR48Ci8PWrFmT0NBQtLS06NGjxyvrk0lLRkGampqvLF4cGhrKkSNH+Oabb7CxsSE4OJivvvqK8PBwSpQoQXp6+ivfI917b5cfbuvUqcP69euZMWPGa8cGli5dGmdnZxYtWiTn8y8aO3Ysn376KXXr1sXLy4vq1avj7u5O3759iY+Pp0KFCnz66afs27dPHjBEsSQtcEJlgoKCGD58OOrq6sTFxXH69GlKly7N9evXCwxOhrwN1IcPH05GRganT59GX1+/wPsS3v4wZ84cTp06hY6OjrKMQokSJWjSpAmpqam0bt2apUuX4u/vT3h4OJqamnh5eb22JUPCxtvlB4f79+8TGxtLixYtgFcnjDx69Ij58+eTnZ0ta5K9Q/6EjgULFuDn58fOnTvZsGEDZcuWRU9Pj5kzZ6Kvr8+dO3fYs2ePslC3EMWNBDihMseOHWPu3Lnk5OSgra3N/v37GThwIPfv32fYsGFKl1S+2NhYPDw8OHjwoOwV+RZbtmwhIyODyMhIJcSlp6ezbds2Bg0axKZNm/D29mbt2rVA3lpwn376Kebm5qotvAhxcnJi6tSpGBgYoKmpyePHj9myZQtDhgyhQYMGb32gkFD8di+3pv3+++/4+/vj6uqKhYUF6enpfPbZZwWWEAE5p6J4ki5UoXIODg507dqVqVOnkpCQgIWFBQEBAejq6hIWFkZkZCQAbm5urFu3joyMDEBmmr1NvXr1WLVqFU+fPqVnz55kZGTQvXt3AgICuHHjBuPHj+fGjRuYmpoSHBys7Bcp5/P1LC0tMTExQU1NjStXrjBixAgGDRrEtWvXuHLlCvPnzyc5OZmgoCDi4uIIDAwkJydHWobfIzMzM5o0aULPnj0ZNmyYXKui2JMAJz64P49XGzZsGH369CE2NpbZs2dz584dqlWrRkBAACVLluTHH3+kVq1aNG3alFq1asmN+y/KD3HJycn06NGDFy9e4OjoyMiRI8nJySEtLU3pfsrfL1JC8asGDhzIxIkT0dHRoUyZMoSGhrJo0SIyMjIYMmQIHTt2pE6dOmzevJnWrVuTlpbGgAEDSE1NVXXphVazZs148OABDx48+MfnSa5VUdxJgBMqY29vz5UrV/j1119xdXWld+/e3Lp1i5kzZ3Lnzh2qVKmCh4cHFhYWpKam4uzs/MrAfJHndedETU2NevXqsXLlSlJSUvjss8/IzMykefPmVK5cmSpVqvDbb78RGRmpBDnpiipo8ODBzJs3jxEjRhAfH0/NmjUJCgpi4cKFzJ07t8BxlpaW9OvXDyMjI+bOncu8efNUWHnh1bRpU/bu3cvmzZupUKECvr6+xMfHk5SUpOrShChSJMAJldDV1eWHH37g1KlTDB8+HMjrIu3Vq1eBEKevr09ubq7ylC4h41Uvh7fq1auTlZVFWloa9+/fR01Njbp167Jq1SqeP39O9+7dlS7ol0lrxqvs7OxYtWoVzs7O7N27V3l97dq1VKxYETs7O54/f668rq6uTo0aNZg6dSolSpRgwIAB8qDxGg0bNmTv3r1MmTKFcuXK0atXLy5dusSxY8eUcZkg16QQ7yKTGMQH8fKsPDU1NdLS0nBzc6Nr167KNk4rV65k+/btmJubM2XKFCpWrMjz588LdLFIeHtVfkiYMGEC69evZ+vWrURHR2NlZUVubi4XL17E1dUVPT09IiMj0dXVfeU75IfyVSVLlgTA1NRU2V4MIC0tjSdPnij7yObLzc3l6tWrfPXVV7Rr144OHTp80HqLip9++omQkBCqV6/O3LlzmTJlCvv27cPX15dvv/2WKVOmUKJECbkmhXgHCXDig8gPGc7Oznz22WeYmppy5swZwsPD6d69O7Vq1QJg9erVbNu2jSZNmtC/f39VllykTJw4UdlkvlevXpw/f56wsDD69esHwK+//soXX3xB9erVC2xJJt5s48aNTJw4UelCBejevTu9e/dm+fLlr7Rk5ubmoq6uzu+//865c+cwNDRURdlFwm+//Ubbtm0pXbo0hw8fZtu2bTx58gQDAwO6dOnCjz/+SHBw8CvLCQkh/qD57kOEeD9q1KjB7NmzefDgAWfPnmXJkiVs3LiRFStW0KRJE2Xh3rCwMB4+fFig20q8WYMGDWjdujUjRowgJiaGbt260aJFC86fP8/ixYvJzc0lIiKCX3/9lS5dunDr1i1Vl1xkhIWFoa6uzuzZs6lbty4dO3Zk3LhxREdHv3bcYU5ODoMGDaJZs2aMHDlSRVUXLlZWVly/fp24uDjltW+//RYXFxdGjRqFr68vhw8fJi4ujmHDhvHw4UOmT5+OkZERN2/eVF3hQhRyMgZOfDAGBgb4+vpSr149IiMj8fb2xtPTky5dumBlZUXHjh2VDb7zyTiYV/05OFhYWNCpUydWrFhBmzZt+Prrr1m4cCGrVq1ix44d1K9fn6+++op169Ypn5Hz+vc4OzsTFBREVFQUjo6Obz1WX1+fihUrcvXq1Q9UXeGlra3N0aNHyc3NpU+fPty+fVu5fm1tbXF0dKROnTrcvHkTV1dXHjx48Mp3yKQlIV5PulDFf65r167UqFGD5ORkFi9eTJUqVYiPj6dnz57Y29uTlZWFiYkJgYGB6OnpFfishIyCXv4xa9q0KZC32OmWLVsAGDRoEHv37mXNmjUAJCQk8OjRI/r27Vvge+S8/j3h4eGMGzcOa2vrt7asaWho8Pz5cwlv/9+LFy+ws7MjNTWV9evXU6lSJeX6PXv2LFWqVCE1NRVbW1slvP15FwsJb0K8ngQ48Z+qXbs2o0aNYufOndjZ2REXF8fYsWNxdXXl4cOHjBs3jsOHD/Pw4UMMDQ1l7ax3yP8xmzp1KkuXLmXIkCEAPH36FD09PWrVqsX9+/eVZUH09fUZOXIktra2Kqy68Kpfvz5mZmYFXvtzgMi3bt06Jk2ahI+PD5MnT37tMTLJ5lX37t3Dzs6OnJwclixZQuXKlYG8h4t58+aRnZ1N7dq1leMlsAnx18gYOPGfunz5Mp6entjb27N48WLatm1LbGwsZ8+epXPnzoSFhREREcHu3btfu7yFeNX48eNxdnbGycmpwLii1NRUjh07xujRoylVqhTNmzdHS0uL8+fPA9IV9Wddu3bF39+fp0+fcvHiRcLCwrh06RLZ2dlv7GJes2YN+vr6WFtbq6DiosHQ0JCnT58CoKmpSVZWFsnJydy7d4+uXbuyZs0aXFxciIuL49KlS2RkZNCyZUsuX76s4sqFKFpkDJz4YDp37kyfPn2wsLCgWrVq3L59m0GDBnH79m3lGBmb9XYmJiaEh4cTHh5ORESE8nr++nh6enpMnDiR2rVr8/DhQ7y8vGSHhbcwNTWlfPnyLFiwgOTkZK5du4aPjw/p6elyzv6BZs2asXDhQjw9PTlz5ozyelhYGFWrVsXLy4uFCxeipqaGo6Mjt2/fZsWKFRgZGb3SzS+EeDsJcOKDqlChAg0bNmTChAnUq1ePFStW4O3treqyioyqVaty+PBh3NzciIqKKvCetrY2L168APIG0ucvMiuLH79byZIlcXBwwN7envT0dAYMGEBaWpqEuL/JysqKL7/8EmNjY0aNGsWVK1dYu3YtFhYWODg4EB8fj6mpqTJmc8iQISQlJfHs2TNpHRbib5IAJ/61/K65v/NjV7JkSVxdXQkJCZFw8QYvd3nmn1sjIyO2bt3K7t27CQ0NJSMjQzmuR48e1KtXj8DAQBVXXrj179+f1NRUdu3aBfxxnjU1NbGysmLSpEk8efIER0dHJRCLt6tQoQJ37twBoG3btri5uVG+fHnS09PR09NTukzzlSlThpiYGA4dOsTo0aMB6eIX4u+SSQziX+nevTvjxo2jdOnSfzm8qaurk5KSQnBwMNnZ2WhoaPzHVRY9L/+Yubu788UXX2BgYMCTJ0+URXnbt2+PhoYGubm5lChRgoEDB2Jpaaniygs3JycnQkJCSEtLU17Lf/jIysoiOjqaRYsWUbJkSdzd3VVYadHx+eefc+jQIQYPHgzA0aNHWb16NXfv3uXTTz9l/vz5xMXFFZgc8vDhQ1q3bo2Xl5fymoQ3If4eaYET/1i5cuWIiYkhJSUFNTU1Nm/ezLlz5zh06JByjHRB/Tu+vr7069eP4OBgIiMjuX//PgAbNmygTp06/PTTTzx48ID69etjYGBAhw4dXtniSeRxdnZmzpw5fPnll+zcufONx+no6DBt2jTq1KnDwIEDZXLNWxgYGLBmzRpatWrFzz//zPbt21m5ciUA7dq144svvqBChQpMmDCBc+fOvbaVTe4RQvwz0gIn/rHU1FSOHz9OQEAAHh4eGBoasmLFCubNm8fnn38OyHpj/4ajoyMDBw6kb9++rFixgvv37yv7mDo6OhIcHMzz58+pUKECp06dwsrKiqysLGnRfI3OnTsTFBSEq6srO3fupHr16kyYMIFVq1Yxbdo0GjdurBybkZHB3LlzqVGjBi4uLiqsuvBLTk7m5MmTpKWlcebMGXr37s3QoUMBOHLkCKtXr+bOnTvMmzePRo0avbaVTe4RQvwzEuDEP/bs2TOioqKYM2cOt27dwsfHh9atW2NgYEBISAi7d+/GxsYGc3NzVZdaJFWuXJk9e/Zw6dIlqlevjrOzM9999x07d+7E0dGR8PBwPDw8GDx4MH5+fkp3tIwpLEhDQ4PatWsTHx9P7dq1qV69OuvWraN58+Zoa2vTu3dvZsyYQa9evZTjnz17RnBwMFWrVlVt8YWYpmbeKlTLli3jl19+ITc3l19//ZXBgwcr6xMePnyYVatWER8fT3h4ODVr1lRhxUJ8XCTAib8l/6atrp536Wzbto2YmBhsbGyAvEU7GzZsyIEDB7hz5w6jR4/mhx9+oFOnTiqruajS1tamb9++eHl5sWLFCjp37sz+/ft5/PgxQ4YMoVSpUkDBsUMS3l6VnZ1NeHg4oaGh9OnThyNHjhAVFcWQIUNwcnKiS5cuZGVlKWO48s/h5cuX0dbWRkdHR5XlFzr5Cx/nd9Xn5ORw4cIFMjMzWbBgAefOncPZ2VkJcUeOHGHjxo1ERERw7do1VZUtxEdHxsCJv8zKyopWrVqxfPlynjx5orw+depUWrZsia2tLdHR0aSlpdG/f39SUlJo3LgxjRs3ZvXq1RIu/oHg4GBq1KjB//73P77//nuuXr1KixYtCAgIwNHRkXv37qm6xCLjk08+wdHRkUqVKrF06dIC+3K2bt2anTt30qZNmwLbYFWtWpUbN26osOrCpVevXsyfP59du3YRFhZGXFwcT548oWHDhuzcuZP+/ftz69YtJkyYQKNGjVi3bh3h4eEFvkPGvAnxfkiAE39ZQEAAHTt2ZNu2baxevZqkpCQgr8vp+++/p3bt2pw8eRJnZ2cSExNf+bx07/11Lw/2fnlNN01NTTZu3EhmZuY7N1UXrzIwMMDMzOyVvUptbW2VHUPydxEQBRkZGbF06VLatm3Lixcv2LNnD7Vq1WLevHmcPHkSNzc3DA0N8fX1xdLSki+++AJra2smT57M3r17VV2+EB8dCXDib/H19aVdu3bs37+fFStW8PTpUzQ1NRk3bhx2dnb07t1bWoX+A3p6evTt25fu3btTrlw5OnXqRFZWlqyd9R5oa2uzZs0a0tLScHNzU3U5hVrbtm2xt7enfv36bNq0iZycHNzd3bl48SK1atUiNzcXGxsbkpKSqFWrFp07d2bZsmXS4ibEf0DGwIm/JH9mo5+fHwcPHqRPnz64ublhZGREVlYWO3fuxNzcnPbt26u40qJBS0tL+Wd9ff0C771uM3UdHR1MTU15+PAhHTt2VGabSnj75/T19enevTvh4eFUrlyZ4cOHA2/ezF7krfH27bffcv36dRwdHTlw4AB2dnZERkYCUL58eYyNjQG4cuUKISEh5OTkKGNmhRDvj7TAiTeqVq0a169fBwqOW1mzZg0dO3bkxo0b7Nu3j9WrV/P48WNmzJhBu3btcHR0JCEhQZWlF1rt27fn6NGjyrkcOXIk7dq1Izk5mYiICA4dOvTGljUtLS0yMzMBGUf0PpQpU4Z58+ahoaGBi4uLEoqlm//dWrZsyZdffkmlSpUYP348Z8+eRV9fHwMDA+7duyctw0J8APJYJF7LwsKCU6dOMXLkSDQ0NJSwEB4eTrVq1WjVqhXR0dFYW1szdOhQ9PX1OXPmDHfv3pXw9gYjR44kMDCQgQMHAvDFF18wbtw4zp07R7Vq1RgzZgyjR49GS0uL3NzcV1qC8sMbyNpZ78PDhw8ZM2YMTk5OEt7+phMnTrB8+XJu3bpFUFAQLVu25Pnz5xLehPiApAVOvJGnpycTJ07E29ubtWvXEhYWRvXq1Rk8eDA3b94E8sbEtW3blqNHj+Ln56d8Vm7irzI1NWXWrFmUK1eOiIgI6tWrx549e4iJiUFTUxN/f38aNWrEgQMHWLx4MZmZmXIePxA5z/9My5YtcXNzo3LlykyfPp3jx4+ruiQhig0JcKKAunXr8ttvvymbeI8YMYIZM2Zw/fp10tLSGDx4MLdv3y7QWjF//nx0dHTw8PBQZemFWn6Xp4mJCUFBQZQuXRpTU1OGDh3Kr7/+CuSNyfL29lZCXEhIiGymLj64vxtmW7RoweTJk7l9+7bcA4T4gKQLVSjs7e2JiYlh1qxZBVZZnzRpEhYWFkRFRXH79m0gb7HT/IHJ48aNkxv3W6ipqSldno8fP2bixIncuXOH8uXLY21trRz3/PlzAgICOHv2LAMHDsTe3l5VJYtiLD+8tWrVii5dumBmZqb8rb9ugsfJkyfx9vZm1KhRH7ROIYo7TVUXIAqP/NljTk5O6OvrM3LkSHJycggLC0NbWxt/f38SExNZsWIFkDcOS7qe3u7l89OnTx/u3LnDiRMnmDJlCurq6nTp0oUHDx6wYcMGIG9/2dmzZxMfH8+WLVtUWbooRry9vXn06BFff/01AP7+/nz++ecYGBgQGxvLtm3bWLNmDS9evHjt33x+K7LcD4T4cCTACcXJkyeJjo4mJiaGESNGsHLlStzc3MjJyeHrr79GXV0df39/cnNzWblyJYDcrN8h//z4+vrSp08fVq9ezeXLl3ny5AlTpkxh7ty5yqSG/BD3/Plz5YdUZpuK/5qBgQGNGzdGS0uLlJQUbty4QYsWLRgyZAiJiYl4eHjQq1cv9PX1WbJkyRtDHMj9QIgPScbAiQLWrVtHdnY2ixcvZtOmTRw5coThw4crIWL48OH4+/vj6urK//73PxVXWzQMHTqUSZMm0bdvX65cuVLgB9DExITAwEBMTU3ZvXu30ropxIdkbGxMYGAghoaG3Lhxg7S0NGbMmAHkLSLt7e1NkyZN+O6775QQJ4RQLRkDV4w1aNAAfX19tLW1lddmzpyJsbExubm5uLq60rlzZ5YvX66MgQkNDWXYsGHs2bNHVWUXOQ0bNmTTpk38/PPPygbg+fLHxGVlZVGzZk0VVSiKMzU1NRITE5kyZQopKSn079+fOnXqKO+npqYSEBDAjz/+SKdOnZgyZYoyRlYIoToS4IopOzs7Dh06xLp165gzZw4WFhYAxMXFkZmZSadOnTh+/DjOzs506tSJZcuWKSFux44dZGdnK7sziNdr1KgRkDez18TEBPhj/bbc3Fy0tbWpWbMmiYmJuLi4MGHCBJXVKoqf/AkJubm5mJmZ8ejRI8aOHcu+ffuoXLkyLi4uyjFpaWnMnDmT33//HUNDw1ceRIQQH54EuGJKT08PyNugWktLiz179uDn50fTpk2ZO3cujo6OWFhYcPToUZycnLC3t2f8+PEFvkMWPX0zHx8fZs+ejZmZGdHR0VhYWPDpp58WOKZKlSr4+PhgaWnJ06dPX7t4rxD/hZfHsI0bN46QkBAaNWpEUlIS3t7e/PLLL9jb2+Po6Kh8Ji0tjfHjxzNu3DhVlS2EeIkEuGJq06ZNjB49mvr163PixAnGjBlDSkoKq1evZsKECZQtW5bGjRsD8MMPP9CxY0eCgoJUXHXR0KBBAxo3boyPjw8JCQl8//33lC5dGmdnZ5o3bw5AuXLlmD59OqVKleK3335TPiuDwMWHkH+d+fj44Orqyvr163nw4AEAiYmJTJo0ifv379O/f38GDRqkfC4jI0MeNIQoJGQSQzHn5ubGzJkz8fb2ZuXKlZiZmeHi4kLjxo3x9vbm8uXLBY6X7YbeztXVlXbt2qGlpYWrqytpaWkAWFtbM3bsWKUrNSUlhdzcXLp06fLGvU+F+C/VrVuXNWvWMHXqVA4dOqS8nv83bmxszJw5c2jQoAG+vr5ERUWpsFohxJ/JSNRibuXKleTm5jJ79mz09fVZtGgRs2fPRlNT87UzzSS8vV12djZWVlY8f/6c6tWr88svvwAQFRXF9evXKV++PI0aNeLmzZvs2rWLnJwcCcXig/jzQ0LJkiUpWbIkP//8c4HjsrOz0dbWJjExEW9vb1xdXTlw4MCHLlcI8Q7SAieAvJajOXPm8NVXX7FkyRJVl1MkvKnVrHfv3syaNYvdu3ezdOlSbty48cbvkHXexIc2atQo4uPj+e2339i+fTtffvkl0dHRwB/Xo62tLYmJiQX2NpVrVYjCRVrgPmL169fn8ePHJCQkKK+9KXSsXr2a3NxcZs6ciZ6eHoGBgR+y1CIp/zzWq1cPXV1dnj17xtWrV9m+fTt6enpMnjyZtLQ0Vq9ezc2bN1/7HfKDKP5rL//NDxw4EHd3dxwdHUlKSuL69ev069ePR48e8fPPP5OTk4O6ujouLi5cvny5QICTa1WIwkVa4D5SXbt2xd/fn6dPn3Lx4kXCwsK4dOmSsofpm27Go0aNwtraGhsbmw9ccdHRsGFDfvrpJwCmT59Ojx49MDU15c6dO9y5c4f+/fsDeVuSjR8/nh07drBu3Tp+//13VZYtirkmTZrQq1cvrl69yvr164G8+8SMGTO4desWJ0+e5P79+wwYMABjY2M6dOggXftCFGIS4D5ipqamlC9fngULFpCcnMy1a9fw8fEhPT1dukP+IWdnZyZOnEiXLl2wsbFhwoQJODs78/TpU6pXr86kSZNIS0ujU6dOADg4OLBw4UJ8fHyU7ceE+NDq1q3Ld999h7q6OgEBASxdulR5r02bNvTu3Rtra2tu3rzJ/fv3GTZsGFlZWXKfEKIQkwBXDJQsWRIHBwfs7e1JT09nwIABpKWlyc35b3JyciIoKAgXFxf27NnDsmXLuHPnDjNnzgTyuqoaNmxIaGgoR44cYeLEiQB07tyZ6OhoOddCpXr37s3s2bM5d+4cvr6+xMbGFnhfX18fyNuLF2TGuRCFnawD95H55JNPMDU1LfBaSkoKa9asYd68eejp6REeHo62trYEir/Bzs6O+fPn4+TkpGwjVq5cuQJbDuXm5nLhwgX27dtHzZo1KVGiBAAHDx5UxhYJ8V97eYeUl6+57du3M2PGDBo0aICTkxNVqlQpcNzz58+V8AYy41yIwk5+UT4ivXv3Jjw8nOjoaNavX0/Dhg2BvJahrKwsoqOjWbRoESVLlsTd3V3F1RYdzs7OrFq16pXX9+/fT+nSpenQoUOB12/evIm+vj5aWloFXpfALD6E/OA1dOhQQkJCCA0NZezYsUDeAt6zZ8/Gzs4OV1dXJcTJtSlE0SMB7iMxcOBAFixYwMGDB/H29qZBgwYMGTIE+GO2ZE5ODgcPHuTcuXN06NABHR0dFVZcNAwZMkTZWmzOnDmEhYXRp08fIC/AZWdn4+rqio2NDWpqahgZGWFra8uNGzdITk5WcfWiOBkwYABeXl4A+Pr6MmnSJB4/fkzJkiXp168fBw8eRE1NjQ0bNjBnzhxsbW0ZM2YM5cuXV23hQoh/RMbAfQTatGnD8uXLmTZtGjt37gTAxcUFc3NzVq9ezePHj5UdASCvm/X48eMsXbqU0NBQFVVd+LVr147169czYsQIpdt02rRpjBgxAk9PT7Zu3UrNmjWVPU8NDQ25d+8eGhoadOrUSTb8Fh+Ms7Mz8+bNY+DAgcTHx7N582Y8PT05evQokDcDdeHChTx58oSePXsCeS10HTp0wMnJSXYBEaIIkgBXxKmrq9O/f39MTExYs2YNqampAOzcuRMzMzNMTEz4+eefOXbsGPPnz1c+98UXX1CjRg0mTZqkqtILvVKlSmFubs7PP/9cYED3tGnTGDlyJJ6enmzZsoUyZcpQsWJFmjdvzt27d2WHBfFBDRw4kIULFzJ06FD27t2LlZUVq1atom3btty9exfIu0+0a9eO2bNnM2XKFGJiYgp8h2zlJkTRIwv5FnE5OTns3r2bUqVKKeFt3bp1VK1aFS8vL5KSkujfvz+fffYZu3btUmaeXb58mbp166Kjo0NGRoYq/xMKraSkJBo3boyBgUGB7lB/f38AgoODycnJISIigocPH3L+/HnlGHV1dQlv4j/Xr18/Fi9ezOrVq9m7dy8Av/32G0+fPsXKyopNmzYBefeJX3/9lVKlSmFmZvbK90h4E6LokQD3EUhOTlYChpaWFnv27MHb25v4+HgAnj17hqurK1WrVlUC3PHjx0lISJDw9haNGzdmzpw5uLm5ceHChQLLruSHuIULF6Kjo8OGDRsKfFYGhYv/mrOzM4GBgXz33Xc4ODhw7tw5tm7dyrNnz7h06RJ2dnYkJCRw+PBhANLT00lISFAe9IQQRZt0oRYDdevWZd68eYwbN47Lly+rupwiQ0NDg8OHD3P+/HlGjRr12mPmzp1LrVq1lHFFQnwIgwYNYtGiRTg7O7N3715lbKaXlxdbtmyhevXqhISE8OLFC3766ScuXLiAo6OjssOCPGAIUfRJgPvIaWtrs2bNGjQ1NRk4cKB0lbzBn8cAaWpqkpWVhbW1Nd7e3nh5eXHu3DkVVihEnhIlSrBo0SIiIyPZt2+f8rqPjw8eHh54eXmxefNmqlatirOzM507dyYlJYX79+/j6uoqOywI8ZGQAPeR0tPTo127dgwePBhzc3M6dOhAVlaWDFZ+h2bNmnH69Gnl36tXr054eDjh4eGsWLFChZUJ8e7JBn8OcZDXkqyvr8+zZ8+Uf5fxmUIUfbIO3EdKT0+PXr16kZaWhpWVFVlZWWhoaEh4+5MSJUpgYGAAQNOmTYmMjCQyMpJhw4ZhaGjItWvXWL16NZ6enlSrVk3F1YriLv/v18HBAV9fXyAv1OULCAggJCSEBQsWKOsVZmdnK+Et/9+FEEWfTGL4SD169IhJkybx9OlTQGZFvo6trS39+vWjZs2a7N+/n++++44mTZowfvx4evbsiZeXF0FBQSQnJ/PDDz/QrFkzrl+/Lt1PQuWaNWtG/fr1gVdnkAYEBJCTk8Py5ct59OjRK0uGCCE+DtKFWgxIt+mrnJ2d8fPz49tvv0VbW5vevXtz4sQJ+vfvj7q6Ovr6+gwfPpxGjRpRo0YNzM3NOXHihExWECqV/7dsYmLC4cOHCQ4OZuXKla891tnZmQ0bNsiDmxAfKQlwotgZNGgQc+fOZejQoURFRQHQvn17vv32W7744gsiIyOVY83MzKhcuTIjR46kUaNGBAQEKGtrCaEqJUqUwN/fHwMDA4YPH/7WY2XMmxAfJxkDJ4qV0qVLs2jRIn788Uela0lNTY3z588THx+Pnp6e8hpAQkICJ06cwMPDgx9//JHmzZurqnRRjA0bNozg4GAsLS3R0tIiPT2d3bt306tXL6ysrN76WQlvQnycJMCJYuXRo0c4OTnRrFkz/Pz8KFu2LLm5ubRv354KFSpw4cIFoOC4InV1dZKSkoiIiKB9+/aYmpqqqHpRXDRo0IAePXrQo0cPKlasSFpaGi1atGDx4sVs2LCBevXqcerUKZYvX46Dg4MyEUcIUXzIJAZR7Ozbtw9XV1fWrVvH06dPuXnzJnPmzGH06NGvXeg4f8JCs2bNSE5OJi0t7UOXLIoRBwcHvL29efHiBRUrVmT//v34+vqyceNGunfvTt++fdm4cSPnz5+nVKlS6OjoKNu9yXhXIYoPGQMniq3u3bsTHh4OwPTp01m+fPkbj9XQ0GDt2rXMnz9faaUT4n0bNGgQ8+fPx83NjYsXL2Jubs4333zD1q1bGTNmjHJc165dqVu3Lu7u7piYmLBx40a8vLxUV7gQ4oOTACeKNSsrKyIiIggNDSU4OJhHjx6puiRRTNnZ2bFq1SpGjRrF5s2blda02bNn07FjR6ytrUlKSirwGTMzM9zc3Pi///s/3NzcePDggWqKF0J8cDIGThRrMTExODk54e7ujpeXF2XLllV1SaKYSk5OBqBGjRqUK1dO6QrV1NQkKSnplckIampqJCQksHLlSho2bPjOyQxCiI+LBDjxUXp5dfq3vQZ5Y+LyQ1yvXr3+48qEeJWamhrR0dE4OjoyevRopbu0W7duODo6smDBAiXg5cvNzVVC3I8//oixsbEqShdCqIh0oYqPjpaWFpmZmUBea0ZWVhbx8fHv3Au2efPmnDlzRpZdECplbW3Nhg0bOHz4MA0bNsTPz48NGza8cQeQ/K7Xli1bcu3aNRVULIRQBQlw4qMxc+ZMgoKCePLkCQDTpk1jwIABvHjxgsTERAYNGsS9e/fe+T2y8KlQtc6dO7Np0yZOnTrFoEGDlC3xXkdXV5dy5cpx48aND1ihEELVpAtVfBTMzMzo2bMnkZGRGBgY0KZNG/r27YuXlxe+vr4kJydz4MABLC0t3/ldEt6Eqh08eJCBAwfSvHlzJk+eTOnSpV97nLq6OmlpaRLehCiGpAVOfDRq1qzJ8uXL0dTUJDQ0lJIlSyr7RJqYmLBs2TLq1q2Lvb09V69eVXG1ojhq27YtycnJBZaieVu3vrW1NWvXrmXHjh1MmTLlrS1xQojiRVrgRJGXPzkhNjaW4cOHk5GRweLFiylTpoxyzOPHj/nyyy+5ePEiW7ZsoW7duqoqVxRTrVu3Zvz48axcuZJVq1Zha2uLlpYWubm5aGhovPYzUVFRDB8+nMqVK/Ps2bMPXLEQojCTFjhRpFWqVIn4+HgAevXqRVRUFObm5gQGBmJmZsZnn33G48ePleONjIzYunUr9+7dY/DgwaoqWxRTOjo6lClTBn9/f4yNjUlPT8fFxYXU1NTXTlL4c+uc7LQghMgnAU4UWS1btsTHx4fg4GDatWuHu7s7DRs2JCEhQelO1dHRoXv37gVaLwwMDEhJSZEfQvHBlChRgvT0dOXfdXV1ad++PWPHjsXIyIhu3brx+PHjN840FUKIP5MAJ4ocY2NjEhMTMTc3Z+7cudSqVQsDAwN69OjBlStXlONq1qxJaGgoWlpa9OjR45UuKGnNEB+CnZ0dVatW5ZtvvuHBgwcFrjtLS0sWLlxIqVKl6NixY4GQJ4QQbyNj4ESREhQUxPDhw1FXVycuLo7Tp09TunRprl+/TrVq1Qoc+/KYuNOnT6Ovr1/gfQlv4r82aNAgQkJCePHihbI24cvX3dWrV5k4cSLJyckEBAS8cSycEEL8mQQ4UaQcO3aMuXPnkpOTg7a2Nvv372fgwIHcv3+fYcOG0bt37wLHx8bG4uHhwcGDB0lLS1NR1aI4aty4MZMnT2b06NEsW7aM1NRUTExMMDIyKnDc5cuX+fbbb6lWrRoVKlRQUbVCiKJGApwoUnbu3ElWVhYODg6sWLGCpKQkjh49iq+vL2lpaTg5OWFnZ6cc7+bmxo0bN/Dw8CAnJwd1dbnkxYdRunRpLl68yI4dO6hbty5hYWHs2bOHLVu2EBQUpByXnZ3Npk2bKFu2LC4uLiqsWAhRlMivmSgS/ryPacmSJTEzM2Pq1KlUqFCB33//HW9vb1JTU3F1dWXatGls3LiRCRMmKF1XgAwQFx9MnTp1KFOmDLq6uoSGhnLjxg1mz57N3r17adq0KRs2bFCOTUlJwc/Pj8qVK2NgYKDCqoUQRYUEOFEk5I8bsre3p27duqxYsYItW7ZQtWpVvL29qVChAtevX2fq1KnExsbyf//3f0Dej2hOTs4bN7IX4r9y5MgRMjMzGTlyJDdv3mTu3LlERkayZMkSgoKCqFChAq1bt1aOv337Nrdv31ZhxUKIokRmoYoiQ1dXlx9++IFTp04xfPhwIK+LtFevXty6dYuZM2dy584d9PX1yc3NJTU1FZC9TYVqmJqaEh4ejqWlJbGxsXTr1k15r3Tp0hw9epQZM2awZcsW5fUKFSpw584dVZQrhChipAVOFFovt5qpqamRlpaGm5sbXbt2xcHBAYCVK1eyfft2zM3NmTJlChUrVuT58+dKeAPZ21SoxoMHD/Dy8uLFixc0btyYAQMGKO+lpqZy7do1njx5AvxxrUt4E0L8VdICJwo9Z2dnHj58yJkzZ3jw4AG+vr7UqFGDgIAAZd03FxcX3N3diYiIYP78+SquWIg/WFpasnHjRlJTUzl16hQnT55k0KBBGBoa0qVLFxmXKYT4RyTAiUKtRo0aHD58mAcPHnD27FmWLFlCSkoKK1asYM2aNQUGgtvY2LB37175QRSFTpUqVXB2dqZjx44kJSWRmJiIm5sbWVlZsvuCEOIfkQAnCjUDAwN8fX2pV68ekZGReHt74+npSZcuXbCysqJjx44kJCQU+Iz8IIrCSlNTE21tbRmfKYT412QMnCiUunbtSo0aNUhOTmbx4sVUqVKF+Ph4evbsib29PVlZWZiYmBAYGIienl6Bz0p4E4VVVlaWjM8UQrwXEuBEoVO7dm1GjRrFzp07sbOzIy4ujrFjx+Lq6srDhw8ZN24chw8f5uHDhxgaGhb4QRRCCCGKA+lCFYVStWrVsLe3x8PDg4iICGJjYzE1NeXOnTuEhYUBecuKZGRkSIubEEKIYkcCnCjUOnfuTJ8+fbCwsKBatWrcvn2bQYMGFVjwVMa8CSGEKG4kwIlCr0KFCjRs2JAJEyZQr149VqxYgbe3t6rLEkIIIVRGApxQCTU1NXJzc/9W61nJkiVxdXUlJCREBn8LIYQo1iTAiQ+ue/fu1KlTh7Vr1/Lo0aO/9Jk/Bz1ZfkEIIURxJgFOfFDlypUjJiaGlJQU1NTU2Lx5M+fOnePQoUPKMTKmTQghhHg7TVUXIIqX1NRUjh8/zq5du7h//z49evRgxYoVbN++nR9++IEdO3ZIeBNCCCHeQdaBEx/Us2fPiIqKYs6cOdy6dQsfHx9at26NgYEBISEh7N69GxsbG8zNzVVdqhBCCFFoSYAT/zlNzbyGXnX1vMtt27ZtxMTEYGNjA8C9e/do2LAhBw4c4M6dO4wePZoffviBTp06qaxmIYQQojCTLlTxn7KysqJVq1YsX76cJ0+eAHnbB8XFxWFra8uKFSuIjo4mMTERDw8PUlJSaNy4MY0bNyYmJka1xQshhBCFlLTAif9U586dsbGxYejQoZQqVUp5PTAwEENDQx4+fMjz588ZPHgwKSkpAJw9e5YVK1aQnZ2NhoaGiioXQgghCi8JcOI/5ePjQ1RUFN27d8fNzQ1DQ0Mgbx24PXv28Ntvv+Hm5kZiYuJrPy9LhQghhBCvkgAn/jP5rWd+fn4cPHiQPn364ObmhpGREVlZWezcuRNzc3Pat2+v4kqFEEKIokUCnHivqlWrpvxzbu4fSwzWqFGDsmXL0qNHD9zc3DAxMeHq1ausWrUKd3d3zMzMVFGuEEIIUSRJgBPvjYWFBadOnWLkyJFoaGgo67mFh4dTrVo1WrVqRXR0NNbW1gwdOhR9fX3OnDnD3bt3SUhIUHH1QgghRNEhs1DFe/P7778TEBDA1KlTef78OWvXriUsLIxq1aoxePBgEhIS8Pf3R11dHWtra/T09PDz82P37t3AH/ujCiGEEOLtZCst8a/VrVuX3377jRcvXgAwYsQIZsyYwfXr10lLS2Pw4MHcvn27wP6l8+fPR0dHBw8PD1WWLoQQQhRJ0oUq/hV7e3tiYmKYNWuWsmDvsmXLmDRpEhYWFkRFRXH79m0gb0Zp/mK+48aNk/AmhBBC/EPShSr+FWNjYwCcnJzQ19dn5MiR5OTkEBYWhra2Nv7+/iQmJrJixQoAcnJypKtUCCGE+JckwIl/5eTJk0RHRxMTE8OIESNYuXIlbm5u5OTk8PXXX6Ouro6/vz+5ubmsXLkSQMKbEEII8S9JF6r4V3755RcyMjJo1qwZTk5OtG7dmtDQUKWrdPny5fj6+jJr1ix69uyp4mqFEEKIj4MEOPG3NGjQAH19fbS1tZXXZs6cibGxMbm5ubi6utK5c2eWL1+uhLjQ0FCGDRvGnj17VFW2EEII8VGRACf+Mjs7Ow4dOsS6deuYM2cOFhYWAMTFxZGZmUmnTp04fvw4zs7OdOrUiWXLlikhbseOHbK3qRBCCPGeSIATf5menh4ARkZGaGlpsWfPHvz8/GjatClz587F0dERCwsLjh49ipOTE/b29owfP77Ad8jepkIIIcS/J5MYxF+2adMmABYvXsyqVavYu3cv9erVY/Xq1Vy4cIGyZcvSuHFjfv/9d3744Qc6duzIr7/+quKqhRBCiI+PtMCJv2XTpk1MnTqVRYsWUbFiRebNm0f79u25cOECJ0+e5JdfflGO/eWXX8jJyZFuUyGEEOI9kxY48betXLmS3NxcZs+ejb6+PosWLWL27NloamoquzG8TLpNhRBCiPdLApz4R1atWkVubi5z5swhOzubJUuWvDa8CSGEEOL9kwAnCqhfvz6PHz8mISFBee1NOyesXr2a3NxcZs6ciZ6eHoGBgR+yVCGEEKLYks3shaJr1674+/vz9OlTLl68SFhYGJcuXVL2MM3JyXnt50aNGoW1tTU2NjYfuGIhhBCieJIAJwowNTWlfPnyLFiwgOTkZK5du4aPjw/p6elvDXFCCCGE+HAkwIkC8rtLS5YsiYODA/b29qSnpzNgwADS0tIkxAkhhBCFgAQ4QZMmTUhPT+fixYsAaGhokJ2djaamJlZWVkyaNIknT57g6OgoExWEEEKIQkDWgSvmWrZsyb59+xg1ahSffvopkLfsh5qaGllZWURHR7No0SJKliyJu7u7aosVQgghBCABrtgzNTUlMzOTsmXLMmzYMBo0aABAbm4uampq5OTkcPDgQc6dO0eHDh3Q0dFRccVCCCGEkABXzJ09e5YdO3YQFhaGpaUlI0aMwNzcHMgbDweQkZHB3LlzqVGjBi4uLqosVwghhBBIgCv2NDQ0aN68OdHR0SxevJgqVaowZcoUrl+/jp+fHwCampo8e/aM4OBgqlatquKKhRBCCCEL+RZjampq3Lp1iytXrmBubk5kZCQaGhosXLiQ5ORkDh06BEBWVhYAly9fpm7duujo6JCRkaHK0oUQQohiTQJcMZa/u4KamhoNGjTg119/ZdSoUdy5c4f09HR69+5NcnIyZ8+eBeD48eMkJCRIeBNCCCFUTLpQBWfOnKFatWpERUWRnJxMmzZtWLhwIW3btsXKyqrAsTdu3FBNkUIIIYRQSAuc4OLFi3zzzTccP34cNzc3cnJy2LVrF0lJSRw/flzV5QkhhBDiT2QhX4GOjg6dOnXi9OnTPHr06JX3ZfcFIYQQonCRACeEEEIIUcTIGDghhBBCiCJGApwQQgghRBEjAU4IIYQQooiRACeEEEIIUcRIgBNCCCGEKGIkwAkhhBBCFDES4IQQQgghihgJcEIIIYQQRYwEOCGEEEKIIkYCnBBCCCFEESMBTgghhBCiiJEAJ4QQQghRxEiAE0IIIYQoYv4fLbxcacyXI7kAAAAASUVORK5CYII=\n" - }, - "metadata": {} - } - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "A100", - "machine_shape": "hm", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - } + "cells": [ + { + "cell_type": "markdown", + "id": "6f678e62-7bcb-4405-86ae-dce94f494303", + "metadata": { + "id": "6f678e62-7bcb-4405-86ae-dce94f494303" + }, + "source": [ + "# Efficient Multi-Head Attention Implementations" + ] }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file + { + "cell_type": "markdown", + "id": "b742938a-4bfc-4527-a1f1-d5963508967d", + "metadata": { + "id": "b742938a-4bfc-4527-a1f1-d5963508967d" + }, + "source": [ + "This code notebook compares different ways to implement causal multi-head attention used in decoder-style LLMs like GPT, Llama, etc." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7898551e-f582-48ac-9f66-3632abe2a93f", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7898551e-f582-48ac-9f66-3632abe2a93f", + "outputId": "7d088260-3fa1-44f2-bd65-2a46e289f9d4" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PyTorch version: 2.2.1+cu121\n", + "Running on cuda\n" + ] + } + ], + "source": [ + "import torch\n", + "\n", + "torch.manual_seed(123)\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(f\"PyTorch version: {torch.__version__}\")\n", + "print(f\"Running on {device}\")\n", + "\n", + "batch_size = 8\n", + "context_len = 1024\n", + "embed_dim = 768\n", + "embeddings = torch.randn((batch_size, context_len, embed_dim), device=device)" + ] + }, + { + "cell_type": "markdown", + "id": "2f9bb1b6-a1e5-4e0a-884d-0f31b374a8d6", + "metadata": { + "id": "2f9bb1b6-a1e5-4e0a-884d-0f31b374a8d6" + }, + "source": [ + "## 1) CausalAttention MHA wrapper class from chapter 3" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "297c93ed-aec0-4896-bb89-42c4b294d3d1", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "297c93ed-aec0-4896-bb89-42c4b294d3d1", + "outputId": "f8a33752-2cd6-4101-8feb-9d1699984719" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 1024, 768])\n" + ] + } + ], + "source": [ + "from ch03 import MultiHeadAttentionWrapper as Ch03_MHA_Wrapper\n", + "\n", + "mha_ch03_wrapper = Ch03_MHA_Wrapper(\n", + " d_in=embed_dim,\n", + " d_out=embed_dim//12,\n", + " block_size=context_len,\n", + " dropout=0.0,\n", + " num_heads=12,\n", + " qkv_bias=False\n", + ").to(device)\n", + "\n", + "out = mha_ch03_wrapper(embeddings)\n", + "print(out.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "21930804-b327-40b1-8e63-94dcad39ce7b", + "metadata": { + "id": "21930804-b327-40b1-8e63-94dcad39ce7b" + }, + "source": [ + "## 2) The multi-head attention class from chapter 3" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4ee6a61b-d25c-4a0c-8a59-f285544e3710", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4ee6a61b-d25c-4a0c-8a59-f285544e3710", + "outputId": "b704a040-3547-422c-ecda-df9982a2da35" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 1024, 768])\n" + ] + } + ], + "source": [ + "from ch03 import MultiHeadAttention as Ch03_MHA\n", + "\n", + "mha_ch03 = Ch03_MHA(\n", + " d_in=embed_dim,\n", + " d_out=embed_dim,\n", + " block_size=context_len,\n", + " dropout=0.0,\n", + " num_heads=12,\n", + " qkv_bias=False\n", + ").to(device)\n", + "\n", + "out = mha_ch03(embeddings)\n", + "print(out.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "73cd11da-ea3b-4081-b483-c4965dfefbc4", + "metadata": { + "id": "73cd11da-ea3b-4081-b483-c4965dfefbc4" + }, + "source": [ + "## 3) An alternative multi-head attention with combined weights" + ] + }, + { + "cell_type": "markdown", + "id": "1fa1a5ea-eaff-4d2d-aaf0-b34cdb6fd4dd", + "metadata": { + "id": "1fa1a5ea-eaff-4d2d-aaf0-b34cdb6fd4dd" + }, + "source": [ + "- The code for the `MultiHeadAttentionAlt` class below is based on code that was kindly shared by [Rayed Bin Wahed](https://github.com/rasbt/LLMs-from-scratch/discussions/51)\n", + "- The main difference between the `MultiHeadAttentionAlt` class and the `MultiHeadAttention` class used in chapter 3 is that `MultiHeadAttentionAlt` uses a single weight matrix, `self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)` instead of separate weight matrices:\n", + "\n", + " - `self.W_query = nn.Linear(d_in, d_out, bias=qkv_bias)`\n", + " - `self.W_key = nn.Linear(d_in, d_out, bias=qkv_bias)`\n", + " - `self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)`\n", + "\n", + "- Here, `self.qkv` combines all three weight matrices `self.W_query`, `self.W_key`, and `self.W_value` to carry out the query, key, and value computation in a single step\n", + "- Using `q, k, v = qkv.unbind(0)`, we obtain the individual query, key, and value tensors, which are then used similarly to the query, key, and value tensors in the `MultiHeadAttention` class in chapter 3" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9a6bd0a2-f27c-4602-afa0-c96cd295c1a6", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9a6bd0a2-f27c-4602-afa0-c96cd295c1a6", + "outputId": "5d948671-176f-4633-bede-97767e36becc" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 1024, 768])\n" + ] + } + ], + "source": [ + "import torch.nn as nn\n", + "\n", + "\n", + "class MultiHeadAttentionCombinedQKV(nn.Module):\n", + " def __init__(self, d_in, d_out, num_heads, block_size, dropout=0.0, qkv_bias=False):\n", + " super().__init__()\n", + "\n", + " assert d_out % num_heads == 0, \"embed_dim is indivisible by num_heads\"\n", + "\n", + " self.num_heads = num_heads\n", + " self.block_size = block_size\n", + " self.head_dim = d_out // num_heads\n", + "\n", + " self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)\n", + " self.proj = nn.Linear(d_in, d_out)\n", + " self.dropout = nn.Dropout(dropout)\n", + "\n", + " self.register_buffer(\n", + " \"mask\", torch.triu(torch.ones(block_size, block_size), diagonal=1)\n", + " )\n", + "\n", + " def forward(self, x):\n", + " batch_size, num_tokens, embed_dim = x.shape\n", + "\n", + " # (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)\n", + " qkv = self.qkv(x)\n", + "\n", + " # (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)\n", + " qkv = qkv.reshape(batch_size, num_tokens, 3, self.num_heads, self.head_dim)\n", + "\n", + " # (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)\n", + " qkv = qkv.permute(2, 0, 3, 1, 4)\n", + "\n", + " # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_head, num_tokens, head_dim)\n", + " queries, keys, values = qkv.unbind(0)\n", + "\n", + " # (b, num_heads, num_tokens, head_dim) --> (b, num_heads, num_tokens, num_tokens)\n", + " attn_scores = queries @ keys.transpose(-2, -1)\n", + " attn_scores = attn_scores.masked_fill(\n", + " self.mask.bool()[:num_tokens, :num_tokens], -torch.inf\n", + " )\n", + "\n", + " attn_weights = torch.softmax(attn_scores / keys.shape[-1]**-0.5, dim=-1)\n", + " attn_weights = self.dropout(attn_weights)\n", + "\n", + " # (b, num_heads, num_tokens, num_tokens) --> (b, num_heads, num_tokens, head_dim)\n", + " context_vec = attn_weights @ values\n", + "\n", + " # (b, num_heads, num_tokens, head_dim) --> (b, num_tokens, num_heads, head_dim)\n", + " context_vec = context_vec.transpose(1, 2)\n", + "\n", + " # (b, num_tokens, num_heads, head_dim) --> (b, num_tokens, embed_dim)\n", + " context_vec = context_vec.reshape(batch_size, num_tokens, embed_dim)\n", + "\n", + " context_vec = self.proj(context_vec)\n", + "\n", + " return context_vec\n", + "\n", + "\n", + "mha_combined_qkv = MultiHeadAttentionCombinedQKV(\n", + " d_in=embed_dim,\n", + " d_out=embed_dim,\n", + " block_size=context_len,\n", + " dropout=0.0,\n", + " num_heads=12,\n", + " qkv_bias=False\n", + ").to(device)\n", + "\n", + "out = mha_combined_qkv(embeddings)\n", + "print(out.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "48a042d3-ee78-4c29-bf63-d92fe6706632", + "metadata": { + "id": "48a042d3-ee78-4c29-bf63-d92fe6706632" + }, + "source": [ + "## 4) Multihead attention with PyTorch's scaled dot product attention" + ] + }, + { + "cell_type": "markdown", + "id": "f78e346f-3b85-44e6-9feb-f01131381148", + "metadata": { + "id": "f78e346f-3b85-44e6-9feb-f01131381148" + }, + "source": [ + "- The implementation below uses PyTorch's [`scaled_dot_product_attention`](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html) function, which implements a memory-optimized version of self-attention calld [flash attention](https://arxiv.org/abs/2205.14135)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1b8e5a0d-1f65-4a03-bf6e-723f0cc428f5", + "metadata": { + "id": "1b8e5a0d-1f65-4a03-bf6e-723f0cc428f5" + }, + "outputs": [], + "source": [ + "class MHAPyTorchScaledDotProduct(nn.Module):\n", + " def __init__(self, d_in, d_out, num_heads, block_size, dropout=0.0, qkv_bias=False):\n", + " super().__init__()\n", + "\n", + " assert d_out % num_heads == 0, \"embed_dim is indivisible by num_heads\"\n", + "\n", + " self.num_heads = num_heads\n", + " self.block_size = block_size\n", + " self.head_dim = d_out // num_heads\n", + " self.d_out = d_out\n", + "\n", + " self.qkv = nn.Linear(d_in, 3 * d_out, bias=qkv_bias)\n", + " self.proj = nn.Linear(d_in, d_out)\n", + " self.dropout = dropout\n", + "\n", + " self.register_buffer(\n", + " \"mask\", torch.triu(torch.ones(block_size, block_size), diagonal=1)\n", + " )\n", + "\n", + " def forward(self, x):\n", + " batch_size, num_tokens, embed_dim = x.shape\n", + "\n", + " # (b, num_tokens, embed_dim) --> (b, num_tokens, 3 * embed_dim)\n", + " qkv = self.qkv(x)\n", + "\n", + " # (b, num_tokens, 3 * embed_dim) --> (b, num_tokens, 3, num_heads, head_dim)\n", + " qkv = qkv.reshape(batch_size, num_tokens, 3, self.num_heads, self.head_dim)\n", + "\n", + " # (b, num_tokens, 3, num_heads, head_dim) --> (3, b, num_heads, num_tokens, head_dim)\n", + " qkv = qkv.permute(2, 0, 3, 1, 4)\n", + "\n", + " # (3, b, num_heads, num_tokens, head_dim) -> 3 times (b, num_heads, num_tokens, head_dim)\n", + " queries, keys, values = qkv.unbind(0)\n", + "\n", + " use_dropout = 0. if not self.training else self.dropout\n", + " context_vec = nn.functional.scaled_dot_product_attention(\n", + " queries, keys, values, attn_mask=None, dropout_p=use_dropout, is_causal=True)\n", + "\n", + " # Combine heads, where self.d_out = self.num_heads * self.head_dim\n", + " context_vec = context_vec.transpose(1, 2).contiguous().view(batch_size, num_tokens, self.d_out)\n", + "\n", + " return context_vec" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "fbc8ba92-3471-41cb-b1b2-4c0ef5be392b", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "fbc8ba92-3471-41cb-b1b2-4c0ef5be392b", + "outputId": "af9e4855-7f20-4d61-8532-4827df8dfb30" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 1024, 768])\n" + ] + } + ], + "source": [ + "mha_pytorch_scaled = MHAPyTorchScaledDotProduct(\n", + " d_in=embed_dim,\n", + " d_out=embed_dim,\n", + " block_size=context_len,\n", + " dropout=0.0,\n", + " num_heads=12,\n", + " qkv_bias=False\n", + ").to(device)\n", + "\n", + "out = mha_pytorch_scaled(embeddings)\n", + "print(out.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "351c318f-4835-4d74-8d58-a070222447c4", + "metadata": { + "id": "351c318f-4835-4d74-8d58-a070222447c4" + }, + "source": [ + "## 5) Using PyTorch's torch.nn.MultiheadAttention" + ] + }, + { + "cell_type": "markdown", + "id": "74a6d060-6324-48fa-a35c-cb09f2a48965", + "metadata": { + "id": "74a6d060-6324-48fa-a35c-cb09f2a48965" + }, + "source": [ + "- Below, we use PyTorch's [torch.nn.MultiheadAttention](https://pytorch.org/docs/stable/generated/torch.nn.MultiheadAttention.html) implementation" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "3799c7ef-3155-42c6-a829-f95656453ae0", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3799c7ef-3155-42c6-a829-f95656453ae0", + "outputId": "2a085df8-0445-4818-9978-6dc74469f568" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 1024, 768])\n" + ] + } + ], + "source": [ + "import torch.nn as nn\n", + "\n", + "\n", + "class MHAPyTorchClass(nn.Module):\n", + " def __init__(self, d_in, d_out, num_heads, block_size, dropout=0.0, qkv_bias=False, need_weights=True):\n", + " super().__init__()\n", + "\n", + " self.block_size = block_size\n", + " self.multihead_attn = nn.MultiheadAttention(\n", + " embed_dim=d_out,\n", + " num_heads=num_heads,\n", + " dropout=dropout,\n", + " bias=qkv_bias,\n", + " add_bias_kv=qkv_bias,\n", + " batch_first=True,\n", + " )\n", + "\n", + " self.need_weights = need_weights\n", + " self.proj = nn.Linear(d_out, d_out)\n", + " self.register_buffer(\"mask\", torch.triu(torch.ones(block_size, block_size), diagonal=1).bool())\n", + "\n", + " def forward(self, x):\n", + " batch_size, num_tokens, _ = x.shape\n", + "\n", + " # Ensure attn_mask is compatible with expected shape and `batch_first=True`\n", + " # No need to manually adjust for num_heads; ensure it's right for the sequence\n", + " if self.block_size >= num_tokens:\n", + " attn_mask = self.mask[:num_tokens, :num_tokens]\n", + " else:\n", + " attn_mask = self.mask[:self.block_size, :self.block_size]\n", + "\n", + " # attn_mask broadcasting will handle batch_size dimension implicitly\n", + " attn_output, _ = self.multihead_attn(\n", + " x, x, x, attn_mask=attn_mask, need_weights=self.need_weights\n", + " )\n", + "\n", + " output = self.proj(attn_output)\n", + "\n", + " return output\n", + "\n", + "\n", + "mha_pytorch_class_default = MHAPyTorchClass(\n", + " d_in=embed_dim,\n", + " d_out=embed_dim,\n", + " block_size=context_len,\n", + " dropout=0.0,\n", + " num_heads=12,\n", + " qkv_bias=False\n", + ").to(device)\n", + "\n", + "out = mha_pytorch_class_default(embeddings)\n", + "print(out.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "a3953bff-1056-4de2-bfd1-dfccf659eee4", + "metadata": { + "id": "a3953bff-1056-4de2-bfd1-dfccf659eee4" + }, + "source": [ + "## 6) Using PyTorch's torch.nn.MultiheadAttention with `scaled_dot_product_attention`" + ] + }, + { + "cell_type": "markdown", + "id": "d2164859-31a0-4537-b4fb-27d57675ba77", + "metadata": { + "id": "d2164859-31a0-4537-b4fb-27d57675ba77" + }, + "source": [ + "- Set `need_weights` (default `True`) to need_weights=False so that MultiheadAttention uses `scaled_dot_product_attention` [according to the documentation](https://github.com/pytorch/pytorch/blob/71d020262793542974cf13b30f2a9099773f015c/torch/nn/modules/activation.py#L1096)\n", + "\n", + "> need_weights: If specified, returns ``attn_output_weights`` in addition to ``attn_outputs``.\n", + " Set ``need_weights=False`` to use the optimized ``scaled_dot_product_attention``\n", + " and achieve the best performance for MHA.\n", + " Default: ``True``." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4a4c2afe-5e1f-4bd7-a118-67031176f147", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4a4c2afe-5e1f-4bd7-a118-67031176f147", + "outputId": "234771f4-8a53-4478-8a9b-cf19f79a5e07" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([8, 1024, 768])\n" + ] + } + ], + "source": [ + "mha_pytorch_class_noweights = MHAPyTorchClass(\n", + " d_in=embed_dim,\n", + " d_out=embed_dim,\n", + " block_size=context_len,\n", + " dropout=0.0,\n", + " num_heads=12,\n", + " qkv_bias=False,\n", + " need_weights=False # NEW!\n", + ").to(device)\n", + "\n", + "out = mha_pytorch_class_noweights(embeddings)\n", + "print(out.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "8877de71-f84f-4f6d-bc87-7552013b6301", + "metadata": { + "id": "8877de71-f84f-4f6d-bc87-7552013b6301" + }, + "source": [ + "## Quick speed comparison (M3 Macbook Air CPU)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a97c0b2e-6593-49d8-98bc-2267b3aa610f", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "a97c0b2e-6593-49d8-98bc-2267b3aa610f", + "outputId": "ebe635b2-5c03-4e9b-da3a-951d308acf7b" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "200 ms ± 5.98 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "## 1) CausalAttention MHA wrapper class from chapter 3\n", + "%timeit mha_ch03_wrapper(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19db9c2c-8e75-431a-8eef-0b4d8284e6e6", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "19db9c2c-8e75-431a-8eef-0b4d8284e6e6", + "outputId": "c6e7bcff-661c-45a6-da82-b1e3f89cf761" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "198 ms ± 6.66 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "## 2) The multi-head attention class from chapter 3\n", + "%timeit mha_ch03(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa526ee0-7a88-4f34-a49a-f8f97da83779", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aa526ee0-7a88-4f34-a49a-f8f97da83779", + "outputId": "92b634f8-43f8-468f-87a1-bb774b64c212" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "236 ms ± 13.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "## 3) An alternative multi-head attention with combined weights\n", + "%timeit mha_combined_qkv(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc2b4256-16d8-4c34-9fd0-d4b4af0e60fa", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "cc2b4256-16d8-4c34-9fd0-d4b4af0e60fa", + "outputId": "80c6e314-0771-470e-b090-628984ce2d85" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "71.6 ms ± 3.32 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" + ] + } + ], + "source": [ + "## 4) Multihead attention with PyTorch's scaled dot product attention\n", + "%timeit mha_pytorch_scaled(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f209e70-ebb6-4a1a-b608-1ff42e41c01d", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0f209e70-ebb6-4a1a-b608-1ff42e41c01d", + "outputId": "3cd37b53-04d4-4dd0-9450-6fc8ebaac083" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "217 ms ± 4.27 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "## 5) Using PyTorch's torch.nn.MultiheadAttention\n", + "%timeit mha_pytorch_class_default(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f4968c2-8d40-4ab9-8dba-052b4f77d756", + "metadata": { + "id": "3f4968c2-8d40-4ab9-8dba-052b4f77d756", + "outputId": "2e86bdb4-7fa0-4051-b000-4a2b591060a2", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "205 ms ± 3.9 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" + ] + } + ], + "source": [ + "## 6) Using PyTorch's torch.nn.MultiheadAttention disabling `need_weights`\n", + "%timeit mha_pytorch_class_noweights(embeddings)" + ] + }, + { + "cell_type": "markdown", + "id": "a78ff594-6cc2-496d-a302-789fa104c3c9", + "metadata": { + "id": "a78ff594-6cc2-496d-a302-789fa104c3c9" + }, + "source": [ + "## Quick speed comparison (Nvidia A100 GPU)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "707a2a14-a089-48a8-88aa-d328e1e0a9d0", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "707a2a14-a089-48a8-88aa-d328e1e0a9d0", + "outputId": "e99a17e9-8139-4b04-dac8-fa1dd5027735" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8.35 ms ± 1.44 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "## 1) CausalAttention MHA wrapper class from chapter 3\n", + "%timeit mha_ch03_wrapper(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "8686dd69-3655-40e4-a57b-a2c55532a010", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "8686dd69-3655-40e4-a57b-a2c55532a010", + "outputId": "5553b42c-b709-41a4-8a8b-be36dae408ab" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6.59 ms ± 231 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "## 2) The multi-head attention class from chapter 3\n", + "%timeit mha_ch03(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "2209d7df-e54b-4910-ae2b-c78cf684d9bf", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2209d7df-e54b-4910-ae2b-c78cf684d9bf", + "outputId": "01b0da88-510b-4b21-919a-0a7519a55ed8" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7.21 ms ± 716 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "## 3) An alternative multi-head attention with combined weights\n", + "%timeit mha_combined_qkv(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "1075abe2-4839-4fd6-af3e-c09bb3651e26", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "1075abe2-4839-4fd6-af3e-c09bb3651e26", + "outputId": "542706db-5041-45ca-f667-9e1bd1c2c7aa" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.38 ms ± 362 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" + ] + } + ], + "source": [ + "## 4) Multihead attention with PyTorch's scaled dot product attention\n", + "%timeit mha_pytorch_scaled(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "868e3670-8edc-47bc-9e06-eb505e44dc9d", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "868e3670-8edc-47bc-9e06-eb505e44dc9d", + "outputId": "13cfc808-2b11-4041-fe67-e5a63abe4f28" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6.67 ms ± 408 ns per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "## 5) Using PyTorch's torch.nn.MultiheadAttention\n", + "%timeit mha_pytorch_class_default(embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "944870e6-de54-4e3b-a455-b8f21f6f92c8", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "944870e6-de54-4e3b-a455-b8f21f6f92c8", + "outputId": "c52858e7-999c-4782-adc9-731f8d69dfa6" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4.54 ms ± 7.17 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "## 6) Using PyTorch's torch.nn.MultiheadAttention disabling `need_weights`\n", + "%timeit mha_pytorch_class_noweights(embeddings)" + ] + }, + { + "cell_type": "markdown", + "id": "dabc6575-0316-4640-a729-e616d5c17b73", + "metadata": { + "id": "dabc6575-0316-4640-a729-e616d5c17b73" + }, + "source": [ + "## Speed comparison (Nvidia A100 GPU) with warmup" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "29b63d3d-6d0b-43bb-9c68-d5514dc81000", + "metadata": { + "id": "29b63d3d-6d0b-43bb-9c68-d5514dc81000" + }, + "outputs": [], + "source": [ + "# CUDA benchmark code shared by Andrei Aksionov\n", + "# and based on code from\n", + "# https://github.com/cuda-mode/lectures/blob/main/lecture1/pytorch_square.py\n", + "\n", + "def time_pytorch_function(func, *input, num_repeats = 1_000):\n", + " # CUDA IS ASYNC so can't use python time module\n", + " start = torch.cuda.Event(enable_timing=True)\n", + " end = torch.cuda.Event(enable_timing=True)\n", + "\n", + " # Warmup\n", + " for _ in range(5):\n", + " func(*input)\n", + " torch.cuda.synchronize()\n", + "\n", + " start.record()\n", + " for _ in range(num_repeats):\n", + " func(*input)\n", + " torch.cuda.synchronize()\n", + " end.record()\n", + " torch.cuda.synchronize()\n", + " return start.elapsed_time(end) / num_repeats" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "CDJAPZaszaqx", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 489 + }, + "id": "CDJAPZaszaqx", + "outputId": "f23e9b83-7fd6-4011-9434-0e6934cf762a" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnAAAAHYCAYAAADNtNW9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAADOM0lEQVR4nOzdd1gUydbA4d+QQVBERMGc1hxX16yYIwoiZgUTyoogCmYExYQYMAfMOec1rVl3zTlhTogYMKGICPL94UcvI6DuXkaC530en7v0dE/V1O3pOV1ddUplZmYWhxBCCCGESDe0UrsCQgghhBDi35EATgghhBAinZEATgghhBAinZEATgghhBAinZEATgghhBAinZEATgghhBAinZEATgghhBAinZEATgghhBAinZEATgghhBAinZEATgghhBAinUk3AVzVqlVZsWIFV65cITw8nKZNmybaZ/DgwVy5coWQkBA2btxIwYIFU6GmQgghhBCapZPaFfheRkZGXLlyhZUrV7J06dJEr7u5ueHs7EyfPn24f/8+Q4cOZd26dVSrVo0PHz58dzmWlpa8ffs2JasuhBBCCPHdjI2Nefz48Vf3UaXHxezDw8Pp3LkzO3bsULZduXKFWbNmMXPmTABMTEwIDg7G1dWVTZs2fdf7WlpacvnyZY3UWQghhBDie5UqVeqrQVy66YH7mnz58pEzZ04OHTqkbIuIiODMmTNUqlQp2QBOT08PfX39RNtLlSolvXBCCCGE+OGMjY25fPnyN+OQDBHAWVhYAPDs2TO17c+ePVNeS0q/fv0YNGhQou1v374lIiIiZSsphBBCCJFC0s0kBk0IDAwkf/78yr9SpUqldpWEEEIIIb4pQwRwT58+BSB79uxq27Nnz668lpTo6GgiIiKUf/LYVAghhBDpQYYI4O7fv09YWBi1atVStpmYmPDrr79y6tSpVKyZEEIIIUTKSzdj4DJlykSBAgWUv/PmzUupUqV4+fIljx49Yu7cuQwYMIA7d+4oaUTCwsLUZqoKIYQQQmQE6SaAK1euHFu3blX+HjNmDACrVq3C1dWVadOmYWRkxOTJk8mSJQsnTpygTZs2/yoHnBBCCCFEepAu88BpiomJCffu3SN//vzpahaqlpYWgwYNwsHBAQsLC8LCwli1ahWTJk36ruN/++03tm3bxrVr17C2tla2d+3ala5du5I3b14AgoODCQgIYN++fZr4GEIIIcRP73tjkXTTAyeS5+7uTteuXenTpw/BwcGUK1eOGTNmEBERwbx58756bObMmZk1axaHDx9ONAkkNDSUUaNGcefOHVQqFe3atWP58uVYW1tz/fp1TX4kIYQQQnyFBHAZQKVKldi5cyd//vknAA8fPsTe3p4KFSp889hJkyaxYcMGYmNjE60vu3v3brW/x4wZQ9euXalYsaIEcEIIIUQqyhCzUH92p06dolatWhQqVAiAkiVLUrlyZfbu3fvV4zp06ED+/PmZMGHCN8vQ0tLCzs4OIyMjTp8+nSL1FkIIIcR/Iz1wGUBgYCAmJiYcP36c2NhYtLW1GTNmDOvXr0/2mIIFC+Lt7U3z5s2JjY1Ndr/ixYuza9cuDAwMePfuHV26dJHeNyGEECKVSQCXAdja2tK6dWucnZ0JDg6mdOnSjBkzhrCwMFavXp1ofy0tLebNm4e/vz+3b9/+6nvfunULa2trMmfOTIsWLZg5cyYtWrSQIE4IIYRIRTILNYH0Ogv14sWLTJ06lQULFijbBgwYgIODA1WqVEm0f+bMmbl79y4xMTHKNi0tLbS0tIiJiaF169YcOXIkybI2btzI3bt3GTBgQMp/ECGEEOInJ7NQfyKGhoZ8+vRJbVtsbCwqlSrJ/SMiIqhevbratu7du1OzZk2cnJx48OBBsmVpaWmhr6//v1daCCGEEP+ZBHAZwO7du+nfvz8hISEEBwdTpkwZXFxcWLlypbKPt7c3lpaW/P7778TFxREcHKz2Hs+ePSMqKkptu7e3N3v37iUkJARjY2Nat25N9erVcXBw+GGfTQghhBCJSQCXAQwePJghQ4YQEBCAubk5YWFhLFmyhICAAGWfHDlykCtXrn/1vubm5syaNYscOXLw5s0brl69ioODAwcPHkzhTyCEEEKIf0PGwCWQXsfACSGEECJj+N5YRPLACSGEEEKkMxLACSGEEEKkMxLACSGEEEKkMxqdxJA3b16qVq1K7ty5MTIy4vnz51y6dIlTp07x4cMHTRYthBBCCJFhaSSAa926Nb169aJcuXI8ffqUsLAwoqKiyJo1K/nz5+fDhw+sX7+eqVOnEhISookqCCGEEEJkWCkewB04cICPHz+yatUqHB0dCQ0NVXtdT0+PSpUqYWdnx759+/Dy8mLr1q0pXQ0hhBBCiAwrxdOI1KlThwMHDnzXvlmzZiVv3rxcuHAhJavwn0kaESGEEEKkplRbSut7gzeAly9f8vLly5SughBCCCFEhqbRWahlypShePHiyt9NmjRh2bJlDB8+HF1dXU0WLYQQQgiRYWk0gJs8eTKFCxcGIF++fAQFBREZGUmLFi3w9fXVZNFCCCGEEBmWRgO4QoUKcenSJQBatmzJsWPH6NWrF66urtjY2GiyaCFEGnTu3DnCw8MT/ZswYUKS+3fu3Jnt27dz+/Ztbt++zcaNG6lQoYLaPtmzZ2fGjBlcuXKFhw8fsnbtWgoWLPgjPo4QQqQajeaBU6lUaGl9jhFr167N7t27AXj06BFmZmaaLDpNM+m5NLWrkKZEBHVJ7SqIH6R+/fpoa2srfxcvXpyNGzeyZcuWJPevXr06Gzdu5OTJk3z48AE3NzfWr19P9erVefz4MQDLli3j48ePdOrUiYiICFxcXNi4cSPVqlUjMjLyh3wuIYT40TTaA3f+/HkGDBhAmzZtqFatGn/++Sfw+XHqs2fPNFm0ECINCg8P5+nTp8q/hg0bcufOHf76668k9+/duzcLFy7k8uXL3Lx5E3d3d7S0tKhVqxbwuZe/UqVKeHp6cu7cOW7duoWnpycGBga0atXqR340IYT4oTQawA0dOpQyZcrg7+/P5MmTuXv3LgAtWrTg5MmTmixaCJHG6erq4uDgwMqVK7/7GCMjI3R0dJTZ63p6egBqK7vExcURHR1NlSpVUrbCQgiRhmj0EerVq1epWbNmou0+Pj7ExsZqsmghRBrXtGlTsmTJwqpVq777GB8fH8LCwjh06BAAN2/e5OHDh3h7e9O/f38iIyNxcXEhV65c5MiRQ1NVF0KIVPfDFrPPlCkTJiYmmJiYoKenh6GhYYq+v5aWFkOGDOHs2bOEhIRw+vRpBgwYkKJlCCFSTqdOndi7dy9hYWHftb+7uzt2dnZ06dJF6XGLiYnB0dGRQoUKcefOHUJCQqhRowZ//vknnz590mT1hRAiVWl8MXt/f3+qV6+OgYGBsl2lUhEXF4eFhUWKleXu7k7Xrl3p06cPwcHBlCtXjhkzZhAREcG8efNSrBwhxP8ud+7c1K5dG0dHx+/av0+fPri7u9OqVSuuXr2q9tqFCxewtrZWbg7Dw8PZs2cP58+f10DNhRAibdBoADdnzhxUKhVubm48e/aMuLgUXbVLTaVKldi5c6cyUeLhw4fY29snSjkghEh9HTp04NmzZ+zZs+eb+/bt25f+/fvj4ODw1aAsfsmZggULUq5cOcaOHZtS1RVCiDRHowFcyZIlqVevHrdu3dJkMQCcOnWKLl26UKhQIW7fvk3JkiWpXLky3t7eyR6jp6eHvr6+8rexsbHG6ynEz06lUtGhQwfWrFmTaCzsrFmzePz4MX5+fgC4ubkxePBgevXqxYMHD5Re+3fv3vHu3Tvg86So8PBwQkJCKFGiBGPHjmXHjh0cPHjwh34ukfFYWlri4+NDvXr1MDQ05O7du/Tt2/erNxKtW7emb9++FCxYkDdv3rBv3z58fHyUiTedO3embdu2yipFFy5cYPTo0Zw9e/ZHfCSRgWg0gDt37hy5cuX6IQFcYGAgJiYmHD9+nNjYWLS1tRkzZgzr169P9ph+/foxaNAgjddNCPGP2rVrkydPHlasWJHotVy5cqmNXevatSv6+vosXrxYbT9/f38l+W/OnDkZPXo02bNn58mTJ6xZs4aJEydq9DOIjC9Llizs2LGDo0eP0rZtW54/f07BggV59epVssf89ttvzJo1i+HDh7Nr1y4sLS2ZNGkSgYGBynCB78ltKMT3UJmZmWnsuWb+/PmZNGkS69at49q1a3z8+FHt9S/Hsvwv7OzsGDlyJD4+PgQHB1O6dGnGjBmDt7c3q1evTvKYpHrgLl++TP78+ZXHMZogiXzVSSJfIURaM2LECH777TeaN2/+3cf06dOHrl27UrFiRWVbz549cXNzo3Tp0kkeo6WlxZ07dxg0aBBr1qz5n+st0j8TExPu3bv3zVhEo7NQzc3NyZ8/P9OnT2fv3r0cOnSIgwcPKv+bkkaOHMnUqVPZtGkT165dY+3atcyZM4d+/fole0x0dDQRERHKv7dv36ZonUT6ZmlpyZw5c7h58yYhISEcOXKEcuXKffUYPT09hg0bxvnz5wkNDeXcuXN06NBBeX3Lli1JLiX1b1JpCCE0r3Hjxpw/f56FCxcSHBzMgQMH6Ny581ePOXXqFLly5aJ+/frA52XebGxslLHZSfkyt6EQ30ujj1CnTZvGpUuXcHZ25unTpxqdxGBoaJgobUBsbCwqlUpjZYqM6788PgFYuHAh2bNnx93dnTt37pAjRw5lOTkAR0dHJfksQNasWTl8+DBbt27V1EcRQvwH+fLlo2vXrsyePZspU6ZQvnx5xo0bx8ePH5N9qnPy5El69erFggUL0NfXR1dXl507dzJw4MBky/kyt6EQ30ujAVzu3Lnp2LGjsgKDJu3evZv+/fsTEhJCcHAwZcqUwcXF5V9leRcinru7O48ePaJv377KtgcPHnz1mLp161KtWjUqVKigBHoPHz5U2+fLANDOzo73798nuxaoECJ1aGlpcf78eUaPHg3ApUuXKF68OE5OTskGcEWLFmXcuHEEBASwf/9+cuTIwciRI5k0aRLu7u6J9o/PbdiiRQu11USE+B4afYR65MgRSpUqpckiFIMHD2br1q0EBARw7NgxRo4cyZIlSySVgPhP/svjkyZNmnD+/Hnc3Ny4fPkyJ06cYOTIkWo5EL/UqVMnNm7cKIuuC5HGPHnyhOvXr6ttu3HjBrlz5072mH79+nHixAlmzJjB1atXOXDgAF5eXnTq1CnRyiDxuQ1bt26douPBxc9Doz1wu3fvZvTo0RQvXjzJSQy7du1KsbLevn3LsGHDGDZsWIq9p/h5/ZfHJ/ny5aNy5cpERUXRpUsXzMzMCAgIwMzMTK0nL16FChUoUaJEknfmQojUdeLECQoXLqy2rVChQol61RMyNDQkJiZGbVt8qpyEw3m+N7ehEF+j0QBu0qRJAHh5eSV6LaVXYhAiJf2XxydaWlrExcXRq1cvZeaQt7c3ixYtwsvLi6ioKLX9O3bsyJUrVyT/kxBp0Jw5c9i5cyceHh5s3ryZChUq0KVLF/r376/s4+3tjaWlJb///jvwudNiypQpdO3aVXmEOnbsWM6cOaMsGfc9uQ2F+B4aDeCyZ8+uybcXQmOSe3xiY2Pz1WMeP36sNu37xo0baGlpYWVlxZ07d5TtRkZGtGrVinHjxqV85YUQ/7Nz587RpUsXvL298fT05MGDBwwbNkwtt2iOHDnIlSuX8veqVaswNjamR48ejBo1ijdv3nDkyBFGjhyp7PM9uQ2F+B4aDeCESK/+y+OTEydO0KJFCzJlyqTcSRcqVIjY2FhCQ0PV9m3ZsiV6enqsW7cu5SsvhEgRe/bs+epyb66urom2BQUFERQUlOwx5cuXT5G6CZHikxjs7Oy+e18rKyt+++23lK6CEP+zOXPmULFiRTw8PChQoAD29vZ06dKFBQsWKPt4e3sza9Ys5e8NGzbw8uVLpk+fTtGiRalatSq+vr6sWLEiycenO3bskNxPQggh/pMUD+C6du3KsWPH6Nu3L7/88kui101MTKhfvz5z587lwIEDmJmZpXQVhPifxT8+adWqFUePHsXT0/Obj0/evXuHvb09WbJkYe/evcydO5fdu3czZMgQtfcuXLgwVatWTXIpKSGEEOJ7aGQprcaNG9OzZ09q1qxJZGQkT58+5cOHD5iammJhYUF4eDirV69m9uzZPHv2LKWL/8++d/mK/7kcWUpLjSylJYQQQnz2vbGIRsbA7dq1i127dmFmZkaVKlXInTs3hoaGhIeHc+nSJS5evKjRVRmEEClDbjYSkxsOIURaoNFJDC9evGDHjh2aLEIIIYQQ4qej0ZUYhBBCCCFEypMATgghhBAinZEATgghhBAinZEATgghhBAinfkhAZyuri6FCxdGW1v7RxQnhBBCCJGhaTSAMzQ0ZOrUqYSEhPDXX3+RO3duAMaPH4+7u7smixZCCCGEyLA0mkbE29ubUqVK0aJFC9auXatsP3ToEAMHDmTq1KmaLF4IIcRPQnIWJiY5CzM2jQZwTZs2pUePHpw+fVpte3BwMAUKFNBk0UIIIYQQGZZGH6Fmy5YtyaWyjIyMZCUGIYQQQoj/SKM9cOfPn6dhw4YEBQUBKEFb586dOXXqlCaLFj8ZeXySmDw+EUKIjEujAdzo0aNZu3YtRYsWRVtbm169elG0aFEqVapEixYtNFm0EEIIIUSGpdFHqCdOnKB27dpoa2tz7do16tSpw/Pnz2ncuDEXLlzQZNFCCCGEEBmWRnvgAO7du4eHh4emixFCCCGE+GloPIADMDc3x9zcHC0t9Q6/q1ev/ojihRBCCCEyFI0GcGXLlmXmzJn88ssvqFQqtdfi4uKwsLDQZPFCCCGEEBmSRgO4adOmcfv2bdzd3Xn69KmkDhFCCCGESAEaDeDy58+Pk5MTd+/e1WQxCktLS3x8fKhXrx6GhobcvXuXvn37cv78+R9SvhBCCCHEj6DRAO7w4cOUKlXqhwRwWbJkYceOHRw9epS2bdvy/PlzChYsyKtXrzRethBCCCHEj6TRAM7d3Z2ZM2dSrFgxgoOD+fjxo9rru3btStGyHj16RN++fZVtDx48SLH3F0IIIYRIKzQawFWqVInKlStTv379RK+l9CSGxo0bs3//fhYuXEi1atV4/PgxCxcuZNmyZSlWhhBCCCFEWqDRRL7jx49n3bp1lChRguzZs6v9S+kZqPny5aNr167cuXMHBwcHFi1axLhx42jXrl2yx+jp6WFiYqL8MzY2TtE6CSGEEEJogkZ74MzMzJg9e3aSC9qnNC0tLc6fP8/o0aMBuHTpEsWLF8fJyYnVq1cneUy/fv0YNGiQxusmhBBCCJGSNNoDt337dmrUqKHJIhRPnjzh+vXrattu3LhB7ty5kz0mMDCQ/PnzK/9KlSql6WoKIYQQ4v+5u7sTHh7OmDFjvrpf5syZmTBhAleuXCE0NJQTJ06oDc8yNjZmzJgxnD9/npCQEHbu3En58uU1Xf1UpdEeuNu3b+Pt7U2VKlW4evUqMTExaq/Pmzcvxco6ceIEhQsXVttWqFAhHj58mOwx0dHRREdHp1gdhBBCCPF9ypcvj6OjI5cvX/7qfrq6umzcuJFnz57RtWtXHj9+TJ48eXj9+rWyT2BgIMWLF8fFxYWwsDAcHBzYuHGjMiY+I9JoANepUyfevXtHtWrVqFatmtprcXFxKRrAzZkzh507d+Lh4cHmzZupUKECXbp0oX///ilWhhBCCCH+d5kyZWLOnDl4eHh883e6Y8eOmJqa0rhxY6UjKGHnjIGBATY2NnTq1Iljx44BMGHCBBo1akTXrl0ZO3as5j5IKtJoAFehQgVNvr2ac+fO0aVLF7y9vfH09OTBgwcMGzaM9evX/7A6CCGEEOLbJkyYwJ9//smhQ4e+GcA1btyY06dPM2HCBJo0aUJ4eDgbNmxg6tSpfPr0CR0dHXR0dPjw4YPacVFRUVSuXFmTHyNV/ZDF7H+UPXv2sGfPntSuhhBCCCGSYWdnR5kyZZJMMZaU/PnzkydPHtavX0+7du0oUKAAAQEB6OjoEBAQwNu3bzl58iQDBgzgxo0bPH36FHt7eypVqvTDVoJKDSkewPn5+TFu3DgiIyPx8/P76r7e3t4pXbwQQggh0igrKyvGjh2Lvb19oh6z5KhUKp4/f46HhwefPn3iwoULWFpa4urqSkBAAAAuLi5MmzaNK1euEBMTw8WLF9m4cSNly5bV5MdJVSkewJUuXRodHR3lv4UQQgghAMqVK4eFhQUHDhxQtuno6FCtWjV69OiBpaUlnz59UjvmyZMnxMTEqG2/ceMGOXPmRFdXl48fP3Lv3j1atGiBkZERJiYmPHnyhPnz53Pv3r0f9dF+uBQP4GxtbZP8byGEEEL83A4fPkz16tXVts2YMYObN28qY9q+dPLkSezt7VGpVMTFxQGfs0yEhYUlWqIzMjKSyMhIsmTJQt26dfH19dXYZ0ltGs0DN23atCRXNzAyMmLatGmaLFoIIYQQaczbt28JDg5W+/fu3TtevHhBcHAwALNmzVIbYrVw4UKyZs3KuHHjKFSoEA0aNMDDw4MFCxYo+9SpU4e6deuSN29erK2t2bJlCzdv3mTlypU//DP+KBoN4Nq1a4eBgUGi7QYGBrRt21aTRQshhBAiHcqVKxc5cuRQ/g4NDaV169aUL1+ew4cPM27cOObNm0dgYKCyT3yi3+PHjzNz5kxOnDhB69atE+WfzUg0MgvVxMQE+Dzw0NjYWG2gopaWFg0aNOD58+eaKFoIIYQQ6UjLli2/+jfA6dOnadSoUbLvsWXLFrZs2ZLidUvLNBLA3blzh7i4OOLi4jh58mSi1+Pi4vD399dE0UIIIYQQGZ5GAriWLVuiUqnYvHkzTk5OvHz5UnktOjqakJAQwsLCNFG0EEIIIUSGp5EA7u+//wY+r3MWEhKiiSKEEEIIIX5aGp3EIMGbEEIIIUTK02gAJ4QQQgghUp4EcEIIIYQQ6YwEcEIIIYQQ6YwEcEIIIYQQ6YxGZqHGy549O6NGjaJWrVqYm5ujUqnUXrewsNBk8UIIIYQQGZJGA7gZM2aQO3duJk6cyJMnT5RFaIUQQgghxH+n0QCuSpUqNGvWjMuXL2uyGCGEEEKIn4pGA7hHjx4lemwqhBBCiPTBpOfS1K5CmhMR1CW1qwBoeBLD0KFDGTFiBHny5NFkMUIIIYQQPxWN9sAtWLAAQ0NDzpw5w/v37/n48aPa64ULF9Zk8UIIIYQQGZJGA7hhw4Zp8u2FEEIIIX5KGg3gVq9ercm3F0IIIYT4KWk0gAPQ0tKiWbNm/PLLLwAEBwezc+dOPn36pOmihRBCCCEyJI0GcAUKFGD16tVYWlpy69YtANzd3QkNDaVdu3bcu3dPk8ULIYQQQmRIGp2FOm7cOO7du0eZMmWoW7cudevWpWzZsty/f59x48ZpsmghhBBCiAxLowFctWrV8PX15dWrV8q2ly9fMmrUKKpVq6bJonF3dyc8PJwxY8ZotBwhhBBCiB9NowFcdHQ0xsbGibZnypQpUUqRlFS+fHkcHR1lBQghhBBCZEgaDeD27NnDlClT+PXXX5VtFStWZNKkSezatUsjZWbKlIk5c+bg4eGh1vMnhBBCCJFRaDSAGzx4MPfu3WPXrl2EhoYSGhrKjh07uHv3LkOGDNFImRMmTODPP//k0KFD39xXT08PExMT5V9SvYVCCCGEEGmNRmehvnnzhk6dOlGwYEGKFCkCwI0bN7h7965GyrOzs6NMmTLUr1//u/bv168fgwYN0khdhBBCCCE0ReN54ADu3LnDnTt3NFqGlZUVY8eOxd7eng8fPnzXMYGBgcyePVv529jYWMbNCSGEECLNS/EAzs/Pj3HjxhEZGYmfn99X9/X29k6xcsuVK4eFhQUHDhxQtuno6FCtWjV69OiBpaVlouTB0dHRREdHp1gdhBBCCCF+hBQP4EqXLo2Ojo7y3z/K4cOHqV69utq2GTNmcPPmTaZOnSorPwghhBAiw0jxAM7W1jbJ/9a0t2/fEhwcrLbt3bt3vHjxItF2IYQQQoj0TKOzUKdNm5bkzE4jIyOmTZumyaKFEEIIITIsjQZw7dq1w8DAINF2AwMD2rZtq8miAWjZsiXDhg3TeDlCCJGa+vXrx969e7l//z7BwcEsW7aMwoULf/fxdnZ2hIeHs2zZsmT3mThxIuHh4fTq1SslqiyE+B9pJICLz6umUqkwNjZWy7WWJUsWGjRowPPnzzVRtBBC/HSqVavGggULaNiwIfb29ujo6LB+/XqMjIy+eWyePHkYNWoUf//9d7L7NGvWjIoVK/L48eOUrLYQ4n+gkTQid+7cIS4ujri4OE6ePJno9bi4OPz9/TVRtBBC/HTatGmj9rerqys3btygbNmyHDt2LNnjtLS0mDt3LuPHj6dq1apkyZIl0T6WlpaMHz+e1q1bs3r16hSvuxDiv9FIANeyZUtUKhWbN2/GycmJly9fKq9FR0cTEhJCWFiYJooWQoifXubMmQHUrr1J8fLy4vnz56xYsYKqVasmel2lUjF79mymT5/O9evXNVJXIcR/o5EALr4rvnz58oSEhGiiCCGEEElQqVSMGTOG48ePf3UGfuXKlenUqRO1a9dOdh93d3diYmKYN2+eJqoqhPgfaHQlhjx58pAnT55kX/9a174QQoh/LyAggOLFi9OsWbNk9zE2Nmb27Nn069ePFy9eJLlP2bJlcXZ2pm7dupqqqhDif6DRAG7r1q2JtsXFxSn/bWFhocnihRDip+Lv70/Dhg1p3rw5oaGhye6XP39+8uXLx8qVK5VtWlqf57Q9efKEypUrU6VKFbJnz86FCxeUfXR0dPDz86N3796UL19ecx9ECPFNGg3gChYsqPa3rq4uZcqUYciQIYwZM0aTRQshxE/F39+fZs2a0aJFCx48ePDVfW/evJlo5Zphw4ZhbGzMkCFDePToEWvXruXQoUNq+6xfv561a9eqBX5CiNSh0QAuIiIi0baDBw8SHR2Nn58f9erV02TxQgjxUwgICMDe3p5OnTrx9u1b5enGmzdviIqKAmDWrFk8fvwYPz8/Pnz4kGh83OvXrwGU7S9fvkw0CeLjx488efKEW7duafojCSG+QaMBXHKePXv2r5JMCiGESF63bt0A2LZtm9p2V1dXVq1aBUCuXLlkTWghMhCNBnAlSpRQ+1ulUpEjRw7c3d25fPmyJosWQoifRrZs2b65T8uWLb/6uqur6zffQ8a9CZF2aDSAO3ToEHFxcahUKrXtp0+fxs3NTZNFCyGEEEJkWBoN4L68W/v06RPh4eF8+PBBk8UKIYQQQmRoGg3gJImvEEIIIUTK08hi9vHGjRuHs7Nzou09evSQNCJCCCGEEP+RRgM4GxsbTpw4kWj7yZMnadGihSaLFkIIIYTIsDQawGXNmpU3b94k2h4REYGZmZkmixZCCCGEyLA0GsDdvXs3yWS99evX5/79+5osWgghhBAiw9LoJIZZs2bh7+9PtmzZOHLkCAC1atXi999/Z9iwYZosWgghhBAiw9JoALdy5Ur09fXp378/np6eADx48AAvLy/WrFmjyaKFEEIIITIsjS+ltWjRIhYtWkS2bNmIiori3bt3mi5SCCGEECJD0+gYOABtbW1q165N8+bNlRUZcubMSaZMmTRdtBBCCCFEhqTRHrjcuXOzbt06cuXKhb6+PgcPHuTt27e4ubmhp6enPFYVQoifiUnPpaldhTQlIqhLaldBiHRH44l8z58/T6FChYiKilK2//HHH9SqVUuTRQshhBBCZFga7YGrUqUKTZo04ePHj2rbHzx4gKWlpSaLFkIIIYTIsDTaA6elpYW2tnai7VZWVrx9+1aTRQshhBBCZFgaDeAOHDhAr169lL/j4uLIlCkTgwcPZu/evSlaVr9+/di7dy/3798nODiYZcuWUbhw4RQtQwghhBAiLdBoADdixAgqV67M33//jb6+PvPmzePcuXNYWloycuTIFC2rWrVqLFiwgIYNG2Jvb4+Ojg7r16/HyMgoRcsRQgghhEhtGh0DFxoaSq1atbCzs6NkyZIYGxuzfPly1q9frzapISW0adNG7W9XV1du3LhB2bJlOXbsWIqWJYQQQgiRmjQawGXLlo3w8HDWr1/P+vXr1V4rXrw4165d01jZmTNnBuDly5fJ7qOnp4e+vr7yt7GxscbqI4QQQgiRUjT6CPXIkSM0aNAg0fY+ffrw559/aqxclUrFmDFjOH78OMHBwcnu169fP+7du6f8u3z5ssbqJIQQQgiRUjQawM2ePZvFixczceJEDAwMsLS0ZNOmTfTt21dtckNKCwgIoHjx4vTs2fOr+wUGBpI/f37lX6lSpTRWJyGEEEKIlKLRR6jTp0/n4MGDzJ49m8OHD5M1a1bOnDlDrVq1ePr0qUbK9Pf3p2HDhjRv3pzQ0NCv7hsdHU10dLRG6iGEEEIIoSkaXwv17t27XLt2jbx582JiYsLmzZs1Grw1a9YMW1tbHjx4oJEyhBBCCCFSm0YDuN9++43Dhw9TsGBBatWqhaenJ+PHj2f+/PlkyZIlRcsKCAjAwcEBZ2dn3r59i4WFBRYWFhgYGKRoOUIIIYQQqU2jAdzmzZvZvHkzjRo14saNGyxfvhxra2ty587N0aNHU7Ssbt26kSVLFrZt28a1a9eUf3Z2dilajhBCCCFEatPoGLjWrVvz999/q227d+8eTZo0oX///ilaVrZs2VL0/YQQQggh0iqN9sB9GbzFi4uLY9KkSZosWgghhBAiw9JIALd69WpMTEyUv93d3ZXEugBZs2ZNNrgTQgghhBBfp5EArm7dumorHHh4eJA1a1blbx0dHVloXgghhBDiP9JIAKdSqb76txBCCCGE+O80ngdOCCGEEEKkLI0EcHFxccTFxSXaJoQQQggh/ncaSSOiUqmYMWOGskyVvr4+kyZNIjIyEgA9PT1NFCuEEEII8VPQSAC3evVqtb/XrVuXaJ81a9ZoomghhBBCiAxPIwFc3759NfG2QgghhBACmcQghBBCCJHuSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOSAAnhBBCCJHOZLgArnv37pw7d45Hjx6xZ88eKlSokNpVEkIIIYRIURkqgLO1tcXPz4+AgADq1q3L5cuXWbduHebm5qldNSGEEEKIFJOhArjff/+dZcuWsXLlSq5fv86AAQN4//49HTt2TO2qCSGEEEKkmAwTwOnq6lK2bFkOHTqkbIuLi+PQoUNUqlQpFWsmhBBCCJGydFK7AiklW7Zs6Ojo8PTpU7XtT58+pUiRIkkeo6enh76+vvK3sbGx2v9qirF+hmn2lGFi8j+/hbRpEqRdNUPaNeVJm2qGtKtmpEC7fs33xiA/9f8z/fr1Y9CgQYm2X758ORVq8xMbeS+1a5AxSbtqhrRrypM21QxpV834Qe1qbGxMREREsq9nmAAuPDycmJgYLCws1LZbWFgk6pWLFxgYyOzZs9W2Zc2alZcvX2qsnmmFsbExly9fplSpUrx9+za1q5NhSLumPGlTzZB21Qxp15T3M7apsbExjx8//uo+GSaA+/jxIxcuXKBWrVrs2LEDAJVKRa1atZg/f36Sx0RHRxMdHa227WvRbkb09u3bn+4z/wjSrilP2lQzpF01Q9o15f1Mbfo9nzPDBHAAs2bNYubMmZw/f56zZ8/Sq1cvjIyMWLlyZWpXTQghhBAixWSoAG7z5s2Ym5szePBgLCwsuHz5Mm3atOHZs2epXTUhhBBCiBSToQI4gPnz5yf7yFT848OHD/j7+/Phw4fUrkqGIu2a8qRNNUPaVTOkXVOetGnSVGZmZnGpXQkhhBBCCPH9MkwiXyGEEEKIn4UEcEIIIYQQ6YwEcEIIIYQQ6YwEcEIIIYQQ6YwEcBmISqVK7SoIIYQQ4geQAC6DUKlUxMV9nlBcr149ihYtira2dirXSoikyc1GypM21YyE7Wqi4UXMfxZyrqaMDJcH7mcVH7wNHz4cBwcHRo0aRWho6E+z7Iim2NjYULBgQbS1tdm2bRs3b95M7SqlewlvNrp06cLz58/Zv38/UVFRqVyz9Cthm1atWhUjIyOuXbvG48ePle3i30vYrh4eHhQsWJAJEybw8OHDVK5Z+pWwTVu2bImVlRX6+vocOHCACxcupHLt0hcJ4DKQAQMG0KFDB5ycnLh06RLv379P7SqlayNGjMDBwYHz589To0YNKlWqRKdOnYiNjU3tqqVr8RdvHx8f2rRpw9SpUzEwMJAA7n8Q36YjR47E3t4eExMTrl+/zvr161m4cCExMTGpXMP06ctzddy4cXz8+DGVa5W+fdmmhw4dolChQtjZ2bFq1SrmzJmTyjVMPySAyyCyZMlC7dq18ff35+TJk+TMmZMyZcrg4ODAjRs32Lx5M0+fPk3taqYbAwYMoG3btrRv356LFy9SrFgx9uzZQ86cOXn06FFqVy/dc3Z2pn379tjb23PlyhVA/c5c/HvVqlWjWrVqdO3alZcvX+Lq6oqdnR3GxsZMmzZNgrj/qHHjxrRp04YOHTooPUTGxsaYm5vz8uVLXr9+nco1TH9atGhBq1atlDa1t7dnxowZhISEpHbV0hUJ4NKpL3/stLW1MTMzw8zMDBsbG2xsbLCyssLIyIjy5cuTPXt2xo4dKz+Q36FEiRJUqlQJLy8vLl68CMDr16+5ceMGvXr1QktLi3PnzrFhw4ZUrmn68eX5WqpUKZYsWcKVK1fIly8f5cuXx9nZmRs3brBnzx527NiRirVNf5o1a0aDBg04evQop06dAmDYsGEMHz6chg0bEhcXx/Tp0yWI+w5fnqtmZmbcvHmTCxcuULJkSRo1akS7du3Q1dVl3759+Pv7y3rb/1KePHk4d+4cFy5coGXLlkycOJEhQ4awfft2DA0NyZs3L9evX0/taqZ5EsClQwkvMHXr1lXGumzatAknJydMTU2ZP38+S5cu5ejRowQFBWFqairB23d69OgRS5cu5fjx48Dn9t6wYQNxcXFoaWlRrFgxqlevDiBB3HfQ19dX1jCsU6cOBw4cwNzcnDJlyvDw4UNat25NdHQ0t2/f5pdffsHU1JR9+/bJuoffycjIiK5du1K5cmWOHDmibH/37h2jR49m2LBh1K9fHxMTE0aPHs2nT59SsbZpW/78+bl37x4Av//+O8eOHSMkJITq1aszZ84cqlevztGjR5k+fTqGhoa4u7sTFBQkAdxXJNWznilTJkJCQqhYsSLTpk3D19eXxYsXA5/HxVlYWPDw4UMiIyNTocbph8xCTYcSTliYMmUKtWvXRk9Pj0mTJtGmTRvq1KmDn58fR48eBSBr1qy8ffs2Naucrrx+/Zo///yTFy9eANCuXTuePHmCjY0Nw4cPp3Xr1sTExFC7du1Urmna16RJE+XCPHr0aCZOnIienh5ubm68e/cOV1dXDh06xLhx4+jbty/Lly/HzMwMLS25NH2vyMhIevfuzR9//EGhQoVwdHRUXnv37h1jxozh/v37ZMmSRYK3ryhZsiSnTp2iZcuWjBkzhv79+xMREcHhw4fp0aMHHz9+xM/Pj5EjR7Js2TLWr1/PkydPMDY2Tu2qp1kJg7dq1aphZmYGwJEjR+jVqxc7d+6kb9++LFq0CABDQ0NatWqFpaWlBG/fQRazT6e8vLzo1q0bXbp0ITg4ONFs0yxZslCoUCE8PT3JkycP1tbWMvj+K8qXL0/WrFl59OiRMntXS0uLT58+oaurS1xcHDExMcq2mTNn8ubNG4YMGZLaVU/TSpcuzbZt2wgJCSFXrlw0bdqUa9euAZ8v7qamprx8+RL4PAxg5cqVvHjxAhcXl9SsdpqV8AcxZ86cvHv3Dl1dXV68eIGFhQX+/v6Ym5uzevVqVqxYoRxnYGDAhw8fpBc+CZaWljx+/BgANzc3vLy8iI2NpWnTply9elVp8/jvvpaWFvr6+ixevBgjIyNatGgh7foNw4YNo0GDBixZsoSVK1fy4cMH+vTpw9ChQxk+fDgHDx4kc+bMDBs2jOzZs1O/fn35vfoO8gg1HcqaNSu1a9fGx8eHU6dOYWFhwS+//ELr1q05d+4cBw8eJH/+/IwZM4YXL15Qp04dYmNjlQuQUOfj44OtrS1GRka8evWKsLAw+vfvz+3bt1GpVGqzzj59+oSVlRXFihVj2bJlqVjr9OHSpUvs378fGxsbjhw5ojauJS4ujpcvX2JsbEzDhg1p3bo1VlZWdOzYMRVrnLbFBwpeXl40bNiQLFmy8ObNGwICAti9ezeDBg3C39+fdu3aERcXx8qVKwGUGb4yUUTdtGnTKFeuHN27d+fmzZuEhYVhYGDAp0+fKFasGFevXlXa69OnT+jp6dGtWzeaNWuGkZERjRo1Ii4uTtr1K4YMGYKjoyNdunTh6tWrytCIJUuWoK+vj6+vL2/fvuXZs2c8f/6cBg0ayO/Vd5IeuHQoR44cHDx4kMmTJ3P79m0cHBwoWLAgxsbGaGlpsXDhQoKCgqhYsSJnzpwhLi4ObW1tuaNJgr29PePHj6dz587cvXuX3377jY4dO/Lbb7/RqlUrzp8/r1xI4meeLVu2jFu3btG1a9fUrn66YGdnh7a2Nn5+fpw+fZo+ffrw5s0b5fXcuXPTt29fcuTIQffu3YmNjZXz9Ss8PT3p1asXXl5emJqaUrp0aTp37kz//v1Zvnw5OXPmZMyYMZQoUQIfHx/27NmT2lVOs3Lnzs3u3bu5fv067u7uPHz4EAsLC9q3b8/QoUPx8PBQgmAAPT09qlSpQp06dRg9erScq1+oU6cOZ8+eVWbmFipUiKCgIHx8fDhy5AjZsmXDysqKpk2bcujQIY4fP06BAgXIli0bERER3LhxQ36v/gUJ4NK45O7sBg0ahLOzMzo6OixcuJCDBw9y6NAhVqxYwaNHjxg4cOA330N8fmRSsWJFunTpomzLkycPvr6+1K1bl4YNG3Lz5k309fVxc3OjcePG3L17lx49egDStl9KeOE1NTXl7du3yszH8uXLs3r1ak6ePMnvv/+uPPa3s7Nj7969yt9y5/2PL/PjZc6cmVWrVrFq1SqWL18OfD4HPTw8GDJkCM2aNVPSCHXr1o3x48dLWyZDR0eHmJgYLC0t2b9/P7du3aJfv37cvn0b+HyN7d+/P25ubqxZswaAMWPGsH79es6dOwfIuZpQly5d8PPzw8fHhw0bNhAREYGVlRV79uxh7NixXLx4kZ49e/Lrr78CULRoUdq0acOBAwfU3keuqd9PArg0LOGJXKpUKYyNjXn+/Dm3bt0CoFy5ckRGRnLjxg3lmA0bNnDy5En8/f1Tpc7pjaenJ05OTpQtW1btji9XrlxMmDCB7Nmz07ZtW16+fEnevHmpXLky69atA+RCk1C9evU4d+6cMvFjwIABVK9eHVNTUwIDAzl+/DhPnz6lXLlyrF69mvPnzzNr1iz69OlD1qxZlUdR4h8bNmzg7NmzjBkzRtmWM2dO/vrrL4YMGcLatWuV7QYGBixbtozr16/j6+urli5EgozEvvzuWllZsW/fPoKDgxk4cKCy4srAgQPx8vJi8eLFlCpVClNTU2rUqCG9Q8kICAigVq1azJkzh40bNyozoZs0aYK5uTlLlizh8OHD7Nq1iy1btnD69Gn8/PxSu9rplkz1SsMSzjadO3cuK1asYOLEiYwePRqVSsX58+e5ceMGxsbGlCtXjhUrVmBhYcHEiRNTuebpx4EDB3j27Bm9e/fGwMBA2f7o0SMWLlyIsbExBQsWBODBgwcSvCWhU6dOLFq0CDs7O3R1denSpQu9e/dm3759hIaGMmrUKLp164aVlRXnz5+nZcuWFCtWjNGjR5M5c2aaNm0qbZmEkSNHEhAQAHzuLQIICwvj0KFD2NraYmFhoewbFRXFu3fvMDU1TZTrTYK3xOLPt8qVK5M7d25CQ0OpV68exYoVIyAggCJFigAwYcIEBg4cSO7cubl16xY1a9ZUxmeJf8S3h5eXF0eOHKFPnz60atUKlUrFmDFjcHJyonnz5gwdOpRdu3aho6ODrq6uMnlE/DdyFqZxHh4edOjQgUGDBlG2bFnu3r1L586dmTp1qrIgcOXKlfHz80NPT09twoJI7MtFlOOTSbZq1YrmzZujr6+vvHbixAmyZs2qXMwTkoDjH8uXL2fFihW4uLjQtm1bihcvjouLCzNnzqRTp04sWbKEli1b4uTkRK5cubh+/TpVq1alZ8+eNG3alJiYGLS1tVP7Y6QpKpWKixcvEh0djaurK4sXL1bSVezduxczMzNcXFwwNTUFPo/Nypo1K0+ePEnFWqcvVatWZdGiRXTo0AErKysliCtatCgBAQH88ssvACxatIiePXvSt29f5VyVoFhd/Oxc+PxU4+DBg7i6utKpUydUKhXnzp3j3LlzGBoaUqxYMZYuXYqhoaGSPkT8N/Irn4bUqlVL7e+iRYvSqFEj+vTpw9GjR6lYsSKtWrVi165dVK5cmUmTJqFSqdi3bx+jR4+mbdu2coH5Ch0dHSXwMjY2xszMjE+fPuHl5cWTJ0/o06cPjo6OyoXI1NSU169fS5LOr4gPvIYMGcLBgwfx8PDAxsZGbZ8pU6awdu1abGxs6Ny5M/nz5+f9+/dcv35dmcEnj6TUxZ+nWlpanD9/ntq1azNhwgS0tLRYuXIlu3fvpkaNGuzevZs5c+awfft2zM3NGTt2bCrXPP04duwYK1euxMbGhvbt26sFcb/88gtjx46lRIkSAGp5NOVc/UfCG+KEvznxQVyfPn2wt7cnc+bMADRv3hxvb28yZcqkNttU/DcyBi6NaNmyJfPnz8fd3V1t1lOHDh3Ys2cPhQsXZsGCBYwbN47ly5ezZMkSGjVqxN69e+ncubNywZdHe+qMjIyoXbs2O3fuVLZNmzaN4sWLo6ury6JFi1iyZAm6urpMnjyZUqVKoa+vz4kTJ6hRowY3btyQtBbJSOpc8/HxUXrfpk+fzqtXr5TX3N3d6du3r5IIVSRWrVo1VCoVf/31F35+fjx8+JB58+ZRpUoVVq1axZ49e+jduzdxcXHUrFmT6tWrkzt3bh49esSECRMk/cJ30NXVVUsNNGTIEGxsbNiwYQMrV67k8ePHWFlZceHCBebPny+5HpOR8Ptfq1YtLCwsCAsLIzg4mOfPnwMwadIkatWqxaxZs1ixYgVmZmaULFmSAwcO8OnTJ5lt+j+SAC4NGTBgAJ6ennh6eqol4QQYN24curq6DBkyhI8fPzJ06FAqVapEcHAwQ4cOlaAtGV26dGHSpElKYBwQEECVKlVYtmwZhQoVolu3bkyePJlx48ahpaVFrVq1qFevHtra2jx9+pTAwEBAAuOviX8MGp+uYty4cTRq1IiZM2eybt06tZQhbdq0Yf369RJgJCFHjhzMmjULgPDwcGxsbKhXrx5Xr14FUAviXF1d1YKQePKD+HVOTk7o6uqyatUqtV61oUOH0rFjRxYuXKgEcebm5rx48ULO1W8YMWIEbdu2JSwsjFy5crFr1y5Wr16tLEU4ceJEatasydKlS5k/f76SB06uqf87SeSbhkyaNAktLS0mT54MoBbE5cmTB319fT5+/IhKpaJw4cJs2rSJpUuXAvJlSM6GDRuwsLAgMDCQuLg4QkNDcXFx4fLlywBcuHBBae9x48Zx8OBBDh48qPYe0rbqErZHmTJl8PHx4ebNm0RERHDs2DGGDBmCjo6OsppCwiAufuak9BIl9uTJE/z9/ZkzZw41atTAw8NDCd5UKhXHjx+nffv2rFy5ksDAQAYPHpxoBRYJ3r6uVq1alC1blsjISLZs2aIEcWPHjqV48eJ06tQJExMTZs6cqQydkHNVXb58+bh//z4Affr0wcHBAScnJ06dOoWnpyf9+vXD1NQUbW1t/vrrLzw9PZk/fz6//vorM2fOVN5Hrqn/OwngUtmXwUFAQAAqlSpRELd//346d+7Mli1b0NXVJUuWLEouMpAvQ3IiIyOZOHEi2traTJ06lXfv3qkFaPGPqydOnEhsbCwTJkxI9B7Sturi22Pw4MGYm5sDn9OI6Onpoaury+HDh/Hy8mLChAk4OztjZGTEwoULeffunfIe8oOYtLdv3/Lo0SMeP35MixYtePjwIUeOHFGWcjp+/DgdOnRg27Zt3L17V2ac/0vdunVj6tSpuLm5oaWlxebNm5Ug+N69exQqVIhs2bKpjXuVc/Ufjo6OdOjQAUdHR2JiYihdujTjxo3j1KlTNG3aFBcXF1auXEm9evVwdXUlLi6Ov//+mx49eiSaQCb+d/IINRUlDN4cHBzQ0dFhzZo1fPr0CU9PT7y8vBgwYADLly8na9as2NraUqFCBSIjIxk6dKiMd/mKVq1aUaRIEbJmzcqIESPQ09OjS5cu+Pj44O3tzbx589T279ChA1OnTsXJyYk//vgjlWqdfjg7OzNkyBDatWvH48ePKVGiBMOGDePu3bvMmzePo0ePAjBnzhx0dXXp3r17Ktc4bfryBi7+71q1auHi4oKuri6BgYFKe8YrUaIE169flx63ZCRs1yxZsqClpcXr16+Va+WMGTOoVKkSs2bNYteuXTx58oQ5c+awfPnyRG0tPosfjuLo6MiOHTvQ1tamYsWK3Lhxgzx58rBkyRJmz57NvHnz+P333/Hy8uLcuXP4+fkpiY/laUbKkgAuDfD19cXOzo6ZM2eyfft2QkNDAZQgztPTM8lB3zLeJWne3t40bNiQbdu2cf78eWVsVqZMmejduzeDBw9ONFkEPqcVOHbsWGpUOU3r0KFDoraaN28eMTEx/P7778o2a2trpk+fzrVr15g2bZryQyg3Gd9ma2tL5syZefv2LRs3bgSgQYMGdOvWDZVKxaxZszh8+DDLly9n165dyioMcg1ILGGbDBgwgBo1alCiRAlWrVrFoUOHlMz/kydPplKlSujo6PD+/XsMDQ2pXr06nz59kkDjC506dWLixIl07dpVbUJYpkyZePfuHf369aNq1ao4OjoSFRWFs7MzTZs25dq1azJGW4PkEWoqa9++PW3atKFz586cOXNG7bX4xyP+/v4YGhom6jWSC3diHh4edOrUiXbt2nHhwgW1wOHdu3fMmjULlUrF1KlTAdQCk/jgTS7e/4hf+mbVqlVqM52joqIwMTFR/o6Li+PgwYPMnj2bwYMH8/r1a6Kjozl58qT8IH7DqFGjaN++Pc+fP8fIyAg7Ozs6d+7Mn3/+CXzu+Zg2bRqvXr0ic+bMODk5KcfKNeAfZcuW5cKFC0qbDB06FEdHR4YPH86nT59wdnamYsWKZMqUie3bt9O/f3/s7e3JkyePMgs9Pp+Z3HD8w9ramilTptCnTx+14G3hwoVs376djRs3YmRkhLGxMXny5OHmzZvUrFmTtWvXKtdX+f5rhgRwqaxChQrs2bNHLXhLeLJPnDiRLFmyYGNjkyiAE+ry5ctH8+bN8fHxUbrsv/T+/Xtlpl9gYCBGRkbMnz9fbR+50Pxj9erVLFiwgLi4OKpUqcLx48eJi4vj5MmTTJ48mdq1a3Po0CFl/4iICI4fP06hQoWwtbXl5MmTgLRpcszMzChatCg2Nja8ePGCcuXKMXnyZDZs2IC9vT1//vkn4eHh/PLLL+TKlYvAwEBZQD0JW7du5fLly1y6dIlPnz5Rp04dWrRoQceOHTl9+jRVqlShTJkyXL16lT59+hAdHc2ePXvYsGGD2vtI8JbY48ePefHiBc2aNWP79u1ERkYyf/58ypYty4gRIwA4ffo0dnZ2zJ8/HwMDA2JjY5X1Y0G+/5oiAVwqMzMzS5QOIC4uDl1dXapXr87hw4fx9vZOpdqlL5aWluTPnz9RT+aXPnz4wMSJE8mcOTPly5f/QbVLn+IHeMcnkt28ebOSi7BixYosXboUFxcXzp8/z5s3b2jUqBHr1q1DW1ub6dOnM2fOHB48eJDKnyJtcnZ2xsHBgQcPHvDw4UPevXvHvn37cHV1ZebMmaxfv57WrVtz9uxZzp49qxynpaUlwVsCzs7OFChQAHt7ez59+oSuri4hISGsXbuW06dPU79+fWbPns2AAQMIDg5m1apV9O/fH2NjY+VxdTwJ3hK7fv06LVq0YOPGjcybN4+4uDjy5ctHixYtePToEQB79uwhNjaWwoULY2BgwIwZM2SM9g8gAVwqu3v3Lh07diRXrlzKlwEga9asdOjQgY8fP/LXX3+lYg3TDyMjo29m9S5dujSdOnVi2LBhjB49mujo6B9Uu/TtypUr7N27l1q1aimzdfv160dUVBRz5sxRZu19/PiRTZs2Ua5cOe7cuaPkfBLqdHR0iIqKwszMDGNjY2WGbmxsLEeOHOH3339n2rRp7Nu3j3r16qkdKz+I6kxMTLh//z4fP37Ez8+PO3fusGzZMoKCgjAwMKBXr17MmTNHeZwXHBxMzpw5+fXXXxMFcCIxlUrF9evXsbe3Z8mSJRQsWJA6deoov1fxQdq+ffvYt2+fcpwEb5ona1ikMn9/f0JCQlizZg0lSpQgR44c5MyZk+nTp5M7d24ZVP8vvHz5EmNjY6pVq5bsPr/99hsxMTHExMRI8JaML6f76+np8fz5cwICAjhx4gQNGzbE09MT+JxKpFOnTowaNYqxY8dSrVo1YmJisLW1JSIigqioqNT4CGnOl20aExPDpk2bGD9+PJaWlsyYMUN5LT6I8/Ly4vHjx5J+4Ru2b99O+fLl2bt3L7179+b48ePExMQQERGBnp4eefPmVVYEyZw5M48fP2b8+PEMHz48dSuexsXfDMc//gwODqZLly48fvyYoUOHYmZmBiR/QyHBm+bJLFQNSzieLUeOHEkuNp0nTx6mT59O8eLFiY6O5vnz58TGxtK4cWNiYmJkAOi/MHv2bJo3b067du0S9VxaWFiwYMECdu3apZZQUvwj4bnWvXt3SpUqRaFChVi9ejVbtmxBpVLh5eVF1apV2bt3b6K8eUWKFMHV1ZWmTZvSsmVLJRHtzyxhm1asWJHs2bMTFhbG7du3efPmDW3btsXHx4c///wTd3f3JI+Ta0DS4ttl5cqVNGjQgD/++IPu3bsrj5izZ8/O1KlT+fjxI8eOHaNu3boYGxvTrFkzZR1eadfE4nvPSpcuzciRI2nTpg0xMTEAFCtWjPXr13Pp0iV+//13Xr58mcq1/XlJD5yGxV8cvL298fX1VRb1Tejhw4fY2tri5uaGj48PAQEBNGzYUFmYXi4w32/+/PlcunSJ1atX06ZNG6ysrDA1NcXa2pqNGzfy8uVLCd6+Iv5c8/HxwcPDg/DwcPbt28fUqVMZPnw4b9++JTAwkGPHjlGnTh1GjRqlHGtkZESuXLnInDmzBG8JxLfpiBEjmDdvHoMGDWLWrFnMnTuX8uXLs2HDBnx9falXr56ydFvC4778b/GPuLg4JfGul5cX9evXZ9KkSZiamgLw7Nkzli1bhq6uLh06dODTp0+0aNFCgreviA/eihUrxurVq3nw4IESvMHnnjh7e3tKlCjBmjVrlNno4seTHrgfoEaNGowdOxY3NzfOnz+f6PXkLiQyhuC/qVSpEr169aJly5aEh4ejo6NDaGgoFy9exNXVFZAeja+pUqUKM2fOpEePHpw7d47SpUuzf/9+fv/9d9atWweAqakpvr6+xMXF4eHhoRyrra2Nrq6uPDr9gpOTE15eXnTr1o0TJ04wbNgwnJ2dcXJy4sCBAxgaGtK8eXNmzZrF2LFjmTJlSmpXOc1K6rsbf620trZmxYoVrF+/nlGjRhEeHg58fnQaFxenTMqRWbxJi2/H4sWLs3nzZtasWcOIESPQ0tJi5syZuLm5KZPuSpUqxeDBg+ncubNcS1OJBHAa1qZNGypUqAB8Hi8kQdn/7nuDrxo1apArVy5iY2O5efMmFy5c+FfH/6zq1KmDu7s7tra22NraMnXqVHx9fVm0aBEmJiYUKVKEs2fPYmJiovwgSpsmLb5dpk2bxpMnTxgzZgxNmzZl5syZ+Pr6smTJEgwNDdHR0SE6Oppq1apx6NAhuUYkI+F55uTkRKFChcibNy9r1qzh7NmzhIWFUbNmTVatWqUEcS9evEj2PcQ/vgze1q5di7e3NyqVit27d6OlpUWLFi2IjIxMdKy0aeqQR6gaZm9vr4wl0tPTkwvz/6BEiRIYGRl994Xi6NGjrFmzhvXr1yvBG8jjqG8xMDDA0tKSNm3aMHnyZCV4A6hevTru7u7kypVLgrfvYGBgAIChoSHnzp2jSpUqzJ49WwnetLW1cXBwoE6dOnz48IEDBw4oyWRFYgkf8Q8aNIi3b9/y7t07fH198fT0xMjIiCNHjtC2bVvs7OyYPHlyokd8cq4mplKplOBt48aNasHb3r17efnyJba2tkkGbyBtmlrkKpGCkpot1rZtW1auXEmhQoVo3749RkZGqVCz9K9Pnz7s27ePnTt3Uq9ePQoXLqz2uszU+9906NCBzZs3A3DgwAHu3LnDzJkzmTlzphK86evr06lTJ6KiotRS3sjF+x81a9ZU/nvAgAG0a9cOgJCQEObMmcO6devo378/S5YsAT6nwLCzsyN//vxq7yM3esmrXbs2NjY2tGvXDn9/f9asWUPevHn5+++/iYyMREtLi7/++otu3bphamrK27dvU7vKaV5cXByFChVi+/btbNiwQS14Cw8Pp3v37tKOaZDkgUshCXshSpYsSVxcHAYGBpw9exZ3d3eCgoJwdnbm/fv3bNu2jffv36dyjdOP+OBszZo13L17FxcXF4yMjNi/fz+rV68mJCREgoj/UUREBDo6OjRp0oSdO3eyZs0aTE1NqV27NhcvXsTMzAx7e3ssLS2xtrYGpOftSzly5GDixIm8ePGCc+fO4ejoSIMGDYDP624WLFiQypUrc+jQIUxMTMiUKRNTp07FyMhIJtYko3v37pw+fVqtBz1z5sw8efKECxcuYGtrS2BgIEOGDFGWdCpdujQXLlxQy0sm56q6pNojfi3jwMBAVCqVsgpIt27dJHhLo2QMXAobOnQoTZo0QU9PD0NDQ/744w+GDBkCfJ4hWaxYMaZOncoff/yRbHe0SKxSpUosXboUGxsbnj9/To0aNXB1dSUqKopbt24xdepUwsPDpU3/JVNTU169ekWWLFmYNm0a2tradOrUCQA7OzuaN29OvXr1uHLlCo8ePeL3338nJiZGxnImQUtLiwoVKrB+/Xq0tLRo3rw5Fy9eREdHh5iYGCpVqoSvry8lSpTgyZMnvH37ltjYWJo1ayZtmoSqVasyZ84cDh48yJw5c7h27Rrwef1oBwcHpkyZwrJlyxg1ahQLFy4EwMbGhqpVqzJlyhQlubRQlzB4a9SoEaGhoVy6dEnt9QMHDvDs2TO6du0qwVsaJgFcCnJzc8PV1ZWOHTty5coVvLy8cHV1pWHDhsranEFBQdSqVQsXFxf279+fyjVO+xJebHx8fLCwsGDYsGG8evWK8uXLs3v3bp4+fcr79+85c+YMu3btUh4Fiq/r378/bdu2ZcCAARw9ehQrKyuOHDnC7NmzmThxorKflZUVz549U2afyQw+dQnP0aJFi7J06VJ0dHQICQmhdevWakvlqVQqWrVqhZ6eHi9evODPP//k06dP0qbJaN26NS4uLly+fJmgoCAuX75M5syZ+euvv8iZMycuLi6sX78e+PyIf/Hixbx48YI+ffqkcs3TvhEjRtC0aVOWLFnCihUriIiIIC4ujjZt2lCjRg2GDRumjHMVaZMEcClES0uLefPmsWfPHtauXUuzZs2YNm0ao0aNYsmSJRgZGSm9Q0OGDMHf31/utr+icuXK3Lhxg5cvXyo9E40bN8bLy4uGDRuSNWtWDh8+zK5du+jfvz8dO3akSZMmRERE4OLiktrVTxfmzp1Lq1atePToEStXruTIkSNYWFjQt29fRo0axeHDhwF5/PQ1VatWBeDYsWNMmTKFqKgoxo0bR4kSJQgICOD169fY2tqq5dH6MliTnrfEdHV1lcC3a9eutG/fnuDgYGbNmkVwcDCNGjVi8uTJnDhxgnnz5mFmZoaTkxM5c+akTp06Egx/w4ABA+jVqxcdOnTgwoULidbjlhuK9EECuBSSKVMmjh07hpeXF2/fvmXlypX4+PiwePFidHR08PLy4tixYxw8eFA5Ri7cSatZsyaBgYGsW7eO2bNn8/r1a+W1devWoauryy+//MKBAwcYOHCgso5kpkyZlP8W35YjRw4GDx6Mnp4eL1++pGDBgujp6fH69Wvu3LmDv7+/WuAh1JmYmLBv3z7u3r3L69evqV+/Pi1atODy5ctoa2tTs2ZNRo0axatXr2jVqhUxMTFMnjyZU6dOsWrVqtSufrrQr18/cubMSdOmTcmZMycbNmxg8uTJ3Lx5k7p16+Ln54eJiQnPnj3j/v37ODs7y+PoJCS8CTM3N2fJkiXMnj2b7du3Y2VlRcGCBXFwcODKlSssWrQoUUAn0iYJ4P6D5HokfH19KVq0KNWrV2fo0KEsX74c+LyE0/Tp09m+fTvLli370dVNl0aNGkXVqlXZs2cPQUFBylqGderUYcGCBWzbtg0vLy9Zz/Rf6t+/P9HR0ezatYtbt27Rp08fcuTIwdKlS8mcOTP+/v6UK1cO+DzbT1ZT+Lps2bJx+PBhsmXLRv/+/ZUF0wG1IC5r1qzcvXuXvHnz8uuvv0rvxnfo06cPnp6edO3alfDwcGrUqIGTkxOnTp0iMDCQW7duoa2tTb58+Xj16pWS7016j5JnZWXFixcv+OOPPzh79iwbNmzA2dmZPHny8OrVK6ytrRk9ejRTp05N7aqK7yCzUP+lhMGbpaUlWlpaSkqFEydO4ODgwN9//62MbzM3N2fq1KkYGxuzYsWKVKt3ehF/8R0xYgSDBg2icePGxMXFERQUxOvXr7l69SqPHz/m5cuXErz9Bx8/fsTR0ZEqVaqwdetWFi1axJ49e3j8+DGzZ8+mUaNGeHh4ULx4cYKDg1O7ummajo4OWbNm5enTp7x7946mTZty//59ZQ3e2NhYDh8+TPfu3enQoQOxsbHY2dkRGxsrPUTfoK2tTY0aNVixYoXy1OLSpUu8fv0aHx8ftLS0mDFjBlevXuXOnTvKcSqVSoK3BOrVq0eFChUICAhg3LhxmJmZ4eXlxZo1a+jYsSMdOnRg3rx5LF68mMOHDxMQEEChQoVSu9riO0kP3H80bNgw7OzsyJQpE6GhoUybNo0tW7bQoUMH+vXrR1RUFK9fv0ZPTw8dHR0aNWokXfvfSV9fnw8fPgBw69YtXrx4wbp161iwYAEvXrygVatWjBs3jvbt23P27NlUrm36U65cOZo1a4aTkxPr1q3j/v37uLm50b17d44fP662r/RmqEuu993Kyoq1a9cSEhLCtGnT+Pvvv5N9D7kGfJ9Fixbx8uVL+vfvr9Zm48ePp3Xr1hw9epRRo0apBXDiH4aGhvTv3x9bW1tCQ0MpW7YsjRo14vr16xgZGWFkZISpqSm3bt1Sjtm6dSvHjx9n7NixqVhz8b0kke93Spgotm3btnTp0oXx48fTq1cv7t+/j5eXF71792blypX07duXBQsWcOLECRYuXEiDBg2Uhenlwp1Yv379cHd3Bz7/uH348AE9PT12797NmTNn2Lt3L40bN6ZHjx6Ymppy9OhR4uLiEiXzFf+oXbu2kq/tS+fPn2fSpEm0bNmSGjVq0KJFC4yMjGjbti2ZM2dW21eCt38kDN6KFClCpUqVyJQpEwYGBoSGhtK1a1dy5cpFnz59qF27NgDbtm3D09NT7X3kGvB9Ll++TMuWLSlSpIhamz179oy7d+/y5MkT7t69m4o1TNvev3/PtGnTeP78OdWqVWP16tVcv34dgA8fPvD8+XNu3bqFoaEhFStWZM2aNWTJkgV/f/9Urrn4XtID9y81bdoUc3NzAJYuXapsHz16NI0aNcLFxYXTp08nOk7uupM3YMAABg8ezPDhw5k7d66SAfzFixe0adOGuLg4fHx8qFmzJjt37mTy5MnUqlWLQ4cOpXbV0xyVSoWxsTHHjh1j/fr1+Pr6fnV/Q0NDunbtiqOjI48ePaJVq1Y/pqLp2NChQ2nZsiVmZmaEhISwatUqNm3axLNnzyhSpAhz5sxBW1sbfX19YmNjqVOnjgwK/4/Wrl1LkSJFcHJy4uHDh0RERLBgwQL++OMP1qxZA8gs6a/JmjUrnp6eGBgYUKVKFTZt2qSkCIrPT9isWTNsbGwwNzenXbt28qQoHZEA7l/IlSsXx48fx8DAgAkTJhAQEKD2iOnPP//k3r179OzZM5Vrmj4kvPD26tULPz8/RowYgZ2dHa9evUq0fIuPjw9t2rShf//+7N69O9F7iH/06dMHV1dXWrZsyY0bN5LcJ/4irVKpyJUrF48ePZK2/IYBAwbQrVs33Nzc2LdvH8uXL6dEiRLKI/6nT5+SN29eateujaGhIQsWLCA2NlYeRf9HFhYWTJ48mWrVqvHkyRNUKhUqlYpq1aoRGxsr3/8vJNceFhYWdO/enZYtW7Ju3TomTZqkvPbbb78RGxvL2bNniYuLk3M1HZEA7l/Q1tamWrVq+Pv78+LFC+zt7fnw4YPypRk3bhw5cuSgW7duqV3VNM/b2xszMzMGDhyo9E64uLgwatQoHj58SO3atZUkkgkvKLa2tpKo9zuUKFGC2bNns3z5coKCgr77jlp+EJNXtGhRJk+ezLRp09i9ezfW1tYsXryYM2fOUKhQIVatWsXChQsTrQAgvRmJJTzP8uXLR2ho6Fd7KW1sbDA1NUVfX59FixbJRJAvZMmSRS3dUs+ePSlcuDAqlYqAgACePXuGlZUVXbp0wcbGhu3btxMQEMDq1au5ceMGQ4cOBeT7n95IAJeMhCeylpaW2uymGjVqMH/+fM6cOYOrqyvv37/n48eP7Ny5k2vXrinjuUTSihYtytGjR4HPj6EHDhyotK2TkxMBAQEMGTKE+fPnK8d8eVcoF5pvmz59Or/99huVK1dO7apkCJkzZ8ba2pq9e/dSpkwZFi5cyPjx41m6dCnr16+nUKFC7Nq1i/Hjx6v9mAp1Cb+7Xl5elChRgqVLl3Lw4MFE3+nkvucSvP1j2LBh9OrVi0qVKvHkyROGDx9Oly5dOH78OIULF8bc3Jw2bdpw/vx5rKysaNu2Lb169SIyMpK3b99St25dyfeYTkkakWTEXzTc3NwoV64cuXLlYtmyZZw4cYKjR4/So0cPZeWFe/fu8fLlSzJlysSAAQNSueZp3/Xr11m1ahUmJia0aNECU1NTevbsyadPn1i8eDGGhoaMHTtWWd0CEg+ml+DtH/ny5eP+/fvK3/FZ7AMDA1mzZg3dunVT1ooU/92bN2/Yv38/kZGRtGvXjj/++EPJ9fjgwQOsrKzQ1taW4O0b4r+7I0aMoGPHjvTr14+LFy+qfafjA7TkvucSvP1jzZo1VKlShe3bt2NnZ4eJiYkSsGXLlo1Jkyaxbt062rZty9mzZ1mwYAE7duygWLFibNu2TZZyS8dkFuoXEs429fT0xM3NjQcPHnD//n3c3d0ZOnQov/76K0ePHsXZ2ZkPHz5QrFgxJk6cSPXq1ZXZpuLrQkJCyJYtG+3bt6dKlSrMnTsXLa3Pp+Ps2bMZMWIEfn5+eHh4pHJN07aSJUty+vRpli1bRvfu3QGUR1FPnz7l5s2b1KlTJzWrmKHEj8nMmjUrhoaG6Oh8vgc2MTFh+PDhDBw4MDWrl27UqlULOzs7HBwc2LlzJ2/evCFnzpzUrVuXLFmy8OnTJ+V6IL7u1q1b9O3bl+fPn7Nnzx5+/fVX5TwNDw+nb9++HD16lNWrV1O+fHnevHnD9evX2bJli9LOErylT/IN+UL8HZ+VlRWWlpZ069YNX19fnJ2dGTFiBMbGxvTs2RNzc3OOHTvG4MGD0dHRYfjw4cp7yJfh2yZNmkTmzJkpV64cPXr0oG7dusyePVu5aM+ZM4eAgABJFfIVNjY2VK9enU6dOqGjo4Obmxt//fUXPXr0oFChQkRERDBp0iRq165N8+bNU7u6Gcrdu3cpU6YM8+bNY9euXZQsWVJJOJvwJlAkLS4ujnfv3vHmzRuKFi3K4MGD2bFjB5MnT2b//v2YmZlJL9s3JDzP7t27x++//86ZM2coVaoU+vr6yj4RERG4ublx+PBh9uzZQ5EiRdTeR9o5/ZIALgk2NjZcuHCBxo0bq40N2LlzJ0uXLqVevXrkyZOH2NhY/vrrL3r27En58uXZtGlTKtY67fL19WXx4sW0atWKrFmzAp+D3M2bN1O8eHGOHTuGk5MTDRo0YNasWUoQN3HiRPr06ZOaVU+TVCoVWbNmZdy4cTx69Ijdu3fj7OxMq1atuHTpEp06dWLPnj0MGDAAMzMzNm/eTM2aNaVn+Bv+TeDl6+vLn3/+yZs3bwgODqZmzZpKb4Y83leXsF2trKzQ1dUlIiKCjx8/MmPGDP744w+yZcvGxIkTlQlgVapUSa3qphvx59lvv/0GfL6p8PHx4dSpUyxbtowcOXIo+0RERDBgwACmTJnC7du3U63OImXJJIYk6OnpMWHCBDp27IiHh4cyziXeyZMnWbFihdp6cdbW1vj7+2NnZ0doaOiPrnKalXDCwu7duylevDgBAQGcPHmSt2/fcuLECbp27crBgwepXr06ixcv5ty5c7Rp0yaVa5626erqcubMGdzc3JSen3jFihWjQYMGdOrUiffv31OyZEmioqL47bffePz4cepUOI1LOAboW+OBkhtAL+OIEvtywkKRIkWYN28ep0+fpm7duhQqVIgHDx7w999/ExERgampKZs2bWLkyJGJzmvxWcI2LV68OIcPH2bYsGHKeOH8+fMza9YscuTIQdOmTZX0KwlvLORczRh++h64pO66o6OjGTRoEJs3b8bX15datWop+5mamhIXF0d4eLjaMQcPHsTa2lqCty9cv34dNzc3YmJiuHbtGkFBQdja2rJkyRK6dOnC0aNHadWqFYaGhkpv5oMHD1K72mle/KzoyMhItW0AwcHBTJ8+nU6dOjFx4kQuXrzInTt3ePLkSWpVN01r0KABv/76KwBjx45l9uzZX90/Pnj78tohP4iJJZyw0K1bN7Zv3658v/fv309QUBC7d+8mKioKc3NzZs2aRVRUFIcPH07Naqdp8W3at29fGjduzMePH/H19cXV1RX453FqWFgYW7duxcrKKlGvsJyrGcNP3QOX8K6kYsWK6Onp8e7dOy5cuAB8vktZsGAB1tbWrF69mnv37lGzZk3y5s2LtbW1fAn+hW7dujF+/Hj69evHgQMHyJ8/P4MGDaJUqVJcvnyZ1q1by1T2b4hfLP3KlSvkzp2bgwcPYmtry+XLl9X2Syr1Qvw2Sb+Q2P79+8mePTvHjx+nTp06NG/enODg4O8+vmDBgjx58oR3795psJbpl7W1NdOmTaNjx45cunQJLS0tzM3NyZcvH/fu3ePZs2e4urpSq1YtTE1Nadq0qawG8A1eXl706NEDNzc3MmXKROnSpenTpw9jxoxRngzly5ePtWvXcuXKFclNmkH91GlE4n/khg0bhoODA+/fv6dAgQJMnDiR5cuXExYWRvfu3ZkxYwbdu3dnw4YN7N+/n8WLF0t29X9p4cKFaGtrExgYiJ+fH9OnT6dNmzYUK1aMBw8eSPD2Dfr6+jg6OlKjRg3q1q1LSEiIki7kS0mNwYqLi0OlUskPYhLq1q3LlStXaN68Of379/9XwVvPnj1p3749nTp1kgAuGdra2jx58oTnz59TtGhR7O3tlRu2ly9f0qZNGy5evEhMTAzz5s2TtBZfyJs3r9pTiUyZMlG3bl0loTTAxo0bCQsLw8/Pj+joaObMmcP9+/extbWVnvcM7KcO4AA8PDzo0KED3bt35/jx43h7ezNo0CCyZs3KtGnTePLkCW5ubqhUKmrWrMnChQuVLOBygfl3goKC+PTpE+PHj0dLS4upU6dy8eJFQBLzfsuHDx/w8PBg7NixbN26ld69e3Pt2jWaNGlC9uzZMTY2VgaHq1QqihUrxpYtW9Qe6Uv7/iP+fItfO/bJkye8ePECDw8P7t27x/Hjx5XX49vty3PU0dGRwYMH4+npKUMnviImJoacOXMSGBhI+fLl2bVrFxMnTuTZs2eMHTuWUqVKcfjwYeWxqVxb/7F06VLevHmjPB6Fz2O0c+fOrXYzplKpWLBgATVr1mTkyJHExsYyb948Zcyr9GZmTD/dI9Qvl3AZPXo0q1ev5o8//qBZs2ZMnTqVLVu20LlzZ4KCgpg5cyahoaFoa2szf/58fvvtN5ydnfnrr79S+ZOkHf82+OrevTtjxoxh9OjRzJgxQ4M1y3hy5szJxIkTadSoEQB37twhU6ZM6Orqoq2tzfv371GpVDx58oQGDRrIRTsJCc9XGxsbrl27xq1bt4DPM82zZ89O3759OXHihNJ+X/Z2Ojo64uvrS9++fdm+ffuP/xBpUMJ2NTY2Jjo6mujoaACaN29O4cKFuXXrFkePHuXVq1dkyZKFLVu24OvrKxMWkpElSxYiIyP5+PEjZmZmvHjxAgB/f3+qV6+Ok5OTcu4C+Pn5Ubp0aapXr07Xrl3l3MzgfroALl782Ct7e3t27txJiRIlWLBgATNmzCAoKAg/Pz969uzJ2rVr8fX15cWLF+jo6LB69Wry5s1LrVq1iIqKSu2Pkep0dHRYs2YNwcHB3L9/n/nz5ys/el+76+vWrRv+/v60b9+evXv3/sgqpxsFCxbE0tKS7Nmz8/z5c2U2r7m5OcOHD6d9+/bY2tpy/PhxMmfOrPQMq1QqZTUA6dlMno+PD82bN2fNmjUsXbqUp0+fAp+DuPh1es+ePcvs2bN58uSJsspKfPDm5ubGtm3bUvMjpBkJzzMXFxcaNGiAlpYWISEhSu+Rjo4OMTEx6OjokClTJubMmUOWLFlo3ry53GgkIeH109nZmS5duuDs7MzVq1epUqUKXl5evHv3jhEjRnDv3j309fWZP38+S5cupVGjRpQrV45WrVrx5s2bVP4kQlN+mgAu4QVm1KhR9O7dm0KFChETE8P79+/x8fEhT548uLq6EhUVxcCBA6lYsSJGRkbY2Ngox2pra5MjRw55ZJKAjY0NmTNnZujQoVy5coUjR44wd+5coqOjvxrE1apVS2abJaNdu3b07dsXHR0dzM3NyZw5M4cPH1aWwTE3N2fq1KlUqFCBNm3acOnSJYCvPvIT/3B2dsbT05O2bdty+fJlPn78qDbuasuWLRQsWJB3797x8eNH6tSpQ0xMDC1atGDWrFn07t1bejeS4O3tTbt27ZgxYwavX79m2LBhXLt2jc6dO/P+/XsMDQ35/fffqVatGpkzZ6ZJkyYyYeE7ZM+enUOHDnH79m369evH7du3sbGxwcnJibJly3L69Gny5MnDp0+fqFmzJv3796dx48Y0bNgwtasuNOinSSMS/0NWuHBhjIyMaNmyJREREcojp8KFC6OlpUVMTAwqlYoyZcowdepUmjdvroyFgc/TryV4U7dt2zZWrFhBjRo1CA4OpkmTJsqapl9bEic+eJPM9eratGnDxIkTmTFjBg4ODtSqVQtHR0eKFy+Oj48PzZo14/nz53h4eHDq1CnWrVtH6dKlAfVxbhK8JU1PT49KlSoxZ84czp07p0ygSRhAtGzZkjFjxjBlyhSsra2VfS5fvkzHjh0leONzUJFQw4YNadSoEY6OjsyePZvw8HCMjY359ddf2bp1K4aGhrx//56LFy/y999/K4nStbW1JXhLIKnr4bNnz6hduzYFChRg5syZFCxYkG3btjFgwADGjh3LgwcP2Lx5M3Xr1gUgd+7chISEYGBg8KOrL36gn6YHDsDW1hYfHx/evHlDmzZtePr0qfIj17p1a2bPns2RI0ewsLAAoHbt2jKY9jvF30Hr6+vTqFEj+vbty4cPH2jdujVRUVHSG/SdcufOzZIlS1i8eDHLli1Te+2XX35hw4YNhIaG0q5dO16+fEnOnDmZN28e79+/p23btqlU67Tty3NPR0eH3bt38/fff+Pt7a22r76+PoULF+bKlStq2+ODDDmHP5syZQpaWlpMnjyZ+/fvA5/T3BQrVozJkydTv359Zs2axfjx47l8+TIbNmxQVlxJmLtQet7UJTxXmzdvTsGCBfn48SNnz57lxIkTZM+enX379vHo0SPc3Ny4efOm2vHm5ua4u7vToUMHmjZtyvXr11PjY4gf5KfpgQN4//499+7dI3/+/JiYmBAXF6csL7R+/Xq6d+/OnTt32L17txK8yYLKSatZsyYtW7bE3t4ePT095SL84cMHtm/fztixY9HV1WX06NFoa2vLD993Mjc3J0eOHEouwnhaWlrcuHEDJycnKlSooDwaCQsLw9HRkXbt2qVGddOF+HMvfhk3PT09Hj58SJEiRciaNataj0eePHlwd3fnl19+UXuP2NhYOYcTuHz5MnXq1KFbt24UKFAAgB07drBmzRqMjIzw8PBg3rx5LFy4kDt37nDv3j3q1KlDQECA2vtI8KYu/hzz8fHBz8+PqlWrUq5cObZv306LFi149uwZ1tbWWFpaMnnyZMqUKaMcmy1bNpycnChfvjwtW7aU4O0n8FNEJ/b29tjY2LB7925mzJjBjRs3mDt3LgUKFFAL0rZu3Yqnpyd+fn5Knje5wCQ2fPhwpkyZQv/+/Zk9ezZz585FR+dzRpr4O+ojR46wYcMGfvnlFypWrAjIo9LvkTNnTgwMDIiIiABQzs343Fhnzpzh7NmzFCpUSDnm5cuXao/5RWItW7bk6NGjFCtWjMjISKZPn07NmjUZPnw4VlZWaGtrkzVrVkaNGkXWrFkT9WwIdQsWLGDs2LG0atWKbt26UbBgQQAePXpEzpw5sbS0ZP/+/cDnoOTKlSs0aNCAvn37pma10wUbGxscHBzo0aMH7du3Z8+ePQAYGRkB8OLFC+rVq0elSpXo0qWLclx4eDjLly+nS5cuiZJ7i4wpwwdwBgYGdO7cmQ4dOgCwb98+Jk2axKtXr5g2bRr58+dXfhy/JI9PE+vbty8dOnSgZ8+eNG/enMqVK1OvXj06duwI/HNHHRMTw/Lly9HW1qZ9+/aAjMn6Hjdv3sTY2BhbW1vgc3smHH8JnwPhV69eJTpW2jd54eHhXL58mXnz5lG8eHHOnDlDx44dadWqFUuXLuXQoUOsXr0aKysr2rdvLwFxMhK2yerVq5kwYQK2trZ07dqV/PnzAyi5xzw9PalVqxZBQUHkzJmTCxcufHVMrPisQIEC7N+/nzNnztC8eXMmT55M//79Wb16NSYmJhQpUoTw8HB++eUXBg4cqHZsWFiYkmpEZHwZ7puU8AKjo6NDVFQU/fr1o3r16srd365duwgKCiIqKoqpU6dSuHBhCda+Q9GiRWnQoAFDhgzh3LlzREZGcvfuXXbv3k2RIkXU9tXS0iIyMpIhQ4ZQoUKFRI+kRNIePXrEpk2b6NWrF3Z2doB6YJY1a1b09PSoXLkynp6elCtXDn19/dSqbpqUVOB19OhRJk2aREhICAsWLKBYsWIcPHiQ+vXrs2jRIrZu3crSpUupV6+eMrBeAmJ1CcdnValSBYBly5Yxbtw4bG1t6d69OwULFuT9+/d4eXlRuHBh/P390dLSonXr1rIayDfEB7YJcxTOnDkTHx8fZTxsgwYN6NixI6amprx580YC4p9chvt/Pv7kd3Z2pmfPnhQoUIB79+4xevRoWrZsSbVq1YB/gjgTExN69eqVmlVON8LDw4mMjOT27dvAPz1Cz58/Vx6hJHzkF//agwcP0NPTS4Uap21JBRpRUVEsW7aMR48e4evrqzwiMTQ0xMLCghkzZpA9e3Zy5MhBtmzZyJ49Ox8+fPjRVU/T4q8B9vb2WFlZKdtPnjzJ1KlTuXfvHgsWLKBo0aLcvn2bFStWMGHCBFasWCGrrHxFfLsOHTqUadOm0b17dwBWrlzJuHHjsLOzo1u3blhZWbFv3z5q1apFx44dsbW1laD4G1q3bk2nTp2Az4vRV6lShZkzZzJ69GgWL14MfE6O3KZNG7S0tNR64CUg/nllyFmoOXLkYN++fRgZGXH37l38/PwICQlhxIgRXL58mWnTpilJeH/77TdOnTolF5bvZGxszNu3b4F/xrsNHTqUQoUKKRd0IyMj8ufPz9WrV4HPi1kHBwcTFhaWavVOS7y9vVm/fj3Xrl1LdnautbU1ffr0wdramhs3bqCjo8Pz58/R1dWV3E5JWLRoEffu3WPkyJHA5xm78+bN482bNzg7O6ude9bW1kyfPp2nT5/i6urKtWvXUqva6Y6npyc9e/akc+fOPHjwQK1d27dvz9ChQ9m0aRPLli1TG0cos9CTp62tzZIlS8iSJQs2NjYATJgwgY4dOzJo0CDOnTuHlpYW3t7emJub06BBA7nBEEAG7IEDeP36NUFBQRw7doxt27Yxb948GjVqxIcPH3B0dFTGasDnu3IZ7/L94oO3hBIuRG9qasrhw4dp0qSJsu3gwYMSvP2/0qVLU6tWLSZNmkSRIkWSPfcOHjyIm5sbDg4O7Nmzhw0bNjB37lwaN24MkOSYzZ9V/OQOFxcXPDw8ALhx4wZTpkwhJiaG2bNnY2lpqex/5MgR7t27h6WlJf369UulWqc/2bJlo3bt2nh7e3Py5EnlOx1/Lq5atYpx48bRq1cvrK2t1Y6V4O0fCb/venp6xMbG4urqStGiRZUbkIEDB7J582Z69uzJ/v37CQgIUG7eJDuCiJeheuAcHBy4ceMGFy5cIGfOnGzevJkpU6Zw4sQJevfujbGxMW3btuXKlSvY2NgoM/3E/2b48OEULVqUPn36sHPnTp48eUKrVq1Su1ppVp06dejduzempqb07duXGzdu/KseCsmdlZi2tjZdunRh7NixTJw4kUmTJgGfxxH16NGD2NhYevTowYsXLzAxMcHf359Nmzaxd+9eCS6+U758+Thy5Ai9e/dmx44daq/FJ+kFaNSoEX/++aeco9/g4uKCsbExf/zxB1evXqVdu3b07NmTCRMmsHv3bgDy589Pzpw5efr0KXfv3lVSX0kPnIAM1AOXO3du7Ozs2LVrF87Ozrx9+5bevXvj4eFBtmzZGDVqFEuXLiU4OJj3798n2ZMk/pt3796RJUsWtm3bRlhYmBK8Sa+muvhUKwcOHGDlypVEREQwefJkChQo8K96geWH8R/xPRGxsbFcuHCBRYsWMXjwYFxcXIDPq4TMnz8fbW1t9u3bh5eXF6tXryZXrlxK8Cbn6fd58+YNwcHBFC1aVJk4E9921tbWDB06FIDdu3fL4PpvyJYtG46Ojri6ujJ37lzs7Ow4duwY9+/fp0qVKhgbGwOfx8MdP36cO3fuKOeqBG8iXob5hoWEhNCtWzeGDh1Kr169mD17Nr/99htLliyhSZMmaGtrc/LkSWrWrEnTpk3lwp2CdHV1qVq1KteuXcPe3h6QMS9JiX/U7O7ujp2dHRYWFlSuXJkZM2Z89XGqSF58MDtixAgCAwPJkiULd+/eZdSoUcri89u2bWPUqFEcPHiQ2rVr8/DhQ+zt7ZX2lvP0+7x8+ZJLly7RvXt36tSpg46ODnFxcRgYGNCxY8dEM9HlRiN5r1+/ZsGCBRw7dowlS5YwZswY2rRpQ2RkJF26dKF48eJA4ptgOVdFQhnqEWq8ihUr0rRpU1q0aIGpqSlPnjyhb9++nD17VtlHLtwpp2jRori6uippWqRtk+fs7MywYcNwdHTk/v371KlTBzs7O3R0dOjbty+3bt2S9vuXGjZsSFBQEK1bt+bUqVPkzJkTBwcHvL29GT9+PJMnT1b2NTExUYZOyKOo75fwnFyyZAmlSpXi8uXLPH36lJIlS2JiYkKdOnXUxsOKxNq3b09oaCiHDh3CxMSErVu3snbtWtatW4ebmxsmJiZ06tSJhw8f0rBhQ54/f57aVRZpWIbpgUvo9OnTTJ48mW7dunH16lV++eUXnJ2d1faRH8iUc/36dQnevoOOjg4VK1Zk1apVHDx4kLt377Jw4UJmz55NpkyZmDJlCvnz55f2+5eyZ8/O/fv3OXXqFPA5memiRYsIDAxkyJAhyuxoQG3cqwRv3y8uLk55JOro6MjcuXN58eIFlpaWnDhxAmtrayVViEiapaUldevWZf369Xh5eaGvr0/37t1p3749pUqVws/Pj4ULF3L8+HGePn1KeHh4aldZpHHpqgcuPjj4N0GCjo4ODg4OrF27Vi7YItVNmzYNS0tL2rZtq/aIyc/Pj969e3Pnzh0cHBx48OBBKtYyfbG2tmbp0qXY2NiorSFbpUoVtmzZgpaWFm5ubqxatSoVa5m2lSlThuDgYACio6OTvcZ+OYEm4X7So/ltOjo6NGvWjKFDh3L37l1OnTpFREQEVlZWTJ06lZcvXwL/7bdO/HzSTQ+cra0tU6ZMoUCBAhgYGHzXMVpaWsTExLBq1SplbVMhfoTkxrKdO3eOPHnyYG1trZbcODg4mH379rFu3TpCQkJ+VDXTleTa9OrVq5w+fRoXFxdKlCihbH/+/Dlr1qyhS5curF279kdVM92pW7cu+/btY+zYsYwfP558+fIlGzR8Oa4t4X4SvH1bTEwMW7ZswcnJiTNnzmBvb8/w4cOxsbGhbNmyyn4SvInvkS564ExMTDh48CDGxsaEhYVx9uxZ/v77b9atW6fsI6kVRFqR8MLbqFEjzMzM0NPTY/Pmzbx+/ZqVK1eSP39+/P39OXHiBO/evWPWrFlcvHiRgIAAQM7nLyVs07Zt25InTx7MzMzYsGEDZ8+epUGDBnh4ePD69WtWr17No0eP8PT0JCYmRlmnV3qIklatWjVWrlzJ1KlTyZ49Oy1btmTt2rWcPXuWbdu2KfvJOZmyDAwMyJUrFyNHjqRRo0bs2LEDR0fH1K6WSEfSRQCnpaXFsGHDuHfvHpcuXaJmzZp4eHiwe/durl27xowZM+TCItIcHx8fHBwcuHDhAkWLFuX169eMHj2aAwcOsHz5cvLmzUvOnDl59uwZ2traVK9eXQKMbxg5ciTt27fnr7/+omTJksTFxbFjxw7Gjh1L9erVadu2La1ateL27dtERETQrFkzGVj/DVpaWowePZo7d+4wf/58WrduTc6cORkwYAC7du3i77//Zvny5dIb9A0tWrTgyJEjymPQf6Nly5Zs27ZNfsfEv5IuAjj4PNMsPhP99evXMTIywt3dnf79+3PhwgU2bdrEvn37lHEcQqSmtm3b4u3tTfv27bl06RIODg7MmjWLzp07s2vXLuDzbOkiRYoQGxvLhg0blAzrchFPWt26dQkMDKRTp05cvHgRgAEDBlCvXj327dunJO/NnTs32traPHjwQBKffkXCXk13d3ccHByoW7cu0dHR6OjocOHCBV69ekVUVBRGRkasWrWKdevW8fjx41SuedrTrl07hgwZwtKlSwkKCuLNmzffddyX33c5V8W/kWbHwMUnioyf+RS/nFDXrl0BiIyMpHnz5uzcuZNjx45Rp04djhw5Qtu2bVOtzkLEK1CgAHv27OHSpUvY2dnh7+/PwIED2bVrF8bGxmTLlo3Tp0+zatUqZYKNBG/qvkwEa2JiQnR0NKGhocq2yZMnc+LECdq0aYORkRHwOSfk/fv3JfFpMqpVqwagBLcAU6dOVXKQAezbt4/r16/j4OBA586duXjxIuXLl5cl8ZKxevVqNm3aRNOmTenVqxdZsmT5ruPiv+/x4zvlXBX/RpoM4GrWrMn06dOxtLRUy+h94cIFJefQ/v37efXqFb///jve3t64ubnRu3dv1q9fn8q1Fz+z+HM1V65chIWFUbp0aQIDAxk1ahSLFi1CpVLRrl07WrRooazMEE+CN3Xx7dG7d28qVKiArq4u2trays1dfCLZwMBAcufOrQQmCcljP3WmpqYsXLhQWQorNjZWOQ//+OMPatSowalTp3j9+jW9e/cmNDSU0NBQevXqRdeuXSXZdBLiJyP5+voqnQk9evTAxMTku9+jUKFCmqqeyMDSZABXokQJChUqxODBg8mZM6dyIV+2bBlGRkbcuXOHiIgIOnbsqCyJFRoaqjyGktmm4kf58scs/lzdtWsXbm5u7N+/Hw8PDxYvXgx8XjOyUaNG5M2bV8ZmJSNhm3bq1IlRo0bx5s0b/vjjDwD8/f1RqVRK+2XLlo179+79p7FHP5tXr17RpUsXcuTIwaZNm4B/VgjZunUr1apVIzo6Gjs7O54+fQok7gmVoFhddHQ08DlJb0xMDIULF8bFxYWePXuSOXPmbx7fvXt3jh07Ru7cuTVdVZHBpMkAbu7cuaxatYqCBQvi7e2NhYWF8lpQUBDBwcF4e3vz6tWrJI+Xbmjxo8T/mNWrV4927dpRrFgxjIyM2LVrF8uWLePp06d8/PiRTJky8csvv7Bo0SKyZcvG6NGjU7nmaVd8m1pbWxMXF0efPn24desW79+/p1u3bpQvX56NGzdiY2ND7dq1GTNmDO/evePcuXOpXPP04cyZM3Tp0oW8efOydOlSJUC7e/cuU6dOJSwsjBw5cij7S8/wt3l5eTF69GguXbpEv379+Ouvv3BwcKBnz55f7YlzdHRk0KBB9OjRQ9IHiX8tzQVw8ReT+fPns3r1aqpXr87w4cPJmTMnAH/99RfZsmWjZs2aqVlNIRQjR45kxowZjBgxgqVLl+Lu7o6RkRGzZs1i586dzJs3j+PHjxMUFISBgQENGzZUxryJpBUuXJh169YRGBio1otx7tw5WrRogaGhISNGjMDf3x9dXV2aNm0qC6gnI35hdPj82Dk2NpYrV67w8OFDmjRpwrp165Rez2vXrlG4cGFlLU7xbVmzZqVJkyaMGTOGDRs2KOlADh8+jJOTk9rj1ITnp6OjI76+vgwYMIAtW7akVvVFOpYmrnYlS5YkW7ZsgHr3fL169dDX16dYsWJ4e3tjZWXFw4cPmTFjBu7u7vzyyy+pVWXxE0v4iK9ixYqULVuWTp06UblyZTZu3EjdunUZNGgQr1+/xtPTkwYNGuDp6YmHhwe2trbKkkPSs5G8+/fv4+joSFhYGDVq1FC2a2lpcfv2bZo2bUrLli1xcHDAwcFB2jQZ1atXZ968eRQtWhT453HpwoULyZo1K05OThQoUEB5nLp//35CQkLo3LlzqtU5vYmMjCQ2NlYJlOOH8AwaNIhHjx7RqVMnPD09MTExUc5PJycnZex2wlx7QvwbqRrAqVQqrKysOHjwIF5eXlhYWCgB3OLFiylUqBD16tVTEp8OHTqU7Nmz8/fff7N//35u3ryZmtUXP5lSpUoB/9xk2Nra0r17d27fvs2ZM2eIiIhg/Pjx7Nixg8qVKzNw4EBy5MjB5cuX2b17N2fPnpWZkUlIalD8x48f2bFjB0OHDqVevXpKipBPnz4pgVpoaCgPHz6UNv2KfPnykS1bNgYNGkSePHmAz9fWIkWK0L59e/744w+cnZ3JmzcvmzdvBsDV1ZVu3bqlYq3TrqTO1ejoaJ4+fUqjRo2UHs74nrYbN24QExODgYGBsg5v7dq1CQgIwMPDQ4I38T9JE3ng7O3tmTFjBrNmzcLf3585c+ZQpEgROnfuzL179wDo0aMHLVu25MWLF/Ts2VMZOCqpF8SPMGrUKIyMjPD09FTyZ82ePZvGjRtz69YtGjdurBZAeHh40LBhQ27fvs3QoUO/Oy/Uz8zFxYWSJUtiYWHBsmXLOHPmDKGhodjY2DBr1izWrFmDp6dnalczXbCwsFAmITg4ONCxY0eePXtGtmzZMDU1xcnJSW293V9//ZVt27axYMECvL29Abm2filh3rzy5cujUqnQ1tbm1KlTWFlZsWfPHk6fPk3fvn2Jiori48ePBAUFsWnTJnbu3KncaMQH1WfOnEnlTyTSu1QL4CpUqMCbN2+4ffs2cXFx2NraEhQUxOPHj3n16hXt2rUjNDRULbGhm5sb+fLlw9PTU2ZCiR+qUqVKnDt3jpiYGPLkycPDhw9RqVSMGDECGxsbVq1axbx585S7bIDhw4djZmbGgAED5HxNQsIfxEGDBtGrVy/Wr19PwYIFyZcvH6dOnSIwMJAbN25gY2PDtGnT2Lt3Lz179kzlmqdttra2uLq6EhgYyPbt24HPiaWdnJwoVqwYXbt25eDBg4nW2ixatCg3b96UoO0bhg0bRosWLYiOjsbS0pKtW7cSEBBA7ty5Wbx4Ma9eveLZs2eYmJhgbGxM1apVlfGZ0rYiJaVKAGdjY8PChQvZtm0bfn5+3L17F4AmTZqwdOlSVqxYgbe3t/JjmNSJLwv9itRgZ2eHi4sLY8aM4dChQ6hUKsaPH0+FChX4448/mD9/vpLaJiE5X5OXI0cOfH19Wbp0KceOHQM+Z7Zv164dt2/fxtfXl8jISOzs7OjQoQP29vbSlskwNzdn69at5MqViwMHDrBx40a2bt0KfH7S0aVLF8LDw/H39+f69etJvocEGslzcXGhX79+dOjQgTNnzuDp6cmgQYNo0KAB58+fx9jYGBcXFzJlykRsbCxjx46VJN1CY1JlDJyuri4AzZo1Y9y4ceTLlw+AnTt30r17dzp27IiXl5cyseHTp0+Jxh7IBVykhvfv3/P69WtcXFyoVasWcXFxDB48mHPnztGsWTO6deuWZNoAOV+T1rZtW86fP8+vv/5KZGSksj0+s32TJk0wNzdXlhtr1aqVJJP9iufPn/PXX38RFxfHp0+faNu2Lc2bNwdgw4YNrFy5EjMzMwYPHpzsJDAJNJJXunRpAgICOHPmDDY2Nri4uDBw4EDOnz+PoaEhb9++JSAgAF9fX/z8/JS8pNKmQhNSJYD7+++/WblyJSNGjKBw4cJMnz6dvHnzAp+TSfbo0QMXFxdcXV0xNzcH5AdQ/HhJBQm7du1izpw5wOdH+vFB3KBBgzhz5gxdu3alSZMmP7qq6dbOnTvZt28fBQoUUK4B8e2+ZMkS4uLiqFu3LqB+DZDrQWLxKyoEBgZy6NAhgoOD0dbWpmvXrjRr1gyANWvWsGrVKrJkycKECROUiQ3i2wwMDKhYsSJPnjyhUqVKzJgxAz8/PxYtWoSOjg6DBg1KMr2VTK4RmpIqAVxYWBifPn2iZs2aNGnShFy5cjFt2jTlAr5lyxa6d++Oq6srtra2qVFFIZQgoVGjRrRs2RI7Ozvg8zqRc+bMITY2lr59+1KzZk3i4uIYMmQIQUFBspzbv/DmzRtcXFw4cuQIfn5+lC1bVml3c3NzIiMjefHiRSrXMm2Lf1IRnyLk3bt3fPr0idevX+Pl5UVMTEyiIG7r1q0EBwdL8thkJHXzFhUVxfr16+nbty+bN29myJAhygorxsbGlC5dmpIlS/7gmoqf2Q8ZA/frr78SERHBo0ePePfuHfB5YeotW7YwatQobt68ye7du7lx4wb9+vVTZkfVrFmTv//+W+5gxA8zbtw4YmNjGT58OABjx46lTZs2vHr1CiMjI549e8bvv//OlStXqFOnDs7OzmhpaREUFMTevXuV95ExL/+OiYkJy5cvp3DhwqxYsYIHDx7QpEkT8ubNi7W1tVwDkmFra4uHhwf79+9n9uzZREZG8vbtW6ytrZk3bx5NmjTBwMAAb29vtLS0WLx4sbIOajwZn6kuYXsULVqUbNmyERISQlhYGKVKlWLy5Mm8efMGNzc37t27R/bs2Zk2bRpZsmShefPm8r0XP4zGA7iWLVsyf/58Lly4wMuXLxk7diwPHz7k+fPnTJo0iQ8fPjB06FDy5cvHtm3buHHjBgMHDuTOnTvKeySciSqEpmTOnBlPT0/q1avHpk2bWLt2LfPnz6d///48e/YMHR0d5s+fT/bs2bG1tSUkJIT69eszcOBATp48qQR94r8xMTEhKCiIevXqsWrVKm7fvs2MGTOUcURyDVCXM2dO1q5dS4ECBYiLi2Pfvn1ER0czd+5crl69ysiRI7l+/ToLFy6kcuXKuLm5kTNnToYPH65MFhHJ8/b2plGjRpiZmXHr1i2ePXuGq6srNjY2dO3aldy5c/PkyRMl4GvcuDExMTFy8yZ+GI0HcNbW1qxbt47z589z9+5dSpcuzaVLl/jzzz+5d+8ea9asoXXr1pw9e5Y8efJw6tQpFi5cyNChQzVZLSGSlCNHDjp37kyLFi148OABnz59onv37nz8+FHZ5+DBgzx//pzWrVsDn3uY45P0CnX/9scsc+bMBAUFkS9fPhwdHbl+/br8IH5Fq1atlNU9rl69ikqlokePHqxdu5b69esTFRVFo0aNiI6OpkqVKjRp0gRfX185V7+hd+/e9OvXj65du3Ls2DECAgLo0KEDrVq14sSJE5QqVYqSJUtiaWnJvXv32Lp1q5JkWm40xI+i0QAu/s6kTp06rF27lvHjx3Pt2jVMTU3x9vbmxIkTNGvWjLFjxzJ9+nRiYmKwsLDg+fPncsEWqSZnzpx07tyZNm3aEBkZSe3atQHQ19fnw4cPNGvWjFGjRmFvb68kmgZ5FPWlhO2RI0cOnjx58l3HmZiYsGLFCiwsLOjevTtXrlzRZDXTpYRt6+DggL29PR8/fqR///5YWlpSo0YNnJycyJEjB9WrV0801k3O1eTp6+szd+5cDh8+zMKFC6lfvz4LFixg+PDhLFu2DF1dXbS1tYmKilI7Tm40xI+m0UkM8ReIAwcO4OjoyODBg6latSrr1q2jatWq7Nmzh61bt7Jnzx5iYmJQqVQ8ffpUFqUWP9SXA5bDwsJYsWKF8njKx8cHgA8fPgAoF+4v77TlB/EftWvXZtCgQQBMmDABf39/9PT0vnmcSqUiIiKC9u3b8/btW2bPnq2kHRL/SHiurVu3jrVr15I5c2YmT57M27dvmTVrFvXr16dmzZqEhIRIGqZ/4cOHDxgYGHD79m0lePPx8WHZsmXo6OjQtm1batWqleg4Cd7Ej/ZDE/k2btyYZcuWsXjxYkaOHJlkwlMhfqSEPRHFixfnw4cPPHv2jIiICCwtLenUqRPt2rVj9+7dTJ8+ncyZM+Pn54ehoSEtWrSQH8Ik6Onp4evry2+//UZkZCQlS5akcePG/2rt4ooVKyq9RmFhYZqqaoZiZ2f3f+3de1yO9//A8VdnOgxlIWdJzmZOI4ecN0qRU2dJci6HORVpZSrHiJpTchjm/EXW0JizzWlzqhkW5ZxD6UCH3x/9uibnbdZdvJ9/cd2f6/Z2Pa77vt735/D+4OrqSkpKCiEhIfz222+A9La9zsuujbq6OlFRUVStWhUTExMCAgKIiooCoEKFCoSFhbF161ZWrVqlipCFUBT6Tgz5SdyyZcuYNWsWd+/eLcx/XoiXmjJlCk5OTjx69IiUlBScnZ1JTExUkjgvLy8yMjLYtWsXurq6eHp6Kr3G8nB8kaamJv/73/9o1qwZkZGRjB8/Hni7ZMLNzY1x48bRv39/JQkRb6dnz544Ozvz6NEjZs+eLdfvNZ69Fxs2bEhKSgqZmZkkJSVRsWJFNm/eTFpaGl27dkVDQ4MSJUoQHh6OgYEB1tbW0uMmVE7zXbxJmzZtSElJ4fTp08qxV31Rf//99zg5ObFixQo++ugjJk2axMOHD99FGEL8I61atcLKygoPDw/KlCmDo6MjsbGx9OjRg7i4OFauXElOTg6DBg3i/PnzhIeHA7I6+lU0NDQoVaoUv/76K1euXKF+/fqMHz+ekJAQcnNzX7huz35XuLq6MnXqVEaNGiXJx//7Oz8StmzZQm5uLqNHj8bW1lau4WvkX1M/Pz969+6Nmpoa8fHxhIeHs3v3biZOnMjSpUuJjY0lOzub1NRUSpYsSZcuXWRvU1Ek/OseOAsLC8aPH4+JiQlnzpxh27ZtfP/99zx9+vS1DzgbGxsGDx6MlZWV9GCIQvX8A7FFixa0bNmSefPmAWBiYsKsWbNo0qSJksRVrlyZNm3asG7dOvnSfolXJRkGBgaMHz+ezz77jN27dxMSEqK8VrlyZRITE5Xr6erqyrRp0xg5cqSyCbv4S/369SlRogSnT59Wiva+Stu2bTl48KDcq2/QvHlzIiIiGDZsGFWrVqV169a0adOGL7/8kt27d2NkZET//v1RV1fnxo0bbN68WVabiiLjnQyhamtrY2xsTEBAAIaGhmRkZODm5kZaWtpbbUQvw1BCFUaMGIGZmRkNGzbk7NmzeHt7K1/KJiYmzJw5k8aNG9OnT58CKyHll3dBz6+IrFWrFurq6uzZs4cjR45QqlQpxowZQ/PmzTl06BChoaFERUVx7do1vLy8AHB3d2fy5Ml4e3uzfft2Vf53ioT8/XVjYmIA8Pf3x9bWFiMjI06ePElERAQxMTEvJBHPf5fKvfpq/fv3p169ety/f585c+YAULt2bTw9PenQoQOTJ09m586dL5wn11QUFe9kqefTp0+5fv06w4YNIzw8nDJlyrB//36MjIxeuqL0+WRNkjdRGJ5diTdy5EhGjx6Njo4O6enpWFlZ0bJlS+X1pKQkxo0bR0JCwgsFeuXLu6D8z6+/vz/+/v58+umnfPbZZ/zvf/9j9OjRPHz4kLlz53Lo0CGsrKw4dOgQpUqVYty4cUBeD+jo0aMZM2aMJG9AqVKl6NevH0OHDqVt27Z06dKFjh074uXlhZWVFVlZWYwcORI7Ozs0NDQKnPv8d6ncqy9XsWJFZb6ggYGBcvzixYtERESwd+9eAgICXrqVo1xTUVT8J4sYzM3NmTt3LqVLl6ZDhw4v1MsRQpWqVavG8OHD2bRpE0ePHkVXV5cFCxbQunVrXF1dOXr0qNLWyMiI5ORk+ZHxBu3bt2fRokX079+fM2fOADBgwACCg4Px8fFh6dKlGBgYUKVKFapUqUJMTIzyIKxevTq6urpS7+0Z5cuXJyoqijt37nD+/HkeP35MaGgokDcsvWjRIj7++GOWLVvG5s2bZTjvH2jVqhXDhg2jWbNmODs7c/z4ceU1c3NzJkyYgIaGBq6uriqMUohX+0c9cJ999hlNmzZ96Ya/AHFxcYwfP56UlBQCAwNf+JUohKp069aNn3/+mS5duij3b1paGp6enhw8eJAVK1bQokULpf29e/fIzc195b3+IRo7diy1atUqcKxMmTLcunWLuLg45VqtWLGCgIAAfH19qVGjBikpKZw7d45du3Yp84gArly5IsnbM9TU1Lh58yZubm5UqFCB0aNHY2ZmpryekpLCsGHDuHPnDgMGDMDR0VHqZv4Dhw8fJjQ0lKNHjxIUFESzZs2U1+Li4pg2bRoDBgxQXYBCvMHf/tT37NmT7du3M3PmTBo2bPjKdhcuXGDjxo3UqFGDihUr/qsghXhXoqOjiYqKwsTEhHr16qGjowNAVlYWnp6eHDhwgB07dlC3bt0C50kPXJ6PP/6YiRMn8tVXX1G9enXleHZ2Nubm5hgaGpKbm4umZt4C9x9++IGUlBSMjY1feC/pNSooP/HNzc3FyMiIpKQkHBwcOH78OI0aNaJjx45K2/wkLjc3l0aNGsmw3j/0888/ExERQUJCAsHBwTRt2lR5LSEhQX68iSLtbyVwtWvXZsSIEcyaNQtNTU0WLFjAJ5988tK22dnZrF27lnLlyuHm5vYuYhXib3nVF++4cePYtGkTvr6+dOrUSdkhICsri6FDhzJnzhwuXrxYmKEWC2pqaty5c4cmTZrwySefEBwcrPQM7d+/n2PHjhEcHEylSpWUVZJpaWnKYibxas8uPvD29iYsLIyaNWty69YtBg4cSEZGBiNHjsTS0lI5JyUlhT59+ihzCcU/c+TIEb755huuXLlCZGQktWvXLvC6/HgTRdXf+lbV19fn6NGjrF69mrZt26KhoUFoaOgrk7jU1FT8/f2pWrVqgYmiQvzXnn0gWltbM3r0aAYOHKjsazpkyBB2795NWFgYnTt3LpDEzZgxo8AQn8iTnxAnJCRgY2NDixYtGDt2LDVr1uTBgwdERUVhYGBAeHg4lpaWWFpaMnPmTB4+fFhgXqF4Uf69OnXqVDw8PNi+fbvSq3br1i2cnZ3R19fH29tbuYcB0tPTpZfoJZ7dfq106dIFXnvZtTpy5AirVq1i3bp1xMfH/9fhCfFO/K1FDCVKlMDY2JiEhAQgb9Pf2NhYsrKy8PLyUgr5lixZkvT0dADq1q1L//79mTlzJikpKe/+fyDEa0ybNg17e3t+++03qlatSnZ2Nrt27cLf3x+Ab775hg4dOjBx4kS2bdv2xvpaIi/J0NTUpFu3blStWpXY2FjGjBlDYmIiXbt2xcHBgS5duhAfH8+9e/fo27cvWVlZUn7hDZo1a0ZERATe3t4cOHBAOZ5fcyx/YYOuri6jRo3i1KlTKoy2aOrVqxdbt25V7rPRo0fz+eefk5qaSkxMDFFRUWRmZr7xXpR7VRQH/3gVqpaWFk+fPkVLS4t9+/YpS9tv3brFtGnT+PHHH/nuu+8AqFSpkrKvoRCFpXPnzsybNw83NzeOHz9O+fLl6dWrFx4eHqxdu1YpKrt27Vo0NTXp06ePiiMu+jw9PRk3bhwODg48efIEQ0NDlixZwunTp/H29lY+56ampqSkpHDnzp2X7r4gXqzZ9sUXXxAQEED79u1f+LGb/31bqVIlxo0bx5gxYyTBeE7fvn2ZMGECGzZsICgoiP79+xMQEMDMmTNp27YtRkZGXLx4kUmTJpGRkSFJmij2/lUZkfwvZS0tLWJjY5XJyxoaGrRq1Uq+sIVKeXh44ODgQMeOHZUvaiMjIzw9PWnVqhUeHh7cuHEDkGLSb2vhwoVkZ2czatQo5Zi5uTnR0dEcOnSI6dOnExcXV+AcubavN2DAAOLj4ylZsiRz5szBwcFBWZWbf+0cHBw4ffo058+fV86TBKSgUqVK4eXlhYWFBT/99BPq6uqcPHmSnTt3oqGhgYeHBz179uTixYtMmDBBkjhR7P2rmcXZ2dmoq6vz9OlT+vXrR506dXjw4AEWFhbKa0IUhmfnteT/+ebNm+jq6hYowXDv3j1iY2Np1qwZ5cuXV47LPKK3U6ZMmQLzWbW1tYmLi2PBggV88cUXBAcHY2JiUuAcSd4KevY+8/DwYOLEiSQnJ5OUlISamhr29vZUqFABQOm97Nu3Lz179izwPpJ4/EVTU5OHDx8SGhrKwYMHad26NX369CE5ORnIe1ZFRkayZcsWzM3NmTFjBiVLlpRrKIq1f51h5eTkYGRkRFRUFPHx8VhbW5OVlYWGhoZ8OESheLaHx8bGBgsLC0qWLMmlS5fQ0tKiX79+lCtXTml/+/ZtLl68+MJ8N0k03mzt2rV07NhRqVD/5MkTAO7fv8+mTZvIyMhQejXFy+XfZ/Xr16d8+fJMnjyZixcvcuHCBQICAnB2dmbixIk4ODjQuXNnNmzYQOnSpQkKClJx5EWTpqam8lkuX748X3/9NQcPHqREiRL069dPSZgzMzNZvnw5mzdvpk2bNnh6eqoybCH+Nc138SalS5cmPj4eLy8vsrOzZb6LKFT5D0Q/Pz/69u1LUFAQFy5c4MKFCwQFBRESEoKBgQEHDhzgypUrTJ06lYyMDM6ePaviyIufI0eO8O233+Lj44OWlhabN2/mo48+4vPPP2f79u18++23gAybvknTpk3ZtWsXWVlZeHt7K8c3bNhAeno6Li4u+Pv7k5CQwK1bt+jUqZMyqiE/jP9ibW1NmzZtGD9+PIGBgXTs2JE2bdowf/581NTUsLS0ZOLEicyYMQPI+8ERFRXFrVu3ZNs2Uey98620JHkTquDq6sqECROU+UNPnz5VXuvZsycDBgygfv36JCUlcf/+fXr16kVWVpYkGv9ArVq1sLe3Z8iQIVy/fh1NTU1SUlLo0KGDrOL9GwYMGMDMmTNZuXIl06dPV4b7AHR1ddHX1wfyeoxBvluflf+57dq1K6tXr+bUqVOYmZnRvXt3ZZ6ggYEBo0ePpnXr1vz4448EBQW98FmXhFgUZ//JXqhCFLbQ0FCePn1aoKjpsw88AwMDDA0NKVGiBPHx8bIy8jmdOnXixIkT3L9//63aa2pqUqtWLRo3bkxmZiZbtmyR3vdXeN01GTp0KF999RVfffUVK1aseGWpJfmh8ZeVK1cyd+5cpYxK/pDopk2bGDJkSIG2+Ulcy5YtOX36NJMmTVJFyEL8J97JEKoQqqSlpUWDBg1eKBabnZ2NtrY2tWvX5tKlS/z555/Ka2pqapJo/D8HBweCg4Px9/dnw4YNPHz48I3nZGdnc/78+RdWRco1fVH+NXFwcKBOnTqoqalx5swZNmzYQHh4OFpaWkydOpXc3FyioqJemsRJ8vaX5OTkAtMfvv/+e/bs2YOPjw8PHjxg6tSpPHnyBA0NDVJSUpg7dy66urro6uqqMGoh3j1J4ESx9/TpU3bv3k3fvn1Zs2ZNgY3RK1eujLu7O4sWLSpQ3kIeiH/59ttvqVu3LkOGDEFNTY2NGze+sSfuZddPrmlBPXr0QFdXl3Xr1jFt2jQcHR35/vvvqVu3LpaWllhbW+Pi4sL8+fPJycnB19cXfX19QkNDlULo4i/5w535cwaHDBnC+fPnWbx4MQBXr15lyZIlAPj6+irD+ebm5kycOFElMQvxX5I6H+K9sHfvXq5cucKUKVOoX78+AGXLluWrr77C1NRUtsd5hfwtxHx9fdmzZw9ubm707t2bjz766K3fI3/vSEng/jJgwACWLVtGQkICTZo0oWfPnjg5OTFy5Ei6du3KnDlzqFKlChEREQCEhYUxa9Ys2rRpI8nbK+TfX/mrSu3t7YmIiMDCwgINDQ2io6MZNGgQTk5OhISE8Mknn7BmzRqmTZumwqiF+O9IAifeC8ePH2f58uVkZWURHR3NwYMH2bZtG+XLl8fW1lbqvL1CfhkQBwcHbty4QZUqVRg3bhz9+vV7qyRuwIABbNu2jWrVqv3HkRYfDg4OfP311wwcOJDDhw9TuXJlNDQ0lB7grKwsdu3aRVRUFLVq1aJWrVoAzJo1i+7du6sy9CItP4HL3wu2Xbt2xMXFER4eTsuWLdHQ0GDXrl04OTlhZ2dHWFiY8vkX4n0kixhEkfeyCdyvmtRdoUIFGjVqRNWqVbl9+zbbtm1TNqaX+Vkv9+WXXzJkyBDGjh2LhoYGX3zxBW3atGHWrFmsX7+eR48evfQ8V1dX/Pz88PLykpIM/69Pnz4sWrSIoKAgZs+eDeSVDFm0aBFjx44tsMdp5cqVOXbsGIMGDSI6OlpVIRcr1atX5/jx44wfP57IyEgAtm7dSo0aNRg2bBhHjx4lKyuL8uXLU758ec6cOSMLlsR7S+bAiSItfw9IADMzM7Kysrh27dorS4DcuHHjhUKyMrn+1UqXLo2VlRUzZsxg69atAGzatInZs2czefJkcnNzlYUNz15vV1dXpk2bxqhRoyR5+3+urq7MnDmTX375heHDh3P48GGOHDlCYmIi6enpDBgwgBs3bnDp0iUgr/czLi6O1NRUFUdedD1f5uP27duEhYXRpk0bDh8+TFxcHLa2tmzdupWwsDBGjBjBsWPHuHnzJjdv3gRkwZJ4f8kQqiiSpk+fTpkyZZTkbcqUKWzdupXNmzcTExND+fLl33rOldR5erX8id75DzgdHR0Axo4dy/nz5/Hw8MDNzQ0DAwPleru5uTFlyhRJ3p7h5uZGcHAwAwYMwNraml27dvHdd9/RqlUrbty4gZeXF61atcLf35+hQ4fSvn17wsLCyMnJ4eDBg6oOv8jK/+x27doVNTU1Hj9+zM6dO6lduzatWrVS2tna2vLHH3+wYcMG6tSpU+A9ZG6meF9JAieKHBMTE3r06MG2bdswMDBQ9jX09vbGz8+PlJQUdu/ejbm5uapDLVZeNgcwNTWVpKQkHBwcgLzthjQ18zrmr127RsmSJalTp45S2qJdu3Z8/fXXjB49WpK3/6erq4udnR0eHh5ER0eTlZXFlClT+N///sf69euxsLDg9OnT9O3bl6dPn+Lu7o6fnx/Z2dl07dqVnJwc2Tf6NaytrVm9ejUbN26kS5cunD59mrCwMAIDA6lRo4bSzs7OjhUrVsgOK+KDIXPgRJFUq1YtwsPD0dTUJCIiAn19faVEgJGREYsWLaJevXrY2dkVKA8iXu7Z4c9GjRqhpqaGjo4Ox44dw9TUlE2bNnHhwgXs7e2VYaslS5awdOlSjh8/XuBcTU1NTpw4ocr/TpHxukr+hoaGBAQE0KNHD/r378+hQ4fQ1dVFS0sLPT09kpKSANlh4XnPT42oVKkSu3btQltbm+3bt/PRRx+xcuVKbG1tMTQ0xMvL64XaebLDgvgQSAInipRnv7zNzMxYuHAhjRs3Zu7cuXz99ddKO0NDQxYtWkTt2rVxdHQsUPtNvJqPjw/dunVDU1OTkiVLsnfvXqZNm8ann37KrFmzyMnJIT4+ngoVKqCnp0fLli2VHiJ5IL6as7MzAKtWrSqQkOUncdbW1vTt2/eFYtOyw8KrVaxYkYcPH5KamoqVlRV9+/Zl37596OrqMm7cOC5cuECpUqXw9/cnJiZG1eEKUeik314UGZUrV1YeZra2tly/fp2RI0dy6NAhbG1tMTIyUtomJyczdOhQ7ty5I0U639KIESNwdXXFy8uL1q1bs2bNGpycnKhcuTI//vgjnTt3Zvv27Vy5coWffvqJVq1aSfL2lnr27EmvXr0ACvSmJScnK/M3t2/fTt26dQucJ8nby/Xo0YMffvgBLy8vqlevzo8//si9e/fIyckhLCwMV1dXUlNTMTMzo1OnTqoOVwiVkB44USS0bNkSX19fQkNDadu2LZ6enjRq1IikpCRlOFVHR4du3boVKGthYGBAamqqPAifo6mp+cJK3YiICH766Se+/fZbrKysCA0N5auvviIqKooSJUqQkZHxwvvI8N7r5Se3devWZdWqVUybNu2lcwPLli2Lq6sr8+bNk+v5lsaMGcMnn3xCvXr18Pb2pmbNmnh6etKnTx+uXbtGxYoV+eSTT9i1a5f8wBAfJOmBEyplaGgIQGJiIikpKYSEhGBvb0+bNm2UOULx8fEMHTqUJ0+esHPnzgIFZlNSUqRI73OCgoI4duwYOjo6yrUpUaIETZs2JS0tDQsLCxYuXEhAQABRUVFoamri7e390p4MSTZeLz9xuHXrFvHx8Xz22WfAiwtG7t69y+zZs8nOzkZDQ6PQ4yxO8hd0zJkzB39/f7Zu3crq1aspV64curq6TJ8+HT09PRITE9m5c6dS51GID40kcEJlZs2axZAhQ1BXVychIYHjx49TtmxZLl++XGB1GeQlcUOGDCEzM5Pjx4+jp6dX4HXpgfvL+vXryczMZNu2bUoSl5GRwaZNm3B0dGTt2rX4+PiwYsUKIK8W3CeffEKVKlVUG3gx4uLiwuTJkzEwMEBTU5N79+6xfv16BgwYQMOGDV97P0pS/HrP9qb98ccfBAQE4O7ujqmpKRkZGXzxxRcFSoiAXFPxYZIhVKEytra27Nixg6ysLLS1talZsyZGRkZ4enqir6/PypUr2bx5c4FzateuzYgRIxg1apQMm7xG/fr1Wbp0KQ8fPqRHjx5kZmbSrVs3AgMDuXLlCuPGjePKlSsYGxsTGhpKqVKlsLKykmv6Cubm5hgZGaGmpsbFixcZNmwYjo6OXLp0iYsXLzJ79mxSUlKYNWsWCQkJBAcHk5OTIz8s3iETExOaNm1Kjx49GDx4sNyr4oMnCZxQOQcHB7p06cLkyZNJSkrC1NSUwMBASpYsSWRkJNu2bQPAw8ODlStXkpmZCUipgDfJT+JSUlLo3r07T548wcnJieHDh5OTk0N6eroy/NS1a1eysrLkmr6Evb0948ePR0dHh48//piIiAjmzZtHZmYmAwYMoEOHDtStW5d169ZhYWFBeno6/fv3Jy0tTdWhF1nNmzfn9u3b3L59+x9fJ7lXxYdOEjhR6J4vnTB48GB69+5NfHw8M2bMIDExkRo1ahAYGIi+vj4///wztWvXplmzZtSuXVu+tF/iVfvF1q9fnyVLlpCamsoXX3zB06dPadGiBVWrVqVatWr8/vvvsl/sazg7OzNz5kyGDRvGtWvXqFWrFrNmzWLu3LmEhIQUaGdubk7fvn0pU6YMISEhzJw5U4WRF13NmjUjOjqadevWUbFiRfz8/Lh27RoPHjxQdWhCFCuSwAmVsbOz4+LFi5w7dw53d3d69erFn3/+yfTp00lMTKRatWqMGDECU1NT0tLScHV1feUeqB+yZ69HzZo1ycrKIj09nVu3bqGmpka9evVYunQpjx8/plu3bkoP5rOkN+NFNjY2LF26FFdX1wKbza9YsYJKlSphY2PD48ePlePq6uqYmZkxefJkSpQoQf/+/eU+fYlGjRoRHR3NpEmTKF++PLa2tpw/f56DBw8q8zJB7kkh3kQWMQiVKFmyJL6+vowcORKAZcuWsXXrVqpWrYqPjw8VK1bk6tWr+Pn54ejoiKOjI1lZWWhoaMhD8Tn51+PLL79k1apVfPfdd8TGxmJpaUlubi5nz57F3d0dXV1dtm3bRsmSJV94D3lQvkhfXx8AY2NjZXsxgPT0dO7fv6/sI5svNzeXuLg4vvrqK9q2bUv79u0LNd7i4syZM4SFhVGzZk1CQkKYNGkSu3btws/Pj40bNzJp0iRKlCgh96QQbyAJnCgUz5ZVUFNTIz09HQ8PD7p06aLsw7lkyRI2b95MlSpVmDRpEpUqVeLx48cF5sjIEN/LjR8/Xtlk3tbWllOnThEZGUnfvn0BOHfuHIMGDaJmzZoFdrQQr7ZmzRrGjx+vDKECdOvWjV69ehEeHv5CT2Zubi7q6ur88ccfnDx5klKlSqki7GLh999/p02bNpQtW5b9+/ezadMm7t+/j4GBAZ07d+bnn38mNDT0hdXoQoi/SAInCkV+L5GrqytffPEFxsbG/PLLL0RFRdGtWzdq164N5PXEbdq0iaZNm9KvXz9VhlxsNGzYEAsLC4YNG8aePXto2LAhn332GadOnWL+/Pn06dMHyEviOnfuzNixY1UccfERGRnJxIkT8fX15ZtvviE0NJSxY8cSGxv70tqDOTk5ODo60rx5c06dOqWCiIseS0vLF0rUbNy4kbS0NKUHfv/+/SQkJODs7EzHjh3ZuHEjubm5XL16VQURC1E8yBw4UWjMzMzYv38/t2/f5sSJEyxYsIDU1FQWL17M8uXLWb16tdLWysqK6OhoGUZ5iefnAJqamtKxY0cWL15M69at+eabb5g7dy5Lly5ly5YtNGjQgK+++oqVK1cq58j8or/H1dWVWbNmERMTg5OT02vb6unpUalSJeLi4gopuqJLW1ubAwcOkJubS+/evbl+/bpy/1pbW+Pk5ETdunW5evUq7u7u3L59+4X3kDmvQryc9MCJQnPz5k2+/fZbbt68yS+//MKOHTto1KgR8fHx+Pr6YmJiorTdsWOHsg+n+MuzD7NmzZoBecVO169fD4CjoyPR0dEsX74cgKSkJO7evav0wuWT5O3viYqKYuzYsXTt2pXhw4e/sp2GhgaPHz+W5O3/PXnyBBsbG9LS0li1alWB/Y5PnDhBtWrVSEtLw9raWknenu/ZlORNiJeTp6P4z3Xp0gUzMzNSUlKYP38+1apV49q1a/To0QM7OzuysrIwMjIiODgYXV3dAudKolFQ/sNs8uTJLFy4kAEDBgDw8OFDdHV1qV27Nrdu3VLKgujp6TF8+HCsra1VGHXR1aBBgwI/HODFBCLfypUrmTBhAr6+vkycOPGlbWSO5otu3ryJjY0NOTk5LFiwgKpVqwJ5Py5mzpxJdnY2derUUdpLwibE25EETvyn6tSpw8iRI9m6dSs2NjYkJCQwZswY3N3duXPnDmPHjmX//v3cuXOHUqVKSfHTtzBu3DhcXV0ZOXIkMTExyvG0tDQOHjzIqFGjCAwM5Pvvv6datWrKXCzZL7agLl26sHTpUlasWMGcOXNo0KCBssr5VT2/y5cv5+uvv6Z169aFHG3x8ezijfzVuykpKdy8eRMLCwuWL1+uzIk7f/48mZmZtGzZUiWxClGcyRw48Z+rUaMGdnZ2jBgxgg0bNhAfH4+xsTGJiYlERkYCeWVFMjMzpcftDYyMjIiKiiIqKooNGzYox/OL8Orq6jJ+/Hjq1KnDnTt38Pb2lh0WXsPY2JgKFSowZ84cUlJSuHTpEr6+vmRkZMg1+weaN2/O3Llz8fLy4pdfflGOR0ZGUr16dby9vZk7dy5qamo4OTlx/fp1Fi9eTJkyZV4Y5hdCvJ4kcKLQdOrUid69e2NqakqNGjW4fv06jo6OXL9+XWkjD83Xq169Ovv378fDw6NA7xvkTRh/8uQJkDeRPr/IrOyw8Gb6+vo4ODhgZ2dHRkYG/fv3Jz09Xe7Hv8nS0pKhQ4diaGjIyJEjuXjxIitWrMDU1BQHBweuXbuGsbGxMmdzwIABPHjwgEePHsnQqRB/kwyhikKzZ88eAgICmDt3LgkJCdStW5ehQ4cWaCMPy788O+SZP6T34MED4uLiqF27Njo6OgXade7cmQkTJgAU2CFAkreC+vXrV2BOoJqaGqmpqSxfvpyZM2eiq6tLVFQU2tracj++pYoVKwKwb98+wsLCuHHjBgsWLGD79u1UrlwZR0dHrl27BsDt27fp27cvxsbGjB07locPH5KbmytD/EL8TZLAiX8t/4v3bVaMJiYmEh0djbW1NYGBgUydOvW/Dq9Yena1qaenJ4MGDcLAwID79+8rRXnbtWunzNkqUaIE9vb2mJubqzjyos3FxYWwsDDS09OVY/lz3rKysoiNjWXevHno6+vj6empwkiLj549e7J3716cnZ0BOHDgAMuWLePGjRt88sknzJ49m4SEhAIJ2p07d7CwsMDb21s5Jj1wQvw9MoQq/pVu3bpRt25dVqxYwd27d9/qnOeHpWSI79X8/Pzo27cvoaGhbNu2jVu3bgGwevVq6taty5kzZ7h9+zYNGjTAwMCA9u3bv7DFk8jj6upKUFAQQ4cOZevWra9sp6Ojw5QpU6hbty729vYv3TtW5DEwMGD58uW0atWKX3/9lc2bN7NkyRIA2rZty6BBg6hYsSJffvklJ0+efGlNNxmmFuKfkQRO/GPly5dn3759pKamoqamxrp16zh58iR79+5V2siX8z/n5OSEr68vvXr14vz580DeYo/83iNXV1eaNWtG6dKl+f333wkMDCQ7O1sS4pfo1KkTa9euVTamr1mzJj179sTc3Jw///yT6OhoTpw4obT/6KOPOHToEAsXLiQiIkKFkRd9Y8eOZejQoaxdu5amTZuyYcMGpQ5hu3btcHd3p2LFiowbN052pxDiHdJ8cxMhXi4tLY1Dhw6xfft2bt26Rffu3Vm8eDGbN2/m8OHDbNmyRZK3f6Fq1ars3LmT8+fPU7NmTSwsLBg0aBD37t1j48aNymrUZ3s1JHl7kYaGBnXq1OHatWvUqVOH+Ph4Vq5cSVJSEmlpafTq1YvmzZuzbNkytm7dioaGBo8ePSI0NBQzMzNVh19kaWpqkpWVxaJFi2jdujW5ubmcO3cOZ2dncnJyWLFiBfv37yc3N5eBAwcSFRVF7969iY+PV3XoQrwXZA6c+McePXpETEwMQUFB/Pnnn/j6+mJhYYGBgQFhYWHs2LEDKyurF/ZBFG9HW1ubPn364O3tzeLFi+nUqRPff/899+7dY8CAAZQuXRooOHdIkrcXZWdnExUVRUREBL179+ann34iJiaGAQMG4OLiQufOncnKylLmcOVfwwsXLqCtra0sFhF58gsf5w/V5+TkcPr0aZ4+fcqcOXM4efIkrq6uSpHpn376iTVr1rBhwwYuXbqkqrCFeO/IEKr4W/J/decPjWpoaLBw4UJOnjzJ4sWLAThy5AhxcXFkZmZSvXp16tati6ura4GhVfF28nuB/ve///Hjjz8SFxfHZ599RmBgIE5OTty8eVPVIRYbH330EU5OTlSuXJmFCxcW2JfTwsKCrVu30rp16wLbYFWvXp0rV66oMOqixdbWltmzZ7N9+3YiIyNJSEjg/v37NGrUiK1bt9KvXz/+/PNPvvzySxo3bszKlSuJiooq8B4yrUKId0OGUMVbs7S0pFWrVoSHh3P//n0gr7ciISEBa2trFi9eTGxsLMnJyYwYMYLU1FSaNGlCkyZN2Ldvn2qDL2byEwsvL68CNd00NTUZPXo0N2/elOTtb3r06BGrVq3CxMREqT2Y33tpaGjImTNnXrimkrz9pUyZMvTt2xdtbW2sra1RV1endu3azJw5k6NHj7JgwQK6d++On58fS5YsYdCgQYwdO5Y7d+4QHR2tvI8kb0K8G9IDJ95aYGAgHTp0YNOmTSxbtowHDx4AeXOMfvzxR+rUqcPRo0dxdXUlOTn5hfNlftY/p6urS58+fejWrRvly5enY8eOZGVlvXRVn/h7tLW1Wb58Oenp6Xh4eKg6nCKtTZs22NnZ0aBBA9auXUtOTg6enp6cPXuW2rVrk5ubi5WVFQ8ePKB27dp06tSJRYsWSdImxH9AeuDEW/P19cXPz49u3bqhrq7O4sWLefjwIWpqauzcuRNNTU08PDxemryBzM96lpaWFk+fPgUK7poAvDQp09HRwdjYmDt37uDg4CCrTd8BPT092rVrh7OzM5UqVcLS0hJ4+fUXeQ4cOEBubi56eno4OTnh7OxMdHQ0zZs3p3bt2lSoUAFDQ0MePHjAxYsXuXjxIiDDpkL8F6QHTryVZ5OFSZMmYWtry4YNG1i2bBn379/H3NycvXv3MnbsWGWbHPGidu3aceDAAeVhNnz4cNq2bUtKSgobNmxg7969r+xZezbpkwfiv/fxxx8zc+ZMNDQ0cHNzIysrS5Lit9SyZUuGDh1K5cqVGTduHCdOnEBPTw8DAwNu3rwpSbAQhUBWoYpXqlGjhvLnZ7+MzczMKFeuHN27d8fDwwMjIyPi4uJYunQpnp6eyio1UdDw4cMJDg7G3t4eQJkjdPLkSWrUqMHo0aMZNWoUWlpaL91aKD95A5lH9C7cuXOH0aNH4+LiIsnb33TkyBHCw8P5888/mTVrFi1btuTx48eSvAlRiCSBEy9lamrKsWPHGD58OBoaGkrCEBUVRY0aNWjVqhWxsbF07dqVgQMHoqenxy+//MKNGzdISkpScfRF04YNGzh79iz29va4urpibm7OwIEDCQ4OpkuXLpw6dYouXbq8NokT79b9+/eV6yzJ299z5MgRvvnmG65cuUJgYCAWFhaAbIklRGGRIVTxSl5eXowfPx4fHx9WrFhBZGQkNWvWxNnZmatXrwJ5Wz21adOGAwcO4O/vr5wrv8ILyh/yNDIyYtasWZQtWxZjY2MGDhzIuXPngLw5WT4+PjRu3Jjdu3cTFhbGkydPVBy5+ND83c/uZ599xsSJE7l+/TojRoz4DyMTQjxLEjhRQL169fj999+VxGHYsGFMmzaNy5cvk56ejrOzM9evXy8w3DR79mx0dHTky/sVnn8gfvzxxwQEBNCtWzfmzZvHnDlzlNd0dXWZPHkyXbt2Zc6cOaxdu1YVIQtBq1at0NPT49y5c9y8eZOcnJxXJnf16tXj/Pnz8qNNiEIkCZxQ2NnZERERQVRUFBMnTlQqrbu5uRESEsLs2bMJCgpS2stE+jd79oHXu3dvEhMTOXLkCGXKlCE4OJjKlSuzZs0aVq9erZyTv8JvyZIlcn1FofDx8eHu3bt88803AAQEBNCzZ08MDAyIj49n06ZNLF++nCdPnry2h0563oUoPFJGRCgMDQ0BcHFxQU9Pj+HDh5OTk0NkZCTa2toEBASQnJys7Ljwul/kIk/+tfHz86N3794sW7aMCxcucP/+fSZNmkRISIiyqCE/iXv8+LHyIJUkWfzXDAwMaNKkCVpaWqSmpnLlyhU+++wzBgwYoBTltrW1RU9PjwULFrw2iZPvAiEKj/TACUWDBg3w9fVl3759DBs2jOPHj+Ph4aEkEEOHDmXatGn4+vqyZMkSFUdbfAwcOJAJEybQp08fLl68WOABaGRkRHBwMMbGxuzYsUNJjoUoTIaGhgQHB1OqVCmuXLlCeno606ZNA/KG9X18fGjatCk//PCDksQJIVRLVqEKxW+//UZmZibNmzfHxcUFCwsLIiIiUFfPu03Cw8Px8/Pj66+/pkePHiqOtvho1KgRa9eu5ddff1WGpfPdu3eP8ePHk5WVRa1atVQUofiQqampkZyczKRJk0hNTaVfv37UrVtXeT0tLY3AwEB+/vlnOnbsyKRJk9DUlMEbIVRNErgPWMOGDdHT00NbW1s5Nn36dAwNDcnNzcXd3Z1OnToRHh6uJHEREREMHjyYnTt3qirsYqNx48ZA3gRvIyMj4K/6bbm5uWhra1OrVi2Sk5Nxc3Pjyy+/VFms4sOTX6ImNzcXExMT7t69y5gxY9i1axdVq1bFzc1NaZOens706dP5448/KFWq1As/RIQQhU8SuA+UjY0Ne/fuZeXKlQQFBWFqagpAQkICT58+pWPHjhw6dAhXV1c6duzIokWLlCRuy5YtylZO4uV8fX2ZMWMGJiYmxMbGYmpqyieffFKgTbVq1fD19cXc3JyHDx9K3TdRaJ6dwzZ27FjCwsJo3LgxDx48wMfHh99++w07OzucnJyUc9LT0xk3bhxjx45VVdhCiGdIAveB0tXVBaBMmTJoaWmxc+dO/P39adasGSEhITg5OWFqasqBAwdwcXHBzs6OcePGFXgPKXz6cg0bNqRJkyb4+vqSlJTEjz/+SNmyZXF1daVFixYAlC9fnqlTp1K6dGl+//135VyZBC4KQ/595uvri7u7O6tWreL27dsAJCcnM2HCBG7dukW/fv1wdHRUzsvMzJQfGkIUEbKI4QNmb2/P/Pnz8fLy4v79+9SvX5/Bgwdz+vRpLCws8Pb25rvvvgPyFjicO3dOVkS+gbu7O23btkVLSwt3d3fS09MB6Nq1K2PGjFGGUlNTU8nNzaVz586v3PtUiP9SvXr1WL58OZMnT2bv3r3K8fwaj4aGhgQFBdGwYUP8/PyIiYlRYbRCiOfJTNQP2Nq1a9HX12fevHn4+Pgwc+ZM1qxZg5ubGxoaGvz2229K2/w/y36Rr5ednY2lpSWPHz+mZs2aynWLiYnh8uXLVKhQgcaNG3P16lW2b99OTk6OXFNRKJ7/kaCvr4++vj6//vprgXbZ2dloa2uTnJyMj48P7u7u7N69u7DDFUK8gfTACQYNGsSMGTOYPn068+bNQ11dHU1NTSkV8Aav6jXr1asXX3/9NTt27GDhwoVcuXLlle8hdd5EYRs5ciTXrl3j999/Z/PmzQwdOpTY2Fjgr/vR2tqa5ORkDh06pJwn96oQRYv0wAmWLl1Kbm4uQUFBZGdnS52nt5SfvNWvX5+SJUvy6NEj4uLi2Lx5M7q6ukycOJH09HSWLVum7B37PHkgiv/asz807O3t8fT0xMnJiQcPHnD58mX69u3L3bt3+fXXX8nJyUFdXR03NzcuXLhQIIGTe1WIokUSuPdYgwYNuHfvHklJScqxV/UaLVu2jNzcXKZPn46uri7BwcGFGWqx0qhRI86cOQPA1KlT6d69O8bGxiQmJpKYmEi/fv1YvXo16urqjBs3jpycHFauXMkff/yh4sjFhyj/8960aVPq1atHcHAwp0+fBmDu3LlMmzaNSZMmcfToUW7dukX//v0xNDRk6tSpKoxaCPEmksC9p7p06UJAQAAPHz7k7NmzREZGcv78ebKzs185FLJ8+XL09PTo2rWrJHCv4Orqyvjx4+ncuTNWVlY4Ozvj6urKw4cPqVmzJhMmTGDv3r107NiRlStXkpWVxdy5c7l+/bokcEJl6tWrx7Zt21BXVycwMFA5/sMPP5CWlkavXr0YPHgwV69e5datW/Tu3fu13xVCCNWTOXDvMWNjYypUqMCcOXNISUnh0qVL+Pr6kpGRIV/M/4CLiwuzZs3Czc2NnTt3smjRIhITE5k+fTqQ17vZqFEjIiIi+Omnnxg/fjwAnTp1IjY2Vq63UKlevXoxY8YMTp48iZ+fH/Hx8QVe19PTA/L24gVZsCREUSd14N5jt2/f5syZM9jY2BAdHU2DBg1Yv349JUuWVOa6iLdjY2PD7NmzcXFxUXahKF++fIEth3Jzczl9+jS7du2iVq1alChRAoA9e/bI9RaF5tkC28/ec5s3b2batGk0bNgQFxcXqlWrVqDd48ePleQNpM6jEEWdPFHeMx999BHGxsYFjqWmprJ8+XJmzpyJrq4uUVFRaGtrS4/QW3J1dWXp0qUvHP/+++8pW7Ys7du3L3D86tWr6OnpoaWlVeC4XG9RGPITr4EDBxIWFkZERARjxowB8koHzZgxAxsbG9zd3ZUkTu5NIYofSeDeI7169SIqKorY2FhWrVpFo0aNgLyhvaysLGJjY5k3bx76+vp4enqqONriYcCAAcrOFEFBQURGRtK7d28gL4HLzs7G3d0dKysr1NTUKFOmDNbW1ly5coWUlBQVRy8+JP3798fb2xsAPz8/JkyYwL1799DX16dv377s2bMHNTU1Vq9eTVBQENbW1owePZoKFSqoNnAhxD8ic+DeE/b29syYMYOZM2dy/fp1vvrqK2JjYxk9enSBdjo6OkyZMoW6detib29PZmamiiIu+tq2bcuqVasYNmyYMmw6ZcoUhg0bhpeXF9999x21atVS9jwtVaoUN2/eRENDg44dO8qG36LQuLq6MnPmTOzt7bl27Rrr1q3Dy8uLAwcOAHkrUOfOncv9+/fp0aMHkNdD1759e1xcXGQXECGKIUng3gOtW7cmPDycKVOmsHXrVgDc3NyoUqUKy5Yt4969e8qWTpA3zHro0CEWLlxIRESEiqIu+kqXLk2VKlX49ddfC0zonjJlCsOHD8fLy4v169fz8ccfU6lSJVq0aMGNGzdkhwVRqOzt7Zk7dy4DBw4kOjoaS0tLli5dSps2bbhx4waQN8etbdu2zJgxg0mTJrFv374C7yFbuQlR/EgZkWJOXV2dypUr88033/DDDz8ox21sbDAxMcHFxYVff/2VgwcPMnv2bAAePXpEaGgoZmZmqgq7WHjw4AFNmjTBwMCgwHBoQEAAAKGhoeTk5LBhwwbu3LnDqVOnlDbq6uqSvIn/XN++fZk/fz7Lli0jOjoagN9//52HDx9iaWnJ2rVrgbw5bufOnaN06dKYmJi88D6SvAlR/MgcuGIuJyeHHTt2sG3bNtLS0gBYuXIl1atXZ8KECfTu3Zu4uDi++OILatWqpZx34cIFtLW10dHRUVXoRV6TJk0ICgrC1NQUKLiiLyAggIULFzJ37lycnJxeOFcmhYv/mqurK/Pnz+eHH37AwcGBvn37Ank/0M6fP4+NjQ3t2rVT2mdkZJCUlKR8TwghijcZQn3PaGlp0atXLw4fPsy1a9cAMDU15ejRozg5ORETE6O0rV69+mv36fzQaWhosH//fk6dOsXIkSNf2iYkJITatWsr84qEKAyOjo7MmzcPV1dXoqOjlbmZ3t7erF+/npo1axIWFsaTJ084c+YMp0+fxsnJCUNDQ9q3by8/MIR4D8gQ6nvm6dOnrF+/vsCxEiVK8PPPP5OQkFDguCRvf3l+DpCmpiZZWVkEBATg4+PDp59+ysmTJ184L79YrxCFpUSJErRp0wYXFxd27doF5PUI5+bmEhoaipqaGuvWrWPo0KG4urrSqVMnmjVrxq1bt+jTp49Sk1CSOCGKN+mBe89pa2uzfPlyNDU1sbe3l7kub9C8eXOOHz+u/L1mzZpERUURFRXF4sWLVRiZEG9ebODr68uIESPw9vZm3bp1QF5Psp6eHo8ePVL+LvMzhSj+pAfuPaWrq0vbtm1xdnamSpUqtG/fntzcXFlt9pwSJUqgpaVFSkoKzZo1Y9u2bRw/fpydO3eyfv16Ll26xLJlyxg7dix79uzh8uXLqg5ZfMDyP7sODg6YmZnh7+9f4DOdv8/pnDlzyMrKYuPGjWRnZyvJG8gOC0K8L2QRw3tKV1cXW1tb0tPTsbS0JCsrCw0NDUnenmFtbc2SJUvYs2cP/v7+aGtr07RpUy5fvkyPHj04cuQIAwcOJCUlhcOHD9O8eXMA2RJLqFzz5s1p27Yt8OIK0sDAQMLCwggPD8fS0lIF0QkhCoMMob7HSpUqxcOHDwFkzstzXF1d8ff3Z+PGjWhra9OrVy+OHDlCv379UFdXR09PjyFDhtC4cWPMzMyoUqUKR44ckcUKQqXye9uMjIzYv38/oaGhLFmy5KVtXV1dWb16tfS4CfGekgTuAyDDpgU5OjoSEhLCwIEDlVW57dq1Y+PGjQwaNIht27YpbU1MTKhatSrDhw+ncePGBAYGKrW1hFCVEiVKEBAQgIGBAUOGDHltW5nzJsT7ScaCPgCSvP2lbNmyzJs3j59//lmpRq+mpsapU6e4du0aurq6yjGApKQkjhw5wogRI/j5559p0aKFqkIXH7DBgwcTGhqKubk5WlpaZGRksGPHDmxtbd84TCrJmxDvJ0ngxAfl7t27uLi40Lx5c/z9/SlXrhy5ubm0a9eOihUrcvr0aaBg0quurs6DBw/YsGED7dq1w9jYWEXRiw9Fw4YN6d69O927d6dSpUqkp6fz2WefMX/+fFavXk39+vU5duwY4eHhODg4YGBgoOqQhRCFTFahig/Orl27cHd3Z+XKlTx8+JCrV68SFBTEqFGjuHDhwgvt8+cONm/enJSUlAL7ygrxrjk4OODj48OTJ0+oVKkS33//PX5+fqxZs4Zu3brRp08f1qxZw6lTpyhdujQ6OjrKdm8yXUKID4fMgRMfrG7duhEVFQXA1KlTCQ8Pf2VbDQ0NVqxYwezZs5VeOiHeNUdHR2bPno2Hhwdnz56lSpUqfPvtt3z33XeMHj1aadelSxfq1auHp6cnRkZGrFmzBm9vb9UFLoQodJLAiQ+apaUlGzZsICIigtDQUO7evavqkMQHysbGhqVLlzJy5EjWrVun9KbNmDGDDh060LVrVx48eFDgHBMTEzw8PPj000/x8PDg9u3bqgleCFHoZA6c+KDt27cPFxcXPD098fb2ply5cqoOSXygUlJSADAzM6N8+fLKUKimpiYPHjx4YTGCmpoaSUlJLFmyhEaNGknNNyE+MJLAifdS/irSNx2DvDlx+Umcra3tfxyZEC9SU1MjNjYWJycnRo0apQyXfv755zg5OTFnzhwlwcuXv7NKUlISP//8M4aGhqoIXQihIjKEKt47WlpaPH36FMjrzcjKyuLatWtkZWW9dpJ3ixYt+OWXX6TsglCprl27snr1avbv30+jRo3w9/dn9erVryzGnT/02rJlSy5duqSCiIUQqiAJnHhvTJ8+nVmzZnH//n0ApkyZQv/+/Xny5AnJyck4Ojpy8+bNN76PFD4VqtapUyfWrl3LsWPHcHR0VHZUeZmSJUtSvnx5rly5UogRCiFUTYZQxXvBxMSEHj16sG3bNgwMDGjdujV9+vTB29sbPz8/UlJS2L17N+bm5m98L0nehKrt2bMHe3t7WrRowcSJEylbtuxL26mrq5Oeni7JmxAfIOmBE++NWrVqER4ejqamJhEREejr6yv7RBoZGbFo0SLq1auHnZ0dcXFxKo5WfIjatGlDSkpKgVI0rxvW79q1KytWrGDLli1MmjTptT1xQogPi/TAiWIvf3FCfHw8Q4YMITMzk/nz5/Pxxx8rbe7du8fQoUM5e/Ys69evp169eqoKV3ygLCwsGDduHEuWLGHp0qVYW1ujpaVFbm4uGhoaLz0nJiaGIUOGULVqVR49elTIEQshijLpgRPFWuXKlbl27RoAtra2xMTEUKVKFYKDgzExMeGLL77g3r17SvsyZcrw3XffcfPmTZydnVUVtvhA6ejo8PHHHxMQEIChoSEZGRm4ubmRlpb20kUKz/fOyU4LQoh8ksCJYqtly5b4+voSGhpK27Zt8fT0pFGjRiQlJSnDqTo6OnTr1q1A74WBgQGpqanyIBSFpkSJEmRkZCh/L1myJO3atWPMmDGUKVOGzz//nHv37r1ypakQQjxPEjhR7BgaGpKcnEyVKlUICQmhdu3aGBgY0L17dy5evKi0q1WrFhEREWhpadG9e/cXhqCkN0MUBhsbG6pXr863337L7du3C9x35ubmzJ07l9KlS9OhQ4cCSZ4QQryOzIETxcqsWbMYMmQI6urqJCQkcPz4ccqWLcvly5epUaNGgbbPzok7fvw4enp6BV6X5E381xwdHQkLC+PJkydKbcJn77u4uDjGjx9PSkoKgYGBr5wLJ4QQz5METhQrBw8eJCQkhJycHLS1tfn++++xt7fn1q1bDB48mF69ehVoHx8fz4gRI9izZw/p6ekqilp8iJo0acLEiRMZNWoUixYtIi0tDSMjI8qUKVOg3YULF9i4cSM1atSgYsWKKopWCFHcSAInipWtW7eSlZWFg4MDixcv5sGDBxw4cAA/Pz/S09NxcXHBxsZGae/h4cGVK1cYMWIEOTk5qKvLLS8KR9myZTl79ixbtmyhXr16REZGsnPnTtavX8+sWbOUdtnZ2axdu5Zy5crh5uamwoiFEMWJPM1EsfD8Pqb6+vqYmJgwefJkKlasyB9//IGPjw9paWm4u7szZcoU1qxZw5dffqkMXQEyQVwUmrp16/Lxxx9TsmRJIiIiuHLlCjNmzCA6OppmzZqxevVqpW1qair+/v5UrVoVAwMDFUYthCguJIETxUL+vCE7Ozvq1avH4sWLWb9+PdWrV8fHx4eKFSty+fJlJk+eTHx8PJ9++imQ9xDNycl55Ub2QvxXfvrpJ54+fcrw4cO5evUqISEhbNu2jQULFjBr1iwqVqyIhYWF0v769etcv35dhRELIYoTWYUqio2SJUty+PBhjh07xpAhQ4C8IVJbW1v+/PNPpk+fTmJiInp6euTm5pKWlgbI3qZCNYyNjYmKisLc3Jz4+Hg+//xz5bWyZcty4MABpk2bxvr165XjFStWJDExURXhCiGKGemBE0XWs71mampqpKen4+HhQZcuXXBwcABgyZIlbN68mSpVqjBp0iQqVarE48ePleQNZG9ToRq3b9/G29ubJ0+e0KRJE/r376+8lpaWxqVLl7h//z7w170uyZsQ4m1JD5wo8lxdXblz5w6//PILt2/fxs/PDzMzMwIDA5W6b25ubnh6erJhwwZmz56t4oiF+Iu5uTlr1qwhLS2NY8eOcfToURwdHSlVqhSdO3eWeZlCiH9EEjhRpJmZmbF//35u377NiRMnWLBgAampqSxevJjly5cXmAhuZWVFdHS0PBBFkVOtWjVcXV3p0KEDDx48IDk5GQ8PD7KysmT3BSHEPyIJnCjSDAwM8PPzo379+mzbtg0fHx+8vLzo3LkzlpaWdOjQgaSkpALnyANRFFWamppoa2vL/EwhxL8mc+BEkdSlSxfMzMxISUlh/vz5VKtWjWvXrtGjRw/s7OzIysrCyMiI4OBgdHV1C5wryZsoqrKysmR+phDinZAEThQ5derUYeTIkWzduhUbGxsSEhIYM2YM7u7u3Llzh7Fjx7J//37u3LlDqVKlCjwQhRBCiA+BDKGKIqlGjRrY2dkxYsQINmzYQHx8PMbGxiQmJhIZGQnklRXJzMyUHjchhBAfHEngRJHWqVMnevfujampKTVq1OD69es4OjoWKHgqc96EEEJ8aCSBE0VexYoVadSoEV9++SX169dn8eLF+Pj4qDosIYQQQmUkgRMqoaamRm5u7t/qPdPX18fd3Z2wsDCZ/C2EEOKDJgmcKHTdunWjbt26rFixgrt3777VOc8nelJ+QQghxIdMEjhRqMqXL8++fftITU1FTU2NdevWcfLkSfbu3au0kTltQgghxOtpqjoA8WFJS0vj0KFDbN++nVu3btG9e3cWL17M5s2bOXz4MFu2bJHkTQghhHgDqQMnCtWjR4+IiYkhKCiIP//8E19fXywsLDAwMCAsLIwdO3ZgZWVFlSpVVB2qEEIIUWRJAif+c5qaeR296up5t9umTZvYt28fVlZWANy8eZNGjRqxe/duEhMTGTVqFIcPH6Zjx44qi1kIIYQoymQIVfynLC0tadWqFeHh4dy/fx/I2z4oISEBa2trFi9eTGxsLMnJyYwYMYLU1FSaNGlCkyZN2Ldvn2qDF0IIIYoo6YET/6lOnTphZWXFwIEDKV26tHI8ODiYUqVKcefOHR4/foyzszOpqakAnDhxgsWLF5OdnY2GhoaKIhdCCCGKLkngxH/K19eXmJgYunXrhoeHB6VKlQLy6sDt3LmT33//HQ8PD5KTk196vpQKEUIIIV4kCZz4z+T3nvn7+7Nnzx569+6Nh4cHZcqUISsri61bt1KlShXatWun4kiFEEKI4kUSOPFO1ahRQ/lzbu5fJQbNzMwoV64c3bt3x8PDAyMjI+Li4li6dCmenp6YmJioIlwhhBCiWJIETrwzpqamHDt2jOHDh6OhoaHUc4uKiqJGjRq0atWK2NhYunbtysCBA9HT0+OXX37hxo0bJCUlqTh6IYQQoviQVajinfnjjz8IDAxk8uTJPH78mBUrVhAZGUmNGjVwdnYmKSmJgIAA1NXV6dq1K7q6uvj7+7Njxw7gr/1RhRBCCPF6spWW+Nfq1avH77//zpMnTwAYNmwY06ZN4/Lly6Snp+Ps7Mz169cL7F86e/ZsdHR0GDFihCpDF0IIIYolGUIV/4qdnR379u3j66+/Vgr2Llq0iAkTJmBqakpMTAzXr18H8laU5hfzHTt2rCRvQgghxD8kQ6jiXzE0NATAxcUFPT09hg8fTk5ODpGRkWhraxMQEEBycjKLFy8GICcnR4ZKhRBCiH9JEjjxrxw9epTY2Fj27dvHsGHDWLJkCR4eHuTk5PDNN9+grq5OQEAAubm5LFmyBECSNyGEEOJfkiFU8a/89ttvZGZm0rx5c1xcXLCwsCAiIkIZKg0PD8fPz4+vv/6aHj16qDhaIYQQ4v0gCZz4Wxo2bIienh7a2trKsenTp2NoaEhubi7u7u506tSJ8PBwJYmLiIhg8ODB7Ny5U1VhCyGEEO8VSeDEW7OxsWHv3r2sXLmSoKAgTE1NAUhISODp06d07NiRQ4cO4erqSseOHVm0aJGSxG3ZskX2NhVCCCHeEUngxFvT1dUFoEyZMmhpabFz5078/f1p1qwZISEhODk5YWpqyoEDB3BxccHOzo5x48YVeA/Z21QIIYT492QRg3hra9euBWD+/PksXbqU6Oho6tevz7Jlyzh9+jTlypWjSZMm/PHHHxw+fJgOHTpw7tw5FUcthBBCvH+kB078LWvXrmXy5MnMmzePSpUqMXPmTNq1a8fp06c5evQov/32m9L2t99+IycnR4ZNhRBCiHdMeuDE37ZkyRJyc3OZMWMGenp6zJs3jxkzZqCpqansxvAsGTYVQggh3i1J4MQ/snTpUnJzcwkKCiI7O5sFCxa8NHkTQgghxLsnCZwooEGDBty7d4+kpCTl2Kt2Tli2bBm5ublMnz4dXV1dgoODCzNUIYQQ4oMlm9kLRZcuXQgICODhw4ecPXuWyMhIzp8/r+xhmpOT89LzRo4cSdeuXbGysirkiIUQQogPkyRwogBjY2MqVKjAnDlzSElJ4dKlS/j6+pKRkfHaJE4IIYQQhUcSOFFA/nCpvr4+Dg4O2NnZkZGRQf/+/UlPT5ckTgghhCgCJIETNG3alIyMDM6ePQuAhoYG2dnZaGpqYmlpyYQJE7h//z5OTk6yUEEIIYQoAqQO3AeuZcuW7Nq1i5EjR/LJJ58AeWU/1NTUyMrKIjY2lnnz5qGvr4+np6dqgxVCCCEEIAncB8/Y2JinT59Srlw5Bg8eTMOGDQHIzc1FTU2NnJwc9uzZw8mTJ2nfvj06OjoqjlgIIYQQksB94E6cOMGWLVuIjIzE3NycYcOGUaVKFSBvPhxAZmYmISEhmJmZ4ebmpspwhRBCCIEkcB88DQ0NWrRoQWxsLPPnz6datWpMmjSJy5cv4+/vD4CmpiaPHj0iNDSU6tWrqzhiIYQQQkgh3w+Ympoaf/75JxcvXqRKlSps27YNDQ0N5s6dS0pKCnv37gUgKysLgAsXLlCvXj10dHTIzMxUZehCCCHEB00SuA9Y/u4KampqNGzYkHPnzjFy5EgSExPJyMigV69epKSkcOLECQAOHTpEUlKSJG9CCCGEiskQquCXX36hRo0axMTEkJKSQuvWrZk7dy5t2rTB0tKyQNsrV66oJkghhBBCKKQHTnD27Fm+/fZbDh06hIeHBzk5OWzfvp0HDx5w6NAhVYcnhBBCiOdIIV+Bjo4OHTt25Pjx49y9e/eF12X3BSGEEKJokQROCCGEEKKYkTlwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFjCRwQgghhBDFzP8BMsXu1QAzqwQAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "embeddings_cuda = embeddings.to(torch.device(\"cuda\"))\n", + "\n", + "functions = {\n", + " \"1) MHA wrapper class\": mha_ch03_wrapper,\n", + " \"2) MHA Ch03\": mha_ch03,\n", + " \"3) MHA with combined QKV weights\": mha_combined_qkv,\n", + " \"4) MHA with PyTorch scaled_dot_product_attention\": mha_pytorch_scaled,\n", + " \"5) PyTorch MHA class defaults\": mha_pytorch_class_default,\n", + " \"6) PyTorch MHA with need_weights=False\": mha_pytorch_class_noweights\n", + "}\n", + "execution_times = [time_pytorch_function(fn, embeddings_cuda) for name,fn in functions.items()]\n", + "\n", + "\n", + "# Plotting\n", + "\n", + "# Customize further for dark mode aesthetics\n", + "plt.rcParams['figure.facecolor'] = '#121212' # Dark figure background\n", + "plt.rcParams['axes.facecolor'] = '#121212' # Dark axes background\n", + "plt.rcParams['axes.edgecolor'] = 'white' # White axes border\n", + "plt.rcParams['axes.labelcolor'] = 'white' # White labels\n", + "plt.rcParams['text.color'] = 'white' # White text\n", + "plt.rcParams['xtick.color'] = 'white' # White x ticks\n", + "plt.rcParams['ytick.color'] = 'white' # White y ticks\n", + "plt.rcParams['grid.color'] = '#444444' # Lighter grid lines for contrast\n", + "plt.rcParams['lines.linewidth'] = 2 # Thicker plot lines for visibility\n", + "plt.rcParams['lines.markersize'] = 8 # Larger markers for visibility\n", + "\n", + "fig, ax = plt.subplots()\n", + "bars = plt.bar(functions.keys(), execution_times)\n", + "\n", + "plt.ylabel('Execution time (ms)')\n", + "plt.xticks(rotation=45, ha=\"right\")\n", + "\n", + "# Calculate new ylim with a margin\n", + "max_execution_time = max(execution_times)\n", + "upper_ylim = max_execution_time + 0.2 * max_execution_time # Adding a 20% margin\n", + "\n", + "plt.ylim(0, upper_ylim) # Setting new ylim\n", + "\n", + "# Annotate bars with execution times\n", + "for bar in bars:\n", + " yval = bar.get_height()\n", + " plt.text(bar.get_x() + bar.get_width()/2, yval + (0.05 * upper_ylim), round(yval, 2), ha='center', va='bottom')\n", + "\n", + "\n", + "plt.tight_layout()\n", + "plt.savefig(\"1.pdf\")\n", + "plt.show()\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "A100", + "machine_shape": "hm", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ch05/02_hparam_tuning/hparam_search.py b/ch05/02_bonus_hparam_tuning/hparam_search.py similarity index 100% rename from ch05/02_hparam_tuning/hparam_search.py rename to ch05/02_bonus_hparam_tuning/hparam_search.py diff --git a/ch05/02_hparam_tuning/previous_chapters.py b/ch05/02_bonus_hparam_tuning/previous_chapters.py similarity index 100% rename from ch05/02_hparam_tuning/previous_chapters.py rename to ch05/02_bonus_hparam_tuning/previous_chapters.py diff --git a/ch05/02_hparam_tuning/the-verdict.txt b/ch05/02_bonus_hparam_tuning/the-verdict.txt similarity index 100% rename from ch05/02_hparam_tuning/the-verdict.txt rename to ch05/02_bonus_hparam_tuning/the-verdict.txt diff --git a/ch05/03_bonus_pretraining_on_gutenberg/README.md b/ch05/03_bonus_pretraining_on_gutenberg/README.md new file mode 100644 index 0000000..1c0f94d --- /dev/null +++ b/ch05/03_bonus_pretraining_on_gutenberg/README.md @@ -0,0 +1,121 @@ +# Pretraining GPT on the Project Gutenberg Dataset + +The code in this directory contains code for training a small GPT model on the free books provided by Project Gutenberg. + +As the Project Gutenberg website states, "the vast majority of Project Gutenberg eBooks are in the public domain in the US." + +Please read the [Project Gutenberg Permissions, Licensing and other Common Requests](https://www.gutenberg.org/policy/permission.html) page for more information about using the resources provided by Project Gutenberg. + +  +## How to use this code + +  + +### 1) Download the dataset + +As of this writing, this will require approximately 50 GB of disk space, but it may be more depending on how much Project Gutenberg grew since then. + +Follow these steps to download the dataset: + + +1. `git clone https://github.com/pgcorpus/gutenberg.git` + +2. `cd gutenberg` + +3. `pip install -r requirements.txt` + +4. `python get_data.py` + +5. `cd ..` + +  +### 2) Prepare the dataset + +Next, run the `prepare_dataset.py` script, which concatenates the (as of this writing, 60,173) text files into fewer larger files so that they can be more efficiently transferred and accessed: + +``` +prepare_dataset.py \ + --data_dir "gutenberg/data" \ + --max_size_mb 500 \ + --output_dir "gutenberg_preprocessed" +``` + +> [!TIP] +> Note that the produced files are stored in plaintext format and are not pre-tokenized for simplicity. However, you may want to update the codes to store the dataset in a pre-tokenized form to save computation time if you are planning to use the dataset more often or train for multiple epochs. See the *Design Decisions and Improvements* at the bottom of this page for more information. + +> [!TIP] +> You can choose smaller file sizes, for example, 50 MB. This will result in more files but might be useful for quicker pretraining runs on a small number of files for testing purposes. + + +  +### 3) Run the pretraining script + +You can run the pretraining script as follows. Note that the additional command line arguments are shown with the default values for illustration purposes: + +```bash +pretraining_simple.py \ + --data_dir "gutenberg_preprocessed" \ + --n_epochs 1 \ + --batch_size 4 \ + --output_dir model_checkpoints +``` + +The output will be formatted in the following way: + +``` +Total files: 3 +Tokenizing file 1 of 3: data_small/combined_1.txt +Training ... +Ep 1 (Step 0): Train loss 9.694, Val loss 9.724 +Ep 1 (Step 100): Train loss 6.672, Val loss 6.683 +Ep 1 (Step 200): Train loss 6.543, Val loss 6.434 +Ep 1 (Step 300): Train loss 5.772, Val loss 6.313 +Ep 1 (Step 400): Train loss 5.547, Val loss 6.249 +Ep 1 (Step 500): Train loss 6.182, Val loss 6.155 +Ep 1 (Step 600): Train loss 5.742, Val loss 6.122 +Ep 1 (Step 700): Train loss 6.309, Val loss 5.984 +Ep 1 (Step 800): Train loss 5.435, Val loss 5.975 +Ep 1 (Step 900): Train loss 5.582, Val loss 5.935 +... +Ep 1 (Step 31900): Train loss 3.664, Val loss 3.946 +Ep 1 (Step 32000): Train loss 3.493, Val loss 3.939 +Ep 1 (Step 32100): Train loss 3.940, Val loss 3.961 +Saved model_checkpoints/model_pg_32188.pth +Book processed 3h 46m 55s +Total time elapsed 3h 46m 55s +ETA for remaining books: 7h 33m 50s +Tokenizing file 2 of 3: data_small/combined_2.txt +Training ... +Ep 1 (Step 32200): Train loss 2.982, Val loss 4.094 +Ep 1 (Step 32300): Train loss 3.920, Val loss 4.097 +... +``` + + +  +> [!TIP] +> In practice, if you are using macOS or Linux, I recommend using the `tee` command to save the log outputs to a `log.txt` file in addition to printing them on the terminal: + +```bash +python -u pretraining_simple.py | tee log.txt +``` + +  +> [!WARNING] +> Note that training on 1 of the ~500 Mb text files in the `gutenberg_preprocessed` folder will take approximately 4 hours on a V100 GPU. +> The folder contains 47 files and will take approximately 200 hours (more than 1 week) to complete. You may want to run it on a smaller number of files. + + +  +## Design Decisions and Improvements + +Note that this code focuses on keeping things simple and minimal for educational purposes. The code could be improved in the following ways to improve modeling performance and training efficiency: + +1. Modify the `prepare_dataset.py` script to strip the Gutenberg boilerplate text from each book file. +2. Update the data preparation and loading utilities to pre-tokenize the dataset and save it in a tokenized form so that it doesn't have to be re-tokenized each time when calling the pretraining script. +3. Update the `train_model_simple` script by adding the features introduced in [Appendix D: Adding Bells and Whistles to the Training Loop](../../appendix-D/01_main-chapter-code/appendix-D.ipynb), namely, cosine decay, linear warmup, and gradient clipping. +4. Update the pretraining script to save the optimizer state (see section *5.4 Loading and saving weights in PyTorch* in chapter 5; [ch05.ipynb](../../ch05/01_main-chapter-code/ch05.ipynb)) and add the option to load an existing model and optimizer checkpoint and continue training if the training run was interrupted. +5. Add a more advanced logger (for example, Weights and Biases) to view the loss and validation curves live +6. Add distributed data parallelism (DDP) and train the model on multiple GPUs (see section *A.9.3 Training with multiple GPUs* in appendix A; [DDP-script.py](../../appendix-A/03_main-chapter-code/DDP-script.py)). +7. Swap the from scratch `MultiheadAttention` class in the `previous_chapter.py` script with the efficient `MHAPyTorchScaledDotProduct` class implemented in the [Efficient Multi-Head Attention Implementations](../../ch03/02_bonus_efficient-multihead-attention/mha-implementations.ipynb) bonus section, which uses Flash Attention via PyTorch's `nn.functional.scaled_dot_product_attention` function. + diff --git a/ch05/03_bonus_pretraining_on_gutenberg/prepare_dataset.py b/ch05/03_bonus_pretraining_on_gutenberg/prepare_dataset.py new file mode 100644 index 0000000..5548b58 --- /dev/null +++ b/ch05/03_bonus_pretraining_on_gutenberg/prepare_dataset.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +Script that processes the Project Gutenberg files into fewer larger files. +""" + +import argparse +import os + + +def combine_files(file_paths, target_dir, max_size_mb=500, separator="<|endoftext|>", fallback_encoding="latin1"): + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + current_content = [] + current_size = 0 + file_counter = 1 + + for file_path in file_paths: + try: + with open(file_path, "r", encoding="utf-8") as file: + content = file.read() + except UnicodeDecodeError: + # Attempt to read the file with a fallback encoding + print(f"Warning: UnicodeDecodeError encountered. Trying fallback encoding for {file_path}") + with open(file_path, "r", encoding=fallback_encoding) as file: + content = file.read() + + estimated_size = len(content.encode("utf-8")) + + if current_size + estimated_size > max_size_mb * 1024 * 1024: + target_file_path = os.path.join(target_dir, f"combined_{file_counter}.txt") + with open(target_file_path, "w", encoding="utf-8") as target_file: + target_file.write(separator.join(current_content)) + file_counter += 1 + current_content = [content] + current_size = estimated_size + else: + current_content.append(content) + current_size += estimated_size + + if current_content: + target_file_path = os.path.join(target_dir, f"combined_{file_counter}.txt") + with open(target_file_path, "w", encoding="utf-8") as target_file: + target_file.write(separator.join(current_content)) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="GPT Model Training Configuration") + + parser.add_argument("--data_dir", type=str, default="gutenberg/data", + help="Directory containing the downloaded raw training data") + parser.add_argument("--max_size_mb", type=int, default=500, + help="The maximum file size for each concatenated file in megabytes") + parser.add_argument("--output_dir", type=str, default="gutenberg_preprocessed", + help="Directory where the preprocessed data will be saved") + + args = parser.parse_args() + + all_files = [os.path.join(path, name) for path, subdirs, files in os.walk(args.data_dir) + for name in files if name.endswith((".txt", ".txt.utf8")) and "raw" not in path] + + target_dir = "path_to_your_large_files" + print(f"{len(all_files)} files to process.") + + combine_files(all_files, args.output_dir) \ No newline at end of file diff --git a/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py b/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py new file mode 100644 index 0000000..8a738d5 --- /dev/null +++ b/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- +""" +Script for pretraining a small GPT-2 124M parameter model +on books from Project Gutenberg. + +Before running this script, make sure you downloaded and +processed the dataset as described in the README.md. +""" + +import argparse +import os +from pathlib import Path +import time +import torch +from previous_chapters import ( + create_dataloader_v1, + GPTModel, + generate_and_print_sample, + calc_loss_batch, + evaluate_model, + plot_losses +) + + +def read_text_file(file_path): + with open(file_path, "r", encoding="utf-8") as file: + text_data = file.read() + return text_data + + +def create_dataloaders(text_data, train_ratio, batch_size, max_length, stride): + split_idx = int(train_ratio * len(text_data)) + train_loader = create_dataloader_v1( + text_data[:split_idx], + batch_size=batch_size, + max_length=max_length, + stride=stride, + drop_last=True, + shuffle=True + ) + val_loader = create_dataloader_v1( + text_data[split_idx:], + batch_size=batch_size, + max_length=max_length, + stride=stride, + drop_last=False, + shuffle=False + ) + return train_loader, val_loader + + +def convert_time(seconds): + hours, rem = divmod(seconds, 3600) + minutes, seconds = divmod(rem, 60) + return int(hours), int(minutes), int(seconds) + + +def print_eta(start_time, book_start_time, index, total_files): + book_end_time = time.time() # End time of processing this book + elapsed_time = book_end_time - book_start_time + total_elapsed_time = book_end_time - start_time + books_remaining = total_files - index + average_time_per_book = total_elapsed_time / index + eta = average_time_per_book * books_remaining + + book_h, book_m, book_s = convert_time(elapsed_time) + total_h, total_m, total_s = convert_time(total_elapsed_time) + eta_h, eta_m, eta_s = convert_time(eta) + + print(f"Book processed {book_h}h {book_m}m {book_s}s" + f"\nTotal time elapsed {total_h}h {total_m}m {total_s}s" + f"\nETA for remaining books: {eta_h}h {eta_m}m {eta_s}s") + + +def train_model_simple(model, optimizer, device, n_epochs, + eval_freq, eval_iter, print_sample_iter, start_context, + output_dir, save_ckpt_freq, + batch_size=1024, train_ratio=0.90): + + train_losses, val_losses, track_tokens_seen = [], [], [] + tokens_seen = 0 + global_step = -1 + start_time = time.time() + + try: + for epoch in range(n_epochs): + + # Iterate over the books in the training corpus + for index, file_path in enumerate(all_files, 1): + book_start_time = time.time() + text_data = read_text_file(file_path) + " <|endoftext|> " + print(f"Tokenizing file {index} of {total_files}: {file_path}") + + # Initialize new data loaders for each book + train_loader, val_loader = create_dataloaders( + text_data, + train_ratio=train_ratio, + batch_size=batch_size, + max_length=GPT_CONFIG_124M["ctx_len"], + stride=GPT_CONFIG_124M["ctx_len"] + ) + print(f"Training ...") + model.train() + for input_batch, target_batch in train_loader: + optimizer.zero_grad() + loss = calc_loss_batch(input_batch, target_batch, model, device) + loss.backward() + optimizer.step() + tokens_seen += input_batch.numel() + global_step += 1 + + # Optional evaluation step + if global_step % eval_freq == 0: + train_loss, val_loss = evaluate_model( + model, train_loader, val_loader, device, eval_iter) + train_losses.append(train_loss) + val_losses.append(val_loss) + track_tokens_seen.append(tokens_seen) + print(f"Ep {epoch+1} (Step {global_step}): " + f"Train loss {train_loss:.3f}, Val loss {val_loss:.3f}") + + # Generate text passage + if index % print_sample_iter == 0: + generate_and_print_sample( + model, train_loader.dataset.tokenizer, device, start_context + ) + + if global_step % save_ckpt_freq: + file_name = output_dir / f"model_pg_{global_step}.pth" + torch.save(model.state_dict(), file_name) + print(f"Saved {file_name}") + + print_eta(start_time, book_start_time, index, total_files) + + except KeyboardInterrupt: + file_name = output_dir / f"model_pg_{global_step}_interrupted.pth" + torch.save(model.state_dict(), file_name) + print(f"Saved {file_name}") + + return train_losses, val_losses, tokens_seen + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='GPT Model Training Configuration') + + parser.add_argument('--data_dir', type=str, default='gutenberg/data', + help='Directory containing the training data') + parser.add_argument('--output_dir', type=str, default='model_checkpoints', + help='Directory where the model checkpoints will be saved') + parser.add_argument('--n_epochs', type=int, default=1, + help='Number of epochs to train the model') + parser.add_argument('--print_sample_iter', type=int, default=500, + help='Iterations between printing sample outputs') + parser.add_argument('--eval_freq', type=int, default=100, + help='Frequency of evaluations during training') + parser.add_argument('--save_ckpt_freq', type=int, default=100_000, + help='Frequency of saving model checkpoints during training') + parser.add_argument('--lr', type=float, default=5e-4, + help='Learning rate for the optimizer') + parser.add_argument('--batch_size', type=int, default=4, + help='Batch size for training') + + args = parser.parse_args() + + GPT_CONFIG_124M = { + "vocab_size": 50257, # Vocabulary size + "ctx_len": 1024, # Context length + "emb_dim": 768, # Embedding dimension + "n_heads": 12, # Number of attention heads + "n_layers": 12, # Number of layers + "drop_rate": 0.1, # Dropout rate + "qkv_bias": False # Query-key-value bias + } + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + torch.manual_seed(123) + model = GPTModel(GPT_CONFIG_124M) + model.to(device) + optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0.1) + + data_dir = args.data_dir + all_files = [os.path.join(path, name) for path, subdirs, files + in os.walk(data_dir) for name in files if name.endswith((".txt"))] + total_files = len(all_files) + + if total_files == 0: + print("No training text files found. Make sure you " + "selected the correct input directory") + quit() + print("Total files:", total_files) + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + train_losses, val_losses, tokens_seen = train_model_simple( + model, optimizer, device, + batch_size=args.batch_size, + n_epochs=args.n_epochs, + eval_freq=args.eval_freq, + eval_iter=1, + print_sample_iter=args.print_sample_iter, + output_dir=output_dir, + save_ckpt_freq=args.save_ckpt_freq, + start_context="Every effort moves you", + ) + + epochs_tensor = torch.linspace(1, args.n_epochs, len(train_losses)) + plot_losses(epochs_tensor, tokens_seen, train_losses, val_losses, output_dir) + + torch.save(model.state_dict(), output_dir / "model_pg_final.pth") + print(f"Maximum GPU memory allocated: {torch.cuda.max_memory_allocated() / 1e9:.2f} GB") diff --git a/ch05/03_bonus_pretraining_on_gutenberg/previous_chapters.py b/ch05/03_bonus_pretraining_on_gutenberg/previous_chapters.py new file mode 100644 index 0000000..0cd8d02 --- /dev/null +++ b/ch05/03_bonus_pretraining_on_gutenberg/previous_chapters.py @@ -0,0 +1,313 @@ +# This file collects all the relevant code that we covered thus far +# throughout Chapters 2-4. +# This file can be run as a standalone script. + +import tiktoken +import torch +import torch.nn as nn +from torch.utils.data import Dataset, DataLoader +import matplotlib.pyplot as plt + + + +##################################### +# Chapter 2 +##################################### + +class GPTDatasetV1(Dataset): + def __init__(self, txt, tokenizer, max_length, stride): + self.tokenizer = tokenizer + self.input_ids = [] + self.target_ids = [] + + token_ids = tokenizer.encode(txt, allowed_special={'<|endoftext|>'}) + + for i in range(0, len(token_ids) - max_length, stride): + input_chunk = token_ids[i:i + max_length] + target_chunk = token_ids[i + 1: i + max_length + 1] + self.input_ids.append(torch.tensor(input_chunk)) + self.target_ids.append(torch.tensor(target_chunk)) + + def __len__(self): + return len(self.input_ids) + + def __getitem__(self, idx): + return self.input_ids[idx], self.target_ids[idx] + + +def create_dataloader_v1(txt, batch_size=4, max_length=256, + stride=128, shuffle=True, drop_last=True): + tokenizer = tiktoken.get_encoding("gpt2") + dataset = GPTDatasetV1(txt, tokenizer, max_length, stride) + dataloader = DataLoader( + dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last) + + return dataloader + + +##################################### +# Chapter 3 +##################################### + +class MultiHeadAttention(nn.Module): + def __init__(self, d_in, d_out, block_size, dropout, num_heads, qkv_bias=False): + super().__init__() + assert d_out % num_heads == 0, "d_out must be divisible by n_heads" + + self.d_out = d_out + self.num_heads = num_heads + self.head_dim = d_out // num_heads # Reduce the projection dim to match desired output dim + + self.W_query = nn.Linear(d_in, d_out, bias=qkv_bias) + self.W_key = nn.Linear(d_in, d_out, bias=qkv_bias) + self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias) + self.out_proj = nn.Linear(d_out, d_out) # Linear layer to combine head outputs + self.dropout = nn.Dropout(dropout) + self.register_buffer('mask', torch.triu(torch.ones(block_size, block_size), diagonal=1)) + + def forward(self, x): + b, num_tokens, d_in = x.shape + + keys = self.W_key(x) # Shape: (b, num_tokens, d_out) + queries = self.W_query(x) + values = self.W_value(x) + + # We implicitly split the matrix by adding a `num_heads` dimension + # Unroll last dim: (b, num_tokens, d_out) -> (b, num_tokens, num_heads, head_dim) + keys = keys.view(b, num_tokens, self.num_heads, self.head_dim) + values = values.view(b, num_tokens, self.num_heads, self.head_dim) + queries = queries.view(b, num_tokens, self.num_heads, self.head_dim) + + # Transpose: (b, num_tokens, num_heads, head_dim) -> (b, num_heads, num_tokens, head_dim) + keys = keys.transpose(1, 2) + queries = queries.transpose(1, 2) + values = values.transpose(1, 2) + + # Compute scaled dot-product attention (aka self-attention) with a causal mask + attn_scores = queries @ keys.transpose(2, 3) # Dot product for each head + + # Original mask truncated to the number of tokens and converted to boolean + mask_bool = self.mask.bool()[:num_tokens, :num_tokens] + + # Use the mask to fill attention scores + attn_scores.masked_fill_(mask_bool, -torch.inf) + + attn_weights = torch.softmax(attn_scores / keys.shape[-1]**0.5, dim=-1) + attn_weights = self.dropout(attn_weights) + + # Shape: (b, num_tokens, num_heads, head_dim) + context_vec = (attn_weights @ values).transpose(1, 2) + + # Combine heads, where self.d_out = self.num_heads * self.head_dim + context_vec = context_vec.reshape(b, num_tokens, self.d_out) + context_vec = self.out_proj(context_vec) # optional projection + + return context_vec + + +##################################### +# Chapter 4 +##################################### + +class LayerNorm(nn.Module): + def __init__(self, emb_dim): + super().__init__() + self.eps = 1e-5 + self.scale = nn.Parameter(torch.ones(emb_dim)) + self.shift = nn.Parameter(torch.zeros(emb_dim)) + + def forward(self, x): + mean = x.mean(dim=-1, keepdim=True) + var = x.var(dim=-1, keepdim=True, unbiased=False) + norm_x = (x - mean) / torch.sqrt(var + self.eps) + return self.scale * norm_x + self.shift + + +class GELU(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x): + return 0.5 * x * (1 + torch.tanh( + torch.sqrt(torch.tensor(2.0 / torch.pi)) * + (x + 0.044715 * torch.pow(x, 3)) + )) + + +class FeedForward(nn.Module): + def __init__(self, cfg): + super().__init__() + self.layers = nn.Sequential( + nn.Linear(cfg["emb_dim"], 4 * cfg["emb_dim"]), + GELU(), + nn.Linear(4 * cfg["emb_dim"], cfg["emb_dim"]), + nn.Dropout(cfg["drop_rate"]) + ) + + def forward(self, x): + return self.layers(x) + + +class TransformerBlock(nn.Module): + def __init__(self, cfg): + super().__init__() + self.att = MultiHeadAttention( + d_in=cfg["emb_dim"], + d_out=cfg["emb_dim"], + block_size=cfg["ctx_len"], + num_heads=cfg["n_heads"], + dropout=cfg["drop_rate"], + qkv_bias=cfg["qkv_bias"]) + self.ff = FeedForward(cfg) + self.norm1 = LayerNorm(cfg["emb_dim"]) + self.norm2 = LayerNorm(cfg["emb_dim"]) + self.drop_resid = nn.Dropout(cfg["drop_rate"]) + + def forward(self, x): + # Shortcut connection for attention block + shortcut = x + x = self.norm1(x) + x = self.att(x) # Shape [batch_size, num_tokens, emb_size] + x = self.drop_resid(x) + x = x + shortcut # Add the original input back + + # Shortcut connection for feed-forward block + shortcut = x + x = self.norm2(x) + x = self.ff(x) + x = self.drop_resid(x) + x = x + shortcut # Add the original input back + + return x + + +class GPTModel(nn.Module): + def __init__(self, cfg): + super().__init__() + self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"]) + self.pos_emb = nn.Embedding(cfg["ctx_len"], cfg["emb_dim"]) + self.drop_emb = nn.Dropout(cfg["drop_rate"]) + + self.trf_blocks = nn.Sequential( + *[TransformerBlock(cfg) for _ in range(cfg["n_layers"])]) + + self.final_norm = LayerNorm(cfg["emb_dim"]) + self.out_head = nn.Linear(cfg["emb_dim"], cfg["vocab_size"], bias=False) + + def forward(self, in_idx): + batch_size, seq_len = in_idx.shape + tok_embeds = self.tok_emb(in_idx) + pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device)) + x = tok_embeds + pos_embeds # Shape [batch_size, num_tokens, emb_size] + x = self.drop_emb(x) + x = self.trf_blocks(x) + x = self.final_norm(x) + logits = self.out_head(x) + return logits + + +def generate_text_simple(model, idx, max_new_tokens, context_size): + # idx is (B, T) array of indices in the current context + for _ in range(max_new_tokens): + + # Crop current context if it exceeds the supported context size + # E.g., if LLM supports only 5 tokens, and the context size is 10 + # then only the last 5 tokens are used as context + idx_cond = idx[:, -context_size:] + + # Get the predictions + with torch.no_grad(): + logits = model(idx_cond) + + # Focus only on the last time step + # (batch, n_token, vocab_size) becomes (batch, vocab_size) + logits = logits[:, -1, :] + + # Get the idx of the vocab entry with the highest logits value + idx_next = torch.argmax(logits, dim=-1, keepdim=True) # (batch, 1) + + # Append sampled index to the running sequence + idx = torch.cat((idx, idx_next), dim=1) # (batch, n_tokens+1) + + return idx + + +##################################### +# Chapter 5 +#################################### + + +def calc_loss_batch(input_batch, target_batch, model, device): + input_batch, target_batch = input_batch.to(device), target_batch.to(device) + + logits = model(input_batch) + logits = logits.view(-1, logits.size(-1)) + loss = torch.nn.functional.cross_entropy(logits, target_batch.view(-1)) + return loss + + +def calc_loss_loader(data_loader, model, device, num_batches=None): + total_loss, batches_seen = 0., 0. + if num_batches is None: + num_batches = len(data_loader) + for i, (input_batch, target_batch) in enumerate(data_loader): + if i < num_batches: + loss = calc_loss_batch(input_batch, target_batch, model, device) + total_loss += loss.item() + batches_seen += 1 + else: + break + return total_loss / batches_seen + + +def evaluate_model(model, train_loader, val_loader, device, eval_iter): + model.eval() + with torch.no_grad(): + train_loss = calc_loss_loader(train_loader, model, device, num_batches=eval_iter) + val_loss = calc_loss_loader(val_loader, model, device, num_batches=eval_iter) + model.train() + return train_loss, val_loss + + +def generate_and_print_sample(model, tokenizer, device, start_context): + model.eval() + context_size = model.pos_emb.weight.shape[0] + encoded = text_to_token_ids(start_context, tokenizer).to(device) + with torch.no_grad(): + token_ids = generate_text_simple(model=model, idx=encoded, + max_new_tokens=50, context_size=context_size) + decoded_text = token_ids_to_text(token_ids, tokenizer) + print(decoded_text.replace("\n", " ")) # Compact print format + model.train() + + +def plot_losses(epochs_seen, tokens_seen, train_losses, val_losses, output_dir): + fig, ax1 = plt.subplots() + + # Plot training and validation loss against epochs + ax1.plot(epochs_seen, train_losses, label="Training loss") + ax1.plot(epochs_seen, val_losses, linestyle="-.", label="Validation loss") + ax1.set_xlabel("Epochs") + ax1.set_ylabel("Loss") + ax1.legend(loc="upper right") + + # Create a second x-axis for tokens seen + ax2 = ax1.twiny() # Create a second x-axis that shares the same y-axis + ax2.plot(tokens_seen, train_losses, alpha=0) # Invisible plot for aligning ticks + ax2.set_xlabel("Tokens seen") + + fig.tight_layout() # Adjust layout to make room + plt.savefig(output_dir / "losses.pdf") + + +def text_to_token_ids(text, tokenizer): + encoded = tokenizer.encode(text) + encoded_tensor = torch.tensor(encoded).unsqueeze(0) # add batch dimension + return encoded_tensor + + +def token_ids_to_text(token_ids, tokenizer): + flat = token_ids.squeeze(0) # remove batch dimension + return tokenizer.decode(flat.tolist()) + +