From 5d1d8ce511e5541d382432cbea4fc5c14ebea03d Mon Sep 17 00:00:00 2001 From: rasbt Date: Thu, 8 Feb 2024 20:16:54 -0600 Subject: [PATCH] add shape information for clarity --- ch04/01_main-chapter-code/ch04.ipynb | 19 ++++++++++--------- ch04/01_main-chapter-code/figures/ffn.webp | Bin 18404 -> 24926 bytes ch04/01_main-chapter-code/gpt.py | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ch04/01_main-chapter-code/ch04.ipynb b/ch04/01_main-chapter-code/ch04.ipynb index 742c962..27eda6b 100644 --- a/ch04/01_main-chapter-code/ch04.ipynb +++ b/ch04/01_main-chapter-code/ch04.ipynb @@ -29,7 +29,7 @@ "id": "53fe99ab-0bcf-4778-a6b5-6db81fb826ef", "metadata": {}, "source": [ - "## 4.1 Coding the LLM backbone" + "## 4.1 Coding an LLM architecture" ] }, { @@ -479,7 +479,7 @@ "- Note that we also add a smaller value (`eps`) before computing the square root of the variance; this is to avoid division-by-zero errors if the variance is 0\n", "\n", "**Biased variance**\n", - "- In the variance calculation above, setting `unbiased=False` means using the formula $\\frac{\\sum (x_i - \\bar{x})^2}{n}$ to compute the variance where n is the sample size (here, the number of features or columns); this formula does not include Bessel's correction (which uses `n-1` in the denominator), thus providing a biased estimate of the variance \n", + "- In the variance calculation above, setting `unbiased=False` means using the formula $\\frac{\\sum_i (x_i - \\bar{x})^2}{n}$ to compute the variance where n is the sample size (here, the number of features or columns); this formula does not include Bessel's correction (which uses `n-1` in the denominator), thus providing a biased estimate of the variance \n", "- For LLMs, where the embedding dimension `n` is very large, the difference between using n and `n-1`\n", " is negligible\n", "- However, GPT-2 was trained with a biased variance in the normalization layers, which is why we also adopted this setting for compatibility reasons with the pretrained weights that we will load in later chapters\n", @@ -558,7 +558,7 @@ "id": "7d482ce7-e493-4bfc-a820-3ea99f564ebc", "metadata": {}, "source": [ - "- GELU ([Hendrycks and Gimpel 2016](https://arxiv.org/abs/1606.08415)) can be implemented in several ways; the exact version is defined as GELU(x)=x⋅Φ(x), where Φ(x) is the cumulative distribution function of the standard Gaussian distribution\n", + "- GELU ([Hendrycks and Gimpel 2016](https://arxiv.org/abs/1606.08415)) can be implemented in several ways; the exact version is defined as GELU(x)=x⋅Φ(x), where Φ(x) is the cumulative distribution function of the standard Gaussian distribution.\n", "- In practice, it's common to implement a computationally cheaper approximation: $\\text{GELU}(x) \\approx 0.5 \\cdot x \\cdot \\left(1 + \\tanh\\left[\\sqrt{\\frac{2}{\\pi}} \\cdot \\left(x + 0.044715 \\cdot x^3\\right)\\right]\\right)\n", "$ (the original GPT-2 model was also trained with this approximation)" ] @@ -680,7 +680,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 30, "id": "928e7f7c-d0b1-499f-8d07-4cadb428a6f9", "metadata": {}, "outputs": [ @@ -688,14 +688,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "torch.Size([2, 768])\n" + "torch.Size([2, 3, 768])\n" ] } ], "source": [ "ffn = FeedForward(GPT_CONFIG_124M)\n", "\n", - "x = torch.rand(2, 768) # input with batch dimension 2\n", + "# input shape: [batch_size, num_token, emb_size]\n", + "x = torch.rand(2, 3, 768) \n", "out = ffn(x)\n", "print(out.shape)" ] @@ -832,7 +833,7 @@ " # Shortcut connection for attention block\n", " shortcut = x\n", " x = self.norm1(x)\n", - " x = self.att(x)\n", + " x = self.att(x) # Shape [batch_size, num_tokens, emb_size]\n", " x = self.drop_resid(x)\n", " x = x + shortcut # Add the original input back\n", "\n", @@ -957,7 +958,7 @@ " batch_size, seq_len = in_idx.shape\n", " tok_embeds = self.tok_emb(in_idx)\n", " pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device))\n", - " x = tok_embeds + pos_embeds\n", + " x = tok_embeds + pos_embeds # Shape [batch_size, num_tokens, emb_size]\n", " x = self.trf_blocks(x)\n", " x = self.final_norm(x)\n", " logits = self.out_head(x)\n", @@ -1002,7 +1003,7 @@ ], "source": [ "torch.manual_seed(123)\n", - "model = GPTModel(GPT_CONFIG_124M)\n", + "model = GPT(GPT_CONFIG_124M)\n", "\n", "out = model(batch)\n", "print(\"Output shape:\", out.shape)\n", diff --git a/ch04/01_main-chapter-code/figures/ffn.webp b/ch04/01_main-chapter-code/figures/ffn.webp index 38ea715ac6ea0cd0216e516f801619d1bfdf27e3..d686cb3f4176e3a2d319fcafe4300f6f495a22db 100644 GIT binary patch literal 24926 zcma&MV|1mCwf8>X@%iSTSz}a<;$0W+ zdXyx^#MBG{0CiCz1vLc@0;xacIx3JHAgV;r8xa1C(G81s2JSMVM#&@xW++R$cix!x ztR!j_(e)42FVs(z+sb|Z$uI9O#~0_1r>CzCeF1*CR&ZTfj1{&-tE`rf=>UzKNruV39;)qY?v3+IJrX@Pzxyoi49uTy@PZ|CRsPjMf4 zJ)awU74KOvR8=8>ojGvBBTb175?dH#P?C-txB)Cx7gFop=-qANgD~cR%TghA4WTZ^nY9=T2)CW zc4nMcP9n@T4$Ri5`%ec3GW;EYXzR>sz4C-2+|Wd3N$03BLwJOJDT za}F2#)?C@6{z>1NW@ywzvKd-Tw2; z;54$7c(o8(eg3yorhp}~=7B<0Q`PuCVj!tQGw$ac-{EL%eB%EV6a2A?|7SpL67zGE zL;UYu^y;7{}Qx>y$84u5*mB!#y~_& z$+Jd>V%1HPhnMcT#w!2x5|aE+CxRSpHl)5SI&aUXscIAg0fvum@}iFUd2m-uNbfy- z*e{)B&%jMJ68fPJ3YkZSTTw#ON#f(Ei`{E1b)~P}H#WgcEB-@f6!ns1lTwH#w#SMd z_Pddj`!DGn(%B}^6wYUe^&Z)f6t#c5Qc5rS9j@78%m-CG@(AMQ+>-}~aFpJ+L7rCj zW{=}u7=6>F||!WEt)`LZe1#?6i{#VM^>;I{1hR*@99 zX%}Z+j3<)e&dA*n9-sJ5l7Eao^?D+jDn)h!Et3KNx^lN&m`6bh{`HB&s~DbWTkjY_C4Wf!d}H@4Ez|8f@do6O zD4jJ)l{+#9{`GeOqtrKj?`&SBFeLk0=dg-@KivLnE4vk4Ikfxrf2#Pi@W1BKKDb2T zi1ybwZTV-(98soq7k>u%Q}F#CI{a(H*8bYLk!ZB96@>rN%=T6xR4FB~AQlS(?F6_A zHZze`D#`87;&G90dsrPkwqNp&z)s`cv)59h76>lesf@hf%ZRk z^)lCSn8wFqlMdtcR7&3W70$c~XA(ZNA2& z0b``#GJ|e|a`dTHEe*>lP5vyLxbX|~#ZnqBGA4*7nXTCai@guTNI{W?Tw z)WtZ>hf69Cs7+(tbDE#HaE*o_v{g-hp(oG#UjhTVzT})c$nx$0YHoWqW6c=93J3F?_U~ z5bNwPStw}rG?S+d*>bzMn;c;ePt8uiO*VlY9-E4os zBtQRT$G%5W7ovdh-MW*s-@-#vmH#4Ga?2LHFV%pCMWEgD`P_EC;pN_9z06eGm#o;L z&fNzsAVRZImXfrF@80Gw@W)LdL~vm#g;AQkHvBeBJ`FnERl5GGwBsElA3?XmFp4VV zH2249Iv5qFPDh!a?#aS0SL3^f?4>0Yb9zy3s`)GslDB~l4!x*2EWSc`^^DX$8?PQ} zZPHM5kjh%4_YjKeYY6X^*7)yLAbDqnr+CuhH(QW9%Xq{h8GG4=pWS;^B?M!0lktj_ zpp67)P3NnXCCY)t{yOO9crp8JYXm85B#UbC+X9%oRcg1*PX%?M++2HK0*H{)WSSWg zHD1UrDW@eluCI-WX$;vSHqy3K`CGix+a`4vJtXA;85@MSW$M+XsdAExaI@*@l$j>D z$1;s>;Dhrr!$8lW)Mv5+l<=W;x@I=#h>#=$-w{SEM=kn|o9TQBqzu@7N=+(AdFx5! zt-r$vh70`e70CR2z~>HbY7P-){-J~n!1D(LPDo5Mpd7rJ*~u2TwTJngh3vv2 zYW#~qHBhfiQkSZ{V*ZfNaMmn?;%==Gv+rjVXk7DfYlsH8^hC3!9TwhS^cY2y`_)AS z2Ds)jM>90{Cnn1w{fn9+-IE5bQUlz<=C#jLSSTB!zjqq-8k-IFJurHXcJ>0739?O0 zcvU9_tA>td55g18R64u;6AJ%EsiGqMSP}yK_XlUUvTtPH!{BNnzFk`HzBMayiiX&U zfDfPTsQRtvi(n013wR_+{RODFC}axouaBIB{{vDwKhF3w{*A_wqEl;ONH!%n|1COC z<72a1p>F@hLc$TVB9nxa-rFx1LAZ?{z}JHc^fbIAr`eSw#I*Bl&-Vg3(_FKTAl}6j zPe8C<#6lfVH0*_=rLeBmYBhF?e*-q3ON!mUiflq7x#3nVG`Yyz1_AhO5Pch^4vU=p zXeF~WK}Qq`08r@k$3_)iyAz9uJZrwwNY2}tk*DRjXK!HSkk;Rv58ssQ(1B>`rd0t4 zephb{3Wl;9ckirn5afZif(EXE!fG&|BXlYoGebiLz2z~w|{4(41ii` zSkpH}gq|6?EB~g?{oETvry-wA3Fp-l=>o}^?L&$n_|$uPil0YEzn6Dj3eW+WY;M?8 zxUFG@O+s)1)R8Q{r!_Xqz@)*>;wPwX8UM4?zl25(sdiTLKQu>7 zTFibLa%`$oy|c^SW+28Ogw#7L@JYPfZ;}UnKg0Y8TfyPK!uv1PQMujyuMn?{#Af)l zXLO%P?Jr52MwbtTBctaWa_=1JcAcahI{Oc~@t3SIpi=u+wCErD{uvgAiBIvrNSJ>p zm2K3Bf83Ms*3E#w1d(jz<$qL0f5CyyG3)HK)Nme?{}728Ce%(D8n}#`Gz*V>zwX#1k1tJk+%H@!e{wcLxdJX*o;!!E5 z&&xnNB{5eLV7d473SQpH9(8C{i{zavcXQ zWR)eLdzw{g#Rj|sGLA zmw?Dmn5$**pJ+=Z7Ny9yI{W+u$(G6*MlN;Lx-Ws;@u3hnp9I;U86&95v)x_Y82HNV zlnrH_EE#Kqe7C8mgZhx|m1bXEiXk}H$mdhI&r;Kp`*4sMsktn211x1lWIn#sFF(5S zs%#7bP{osBR!ma*Ny8zEZ3vJ*y_+>0vGf-1bSpu6y|ekmk;l6=Cf#e|*1g&q#3kBS z!z^Qlp}VwcjOZk=(t7VQv!ltM6JE>TF+mUen4Q_rg5xgbvS%iYsZG$5BPEN3chSwn@iRJ?E+lOw>}h=KH(a+>+j zg!ZtP2Bu*(EmkCq#DyGmaS5XGbAE0-!wTPqJ*IFV&{t~8Y#Fm+^vi5Bbznn%v8aOW zkuM2GzG`~K=@DtZ7fc*@reAFp1$5YZ9uxB0uD)L4Q~~qwh(*chAvj zRrJvkWf*7gB&2;d2^=$6z{0@EjhvpijacA;c5`sAz!I77<*GokIsZE}D*pCwU~`R= zd^Jy(MDT@8U1Y;v(zyP-o5?Jh*+s1oOKvVxV*ABguf_A`rH%WPlewW0v)fk{T}~BY zad+KWlnAd*d#Z1l%8r_Cmnd)9?9^99lk9iGFJPoZK^69#jkz1WHG#iJnK;~^Rp6fD z#&$uVW?}`$2TQdt3EV|p#f%$JGgJE|pnvN)!YRU$WyG(%DDS^H!;#B>(Uc?F8vwrgSp7`kFB`_k_7|u7I5dC!ClC2A?*=Yj z-Yg0B4=?+-HJfnC{uh1xCzJR8nMoZw|34tA9QNH^fd2oohJQ~uDr6SsAJ&lB9lD>> zlT?w;EUbLm>g8a8!V&VcOKsPLI$q=@lzG3;fU%}=_%{=%@N68THg)RT-;VSef8hyF zBYmFyk%FMK?SQcC^l?}P9M5lY6=`IU(Zo?5IVh;M!_h)M&-O1oJhRpA7rns zVF|m8@*!ktldtj`q0Al%tQRW>{x!Q>z{+<5nAYgaEwt1$yDNsnALBd0Jo}wy19QAn z81PM@d2uO?2%)Lc3x4m->K)z0&tMx33OmOQSq5qMj}Sxzc$%+F@gpI6Uc%@g3LQ9d zH+M{N>HM%+gRK@nd*3vLUOLE4GSyb=DF>Tv9F6>UGnRCY&J~IC#{95(T`>uXXuXsc zF2JyXG$EIYNtHvbN3^p23#_oc7&)?cbJ%KNJ(^O_N_wD4Asa<)X#RHR!*Khj5ny$= z=VJ=cFIC}5?`Nt?b8JdJH!-Mh`gy@8c!LrXpb8!N;kzLLkaX4iTOo9N{fuLna1Q;V zu>R{W!Ss;=u4gN0M2qSxR7r=AXD~c5@gy%XRWWQ$**}!}R=vboY~z3(dJUc-QPjS(lodmRNl+9JUI`#&vX3^oFXHPHdP2i#L{ppt2(JhIp-Fe(w0LhQ$W_1*WrQe{W>2WtCnipYsnE3g@EFlPIO z;$>KZjQGvbyM~K`z>cQ3!#+ZDVf|Bb9v&SW77AnARWF)D>!@Xt6)MZ}Ov_GqeZt_V zfD52IZ=NMLJNIq62p3{I(NLg!wt*poB=>sEr2Zx=qq|xTwlUriAi8dO^peIcjNq%D z(#i78su&+O;s*mBvkEe{H=_@8U5gx`Rd%^Z*2XaJoaIZFpTWUq+p+ap<88|iG!cyK zUc2v7fBG>pJBnUAU^P0>8c#w&p`m#6Nphpb88D*`R1{=CKW|_g%fu2H>_Vfq{U~bZ6 z8iGjW?DcPu@bne_0>yVVmeIESOF>O{&>UT~;w8+Pv+HZb&c1vQ8VRs`gNY*}50L&x z9JqeqWYTEn7Z!q4;*Km*X9#egZ}C@(cPa)b!yFC$?Z!6RLuVB0`3?7=mqzK5C$;!> z<}XSs4su#bRR7L;nl}&)P^0{%DgH*Y|B^Ox*kTc0rLQd5$TS&{@_r1YFEI+G{7#dE zLHn@tfRxdFa(d9hAM^JQ==xVe>~9zNck;`kO<`4G3=ssA;}{RIqhf_VhnbKh zU+E*<5#y-nuSvhx{wMSJ&z#vkUWO6^NxtY)`>?P6U%9rf?*A^A_D6_@^Riq0Gi~^f zViy4L`SE!J000=-S;K}&Fnbz9nVf%var|ax>_Zr(`0|#EQ zr&|@asxJtPx?JUWSm0z`_X`HZC4+Ng%1%BxLUPPhxQ?AZ6_X-$rimH+0MG35R9&W5 zQ~p%lPuP3MPGry6cSFZvp-pg|ka@60M+q!rJynAjo@%T2^26N7?pm<}28;>defw2W zcrVEo(jzDa3LU=%5@qPWR z)r4NzE}vhV`v6gp`K*y$4?sZwmRgIhlWLz4b=O=`dH}dXp8m|sW+my@p{Jq`@sm1C z^1qY{h|crT#Bs$$p@H`v2raZ_rtY!gyXqj99`6O~%w`oU&650G8R0(?7jws~RNT{1 zH9ogQ+n7a8A?i28&VcCc>qH@;blT;UScp|%)C*zT<=_JHtv14zsE9J+GMw)@yzW z&TXtAdN?dC>gfJK3^mNYT?l6=NSeza&nXU08DY1=;ws0mzyncAeKJqCpoU-)Wr7R% zAqA}R%A830;H*1S3aa$k*rXX9Gxu&e)o#??VV}{4atCY5${6$>I|(Ke0DxEwILM%5 z?@(Z{cP^^6S0Og|s%zLY*+q-t8&*DC(S`3_!Ob@E-j?&`Qe+ebW6b`EupHKGF;{gK z;JdjUIqTwLY-*JWviIY_aj_w^Q$9!kN(T<|(MHiF6ozhUk1Yz5WHB~T5?N%CPRMt9 z>8rN`Yza9?RR`Y6$TUhc5RD3IqWzZp{QP)<6^rEEX>G4JaN zFU@aEcpIVxPee;b%BmOaS?l_Cxl<2jE%vRfqE07%=v`?H_1=BDh7Y8~!+jil)TW>y zV2!*E7?U0hJ2i3<{SWDwy~tI85w|6yE@_J*-q8HV!+TOzx`U>uS;g7kLjdlC*11k<25VW(~<!?X0RTuNegv!$g6FH7TPf42PxxXLL9Okcw|j0} zw*e)&-K2!lpQwAAlpS@c0J5hD>=aNR`JHU=R<{M!aM~2Gkf9n=f_W5rIw{`k`ZHP= z{M*lf@KWvtU?=6d2_^9QjBN%0nsDRrpp-2cuUO2HeMQy6_G*9~bXx=;g`Vbj8P2C| zGsG79YzW%YA9IP5cTb?^&GSw}tU>T^qc-`zj(Bh}Uqy-v32+iora>LadSh;x#D;w{ zeSs@R%zR3`EKSF$`an7LJz@USuIAPbuVJ%D20&}x;0|?XY~w8>fF?@rAk8dhiK&mM z&$OHJB-c6ct&K{%nk%uQQp|b<^qQNmnq@1iI>Im;cx@G()j{mn}yn?gxh=|g|~oGL5*ibeh+pVL(56pFR< z30O>foie;MRYxCwvY6|y-;Bf~l;{)AYaltyb9oyVC}~dcd36_X3F5KSpHke{+d2?# z_cy|TObWUU{ws<%?Cx_u@$%^8 zvEQO=udV6vCKB+5kclI)H}Kcy)sBMiIF^phTj&@7KDTxVjR5qG$FtEo7%hTgO6GZ~ z5@R^Kzy#^|j9dzQATngfqflV<{Ywz;41S-DQh7#t^vd^<-xX+?vcK7+Khzz&XfDi3 z$^~A0J9@>JIOD`8#Fpa+S$knXGwt73pjTW7D*Ni1u%mto1Q$H-39TZuzJeys4jxH>XVK!;HaPyV~iB$ ztkvHH&9oK%u3TGSJ69$;%8e5zeU>cAR-_RG9~If12Pq=7;!`Y zI*Ww-cw_;+AB8*n#aC@*KN8HdvNIdWF*!h~VrD^YRLTWu>#~SS6mtrc-O@%Ga%eA| zr(RgPXr;I8o-fK32hTwAniHee!=63;UvfFAHp%&FLGm9sumpfG&x&|VpS|$gI6xxw z+0e2AU|~AMtAQ0Qq9__*mcLMP5|P5rkda~5>P`{2cKa5G)9@b zC~%EI5FYhXP1XdW;WdCtbM#T0f^lg)ucLmY+)ECk_G7lC@pMa(4T_;re~uMmkq;8L zIPVaSFVslh$gHDfYipC9&Kd=l#t+_ED;dIl#F0* zWr$6_Wa!JuLa;CqO^Ia(Sv)yQFm8-*R0H(>5ZEKcI|z_MhI6@CcuyYse)E1}bO;X& zv#8QYQ$O|`yh>8Jr3X8*W0Mkc_xApWBryi?RN@Rxfp`R2HLQ+zDIu(GMnaS9rm1y5 z+wQL{fxniz+z&;isnj~p!-zg+x=_W_cEF)$l+vHLOnGprQU0_&h$9z?Nb#Jd}($A zLmZZ%7RURH+9Mw@$U4gV5Y$Ho@LS}>f*9(cZd&Y;5f5LRKyW)xVNJEDox>7Vuy5dp3221SM`kJ`YG%%Ju@+c1+_#ZdRs z&#ID@y$7SMl2O8o@rjaa-#v=+XRS$sVz`{f__enZ^&r^K($H0z+g)ldN9He*c-0f8 zI{*VNXTTW6_!51gW0frP;1azuOnjqF!Lqin!^)6Tq^p0)77=InBfKqM5$?iRNIBe@ zuOYzq2f=X5kXopGAsS6oPJN)|g6&jc6kP!KP~cYBsDDUPv@gw6Bd&DWd`Ro-*^b2X!;eIC; z@TkR8-nWEjIzm(BovR>F5MYFgVZpEA5sT4~8$!jr#rRRUbl1WmHLVJnO}=zpF&JUaHs{}k*$3^ zpt+N+urMj4bouqxQI{IL>4FthNuw7pdfH6~sJJ_%z$Q-EwaHk%yTmc&fTUjjGO6kE zBiovjMTHqFoM5V~syl<9;pFzQ`x!e*WxVD!Syri$Q3rO?y{p)O&3rw{Gk+5Yl5QM_ zc+0?d$5+qO%)FsLS?3#u1%~yVm(tF0t`^Sds9;UM%pqx`k0{aXi>ShUcLUPJo8nmd z(6G6J;-nbzN&#m0hq*C9nPJBOxYxefeGKxdhxbkV0L``}Y6>ieql0#;xSV6%9I@m< zV!>vszpd*^967BKN!xgl{$ZNTKxgW=jH0S~sZd^wz*KAqJSaxnh}(=OFQOoNbrH1S z;DpR|vPtU&aF=@DmX)rei?Sf$qoE2FYmXge2X-0*?gH9*0%W}t2a%tsLERUlJ06Oc zrYv+Fa|h?FhqD3DE%03psXX)Ko%PptiVRxt!@vwrq+8k(Bx1I~jU{8GY+3W~MOMco zgt>=FKHG(X5IWQ>fZzPp6HW}LxV0H=d}puJ7mYbH)5;_YyG>G{FgU6Ul*VE@UYlhLz`+eWcI7vh@pF4sQy4eP6COK`?(x| z>UU1J`1jt68`=?=KmeEd!e_IK6K7>oKz2_xj$`lih#z*XQH#Qvu|w|@fOD}uJmj@N zI0`QXEWD$~JI#R4Y;*S@WfD?gl%SA`UFMee%47d@7>C>V$+qzz?x+?sFQRhV2vf^Z zwE*l{6#L}^u?V>@z=q>(ziP1-LaRV4NMg3Fo78R}w=-Nw1i2@cje5cvxPZ!wC51v6 z$RND7*hJ{H!=fmSegCT;ho4Ps7q0~9p$&97?=_&vsS8FZTZ#7e`X2G8ylr!vhj_4A zbF*}*M}%j(d67XHR}fA}n-({fz_y8xuubI9M6-5lM76eYhCbDP1@RG1%Zzs|WcEkwco%YYRLW2=!7Y`rQuNAm3<&XGN?(=rATW}RIrgZEpdqG=u=zKMl!2ZyC3baj$oE`&@P0rvt_X0)8?XEF1t{lp2Dq~-5mBF6DvQaA6o>Pu8 zhAu0XHTF?U%UeOWUm$C16f7UKYdgdp4g|Wddd!SH%89Jhu-R`bF~XC*u{V*M^w-B3 zc`vMzYCi&gE&P(C3E)v0P@}A%IbFIhRL^O4wUF!fh=N=$l1FznHx$7Gb^LJ+gOer$ zHcXIqjOR-bXZoP;l+sP^@-Y&=TaW|45+k#H;UY!-y%XX$Ay5}d3ExXvImR29*-L58 zGR)Eau>H!=Xu{~1I*V<*csQQLZO60F{n5i?!j(ZdRJXHjgkWLBT9c4rk4nk;c7M)( z>*vo-PRJ8>v6HJMl0EAu_Zxi{Mh5~~Zt=@6>nUMuBsJkgW<6FGF@cxm04n&BQX=u= z2P<1}Zed#!B{0D`USNr$13ybs$RmWYLUTSteobMY?_2CLT|=O_d6MEAY2#G$5`eL((! zKoe#$E_1y^Vu%~U0naQ%_q48LIfYrhm?QV>6;eBA;ndO460(be8o0gDuwzS?`RGIC z@#T5ztPlkO3TTv;HHN%PC7})66ozL2?U*o(?G-GtH$Ft35O{23kCPAj^FCXxsrg9d z1{gD0E0G}W-mHMcPb~f%(KyWdOqva;2m%sNB%;F-{T?`8YH(w#NQn7~%m!OIGcVtl);9(|A=nm% z;VZ%Y^eVKC7?uJN1cwP`?LY>r#0-|CT0gHD-)f@xvgOQ524_(~yjHj5Zc)j&a?kf9?#w8Rg31^4Jwmu zomM#Pe;hW;0$PIKP~`}S{r;xiWt%m$??CVCrQ&$IQ(t(s8;hpBF%<{pwYS#z6p#ji6K(j^l*# z%hcg{Aod2mq%?AR3pD|kr-Eu;roQDR!BKMuV!>~}i(yq7j~ayCVO6mmpA8Dq3zQD! z<&xjO`H<%^w-!6RqLuT29ZwIoiOd%>0N~xc5A$K~y-3^9;4& zop;S*m$%j0I^d!sR?OPd!ATvb(o7{Zn$<9R9FmF=Rm`F)z`%2m4xIst4^b{)MS*cx z|I2$?sJDWg7i$$clhvB&n9z0u7o>Cv&Z-q1LnGc4WdhEeb$@pxxLD!T76}&D>(vKW zjB}3W^jCE|jRmvb%h2_;O<=xfiataRz4?;f%AAIFKTmsvhDM+a zQS0h{ot2}%iUNobDj$3it2Z76b3H_ULHlz>f|25h9o*Zxn^B0bbD&4XOA$)?BK1Wu z%fjJ@ic+!7g68sFS=R6PdaHB?c|q*u7A0>0%>vaF@w6U!^u10>0OrX`3eKGIC3=o+ zEuK)f%O)A?9)q*R*3qSvE*4yYAdx6rz3jK;jCDI{LRB7MQpYOZq(@egqLFUd-8=Yr zsmXY1W^LVrSF-2__nv+_gKg8zIh!=ifjSWhx212uyq3mi==ar4G|$`=I~1Vsr5P5$ z1}LO$dy69ZgHE#C)}H!8sKVgiRcKd93g}S;8q-ENNX9t|J0y;Ug)G!lUK7BB9>d@C zjFt;zE9Y}73_Mp&J4H86@*VTj&sjpNRdP}`KDo4@rZwEfW`2A*(M{<0Xr-Qzl`T&6 z*P^{-5c3vxy`$4)OjQ@(rh&yON^}LvHub)XapjmT>`+k>dC&_U< z>wu9u*6z+9ALe7}v;nBCYmFZqd`l;xuuHrE?@&zit{mhO5)?R%z%R*g#&EGs%!rT3 zAuKu(X;LCUdr#w#m>x{8)R69BMWLB<^ER{ml|)>*3B~m3Djr6&JqYAXU$72MH$#OF zG$Aiv#*ijuL$>@wqi8R@zuT}Orjjr^)}QdK5N`~{jBzp1Er`zlO@CxRZr-|l(+F$H%<*itXhSr$i|yb=Qv+%=-AAcp zZ57w(}X-Ri#$&4f_bB16kue5GWWud;9^p$(mL2EJDEs75g*cHzA zc~9ruwqqZ0C0fI#ruk1`?}1?s@{xH4!@dFPPVv#Pq!G~s3S2v;mUO9pPn^`r-JHSA{dD2d3R*zZJX4EPJUmi4IVS z4`BTBUIqZVG%CeZCc1dr&l#y7i-5xZT5fO4bhQWd=ps895qYL_dsrDO&9G9VW<$<3 zVuDp?!^&6$_z4SN3rZTpC*&n)42hqjli{vNWo?Lye|g~d?Lv8gUzkFHwCJsvZ0X>v01bzHkc1m09km@i#eD#(l<#! z!_B?4{dASCvZZigNDwPDVe*qnVMz?EQ=kK;KY@db!R$Keq=XVVb&7Lo7WCF=JGC{X z@`9pV7>UETHK#C4kQ~@ti(J0aGKH0pSUT11O%llTV zWD?fG9Y&M4`R(JdY(HNN$6T<7*!f+wC(0qJ6ZQo3(&EWkOf_CmZeEI5K^4 zHiJS9Rz6pT2CO=xULoj;DhJTn^GZtit1@LD@(OKwD;J_c|16Db;*^Iz3~|bL(UPXm zbZ;WF3sp@hw3~?7%}eGUqcpp80HCXqv&(sr2Swf#aXRWB} zlp1~djFrz6fWYq9q&^UgtK}Z2tC^dLUz4h0JmQo{T$!)&?c`a_itQA7W$Y8l*`o48 zCepm{jX;w(-mppx7t`q}JdI*xp!?burieJE*(|(Us&Lou%bM@n+^W1BE|gw;UHeOq zUu!vlt;AFv7Em07T>}1_QiVXMI;fT=8`d9*cQc+ef&r5GgFTH2>Q{gZ%4I{VgE;FB z^#}u=)kZ3gB6!`sH%>ZQCpRCeB4#_Q_?CU+>m3Th4$Q@`&hqlfKY0ao4v?T-x*y() z_LIf=`r`@j2n9s%>e6%Jr;j;cv~>@>Lt5eBwL_+O7s0}9N-~-lH~8p%pCS71D*gBU zEQ%PdMfi%3h)*?72ps*=wS3q7TKQ=%{~eE5v+K5&(b< zjf>Mz2GX-IT0qG*h{K$pLr)=b%C*3gaLI?^j~^cQ^2%GX_+4CnOnkxAO74$ax(t%k zbOg+AD>|9NeYJN{Dn-*c3`^AQO7%k@MyC79S6v>iG+TAv`}9PjCq{j+LkOcsaaQT^ z7`mg;Tc@g?{T_;$a>gBRv2pgL51Ha@*g}m~mgi5&e-jO|>*Z>-)F;+-n-~~Ux9Cwn zhb02-2~0WXuq40@4Op|DAmA+&zrdEVLyRu?{!8`b$JNgQU(1HL4oeO$Uk-D_>e;x< z!Q#RhtU_R*NdhL6ocW9*`bfwzmP=`xipfqooaP|QG#!Xb0-pkGli>? zQm}gG9_}=WqzB^^uGh!TDev(JnE)A zl;8(6H}4|kMr;&r zyvixPvY~$NXRIFxn}F6zPJv-pop}TO!n#Xvb40)xri5;fsv@5afFq7TzgLnED&uD} zYpsm4d4>k{VfnS(@EgqbW(NQw^tj#9UdJ;x2?sq9UyZ(W0>oO9^ky-+h3p;Y<}-mG z9tI~AnY)GeHWnZK(U$kVxapvr(oVy{*INyhEh?Yi@xf1fQR25DrnyX7l z@N9Qd?RMGSl9k|*EjE>eP=9#D0|sM5cvxv8{kd8NGa#KN zKb#FsTngfe6>xQ?LY#o(%dAH#Z@2>sIoL89}s` z;Y<6lxAE>Nt=mVi_#zzWby3S`cHzrK_erD1vhknYC5+X(&>8M7P2vLNYRnxhaAL7L zl;P?f!$a783)V3(miXk@Uh%E|{c<1?L{7_BLRK!Wp5xoyV0ZO~WLLqGm02ApRO%^B zNKouLM5wdArl)Crg$Q3#zxq4JEEVSW!j%HZlrA+TVKg|#KpFU)j)^(EZJlsV&c*>& zLI?6W0b?OTP|IdB%C5u;jgt(kS-Wr4iZYh=$jJ=jC@d0BMiPN3cB_lh{)jjmyq&#& z-a5ed$0tLZy1u`87iaIwz8H`_OxxzDJ`H6FQe!A1pnAp@YmLWun( z8J(B)+*SJED#&)VV#PaL4yeFjzVl7(BhjbNXQf+8+Id-OU9-(a0d?7f_P+;tGixJFvW{ z2R-AgR;%~Cs@Ts9#yg) z5($|%k`p&Xp>-aI(qbaFV?s&eZtA`QjpVV=p`N?FIMjo;D6IiCG>5fj)~5V~QDe_B zj8RYG&7?3T4)$gAuQ}hYKKI|{G?x5^$O|ZLQKvstM0mOs^K9Y%s0myT_3HGMg_Qvz z4k^=#LJfjvgE=aq?O*WF%dx;mH2zH+Z)1T}GC&023V&%25E{X#v9g2Q5Fbk{E($^f zA~f#t%mbvIzv=)~)_?y2fRXfx1r(DSbGC=?iZ6ep6#R@uU31tVCQf-*Y6*>a2HyLUGls?Qg{ z`68=yJZ%-`SKlvPFIg9|?Lq86z~T;`t-l0~(V%^b;dK4ugarKM3flYi@`m?e!gT;$ zWiCu&fZQ6ajYFnrJB3`BXIT&?IAd+BhR?=n*5t04I`CAbSvf|6950Xp$>htBS(#_! z&&z#j=fySbFKZ92v7ivWuY#bnb8ZdBOu|;LenPW$V>a&;lw?&|4ik^P^a?Vjp8A_* z6!bbsWK$Sn;9BYYolZdr%XC*oVq9>(#9C4bOs@ljNcR22nttp{W?)~GmBO}_N`^#1 zaB+s0CZXwSi#u#G&tQ&J0gj~#22SW+m&abess*{jh)e!^mlCWy4PO;iV<(?BJ?-!$z#S>KH#&kMD;edovRQ4a%PZ`<@O;20Z%P*WgY`^ld-u7>nwTj9z9G3Js9KF;9k_~sN^JCmT)TXG z6x*b!&jUoZ0&NDq`2IW(Rawg8gEwu%&fhZgz#=#O1k?6I&eqGCk&K_jPRvifK2+pQ z6kya`r+13z!%323<2okV`APw6x-)i|8f}0@eF`)T(Ok}uVvTl&M|c{x_XKS(cO75V z@QZYiwZ`O_?!z`yN#K8rg{G%AQ`I@gg@{h@{qy0JGA{V%W-2H1u@UM=ajl4O?8L<@ z4NdOn3+7289oIeM5jgAh;=O*)T2hYB4BVoAtL!A5-wJIa1iTVC7(H(}hf38T3alI~ zu6p4eAC5de_;|*zc)gvhN9Fo;Wrjg>jpz%BOYTn>HHX<4lVY7fHF&g^K(<+1qcK_xNOkt=X7iH;Tt{dTlwxO3yLP zN$BV-uS&q&IsN}HXdRd0-lksH-QKE4T|UFv%pROLArMiBuj#Xk3wNFJiNvEl`6t=e zY5n4Kz>hFU)*IV<7;KPW^Fl z@0}%jo6K8yizv9QRFhj7Bd%Lax7!1T-$TuC?-Xd}=Y68zsdQqj!YOm{B?E8d+u zGS?wRYV4(UE4{oSt>IBDXvrjwx?}309V&ME`PQ*S?fKx22|-- zo!H8qX3niQ9*HWkeqTkCcJDaWi-)A*Es%xrQ)_W4EBK`NyR;}2y2@5a6_?5scKP33 z_5D(k0CVOx)Zg;Ng~(C`VhpgYhnJ*zIY>fr-iugH+)w!GMzD`RnO47Iu|9=WBx8pt z!Kk+sEAfO25OWq-5Pus&Vhh`Yp0Jpj#4;fA`JKK!>KKM(L2le9Rpw*;W%iDgSSS*) zLigtF=UdVC@(s>EvS}wm!s`i#)f|2jEwzB*722K;D+T#jgkUz9okJi&p4`K^;QB)Q zK#)T^;Wr7E2V*`)SjPw>VuC)k0OQzeBAe?acVK+YPXf$`Au2&%zkD% zD})ywEfHZj9#~_W&V-9oa5BSi#=Ba7upE^dobhMFuW=5iafV&kDFAN};1*3QM87i% z?sxNsQp&h16VUcpP10BG2%l#=`I%Xjw7|09o%#cFl)^KRr4pB{awOBH2-W3y%H##0PV{B|s76 z?fa5BPX=qOo1)p34}!Q?xwm72if=Q3=NE~-se#Lr9d|IBkE5GoV>chuXV^J7u0)Io zJ}92<#Y^Te+eD+IOvp7ZNGMU-f_H0`*>XtZFuqHB z$+pihRQ#D*f~1PvTox!rnt;@Cf1fT%hk^cNx$fTq=1+iMHHkhJb{U05*Y_fD#HL&N zc5kUcK@In=;nN(V?$D?L&z_%{F`++aoDdroVnq(L5E*58xAS93sHiP>mSspY=5;^u z0sbq`6eiQu#vyN=+NsQZd4h+EfZnokZTLrjoCh%%aw>I|;Y)xr4on!WlAgbaNQyuoy;Tj_cnwr8xkCD>sJ1E0o0X1U2TEx%hN!m4X{)dtQPc-m}QT z?w{FoJNVCG75SK|;P5QfoZx}(eVhfe_MY$G`$duV1%4oDbe$Hx(hVZbgEDia%Oke0 z1-@}4XVmoad52{-Hy)M zn)BNpo)~$OLGfRNm^vn^ZbaY2BmDR}r+4F^5FNDB_xwauSxfIV-`okB6zK{pZr8Ur z?PJcLt&d9r_139v)7D+!cvlt3OV7p~GyFN6;#J0p3z07YnOV3Mug5eQ^v0`s{>9o1sq zeE{b8W~m8WV+@GDFz@lyBiaY^ds{x-8@)0QbCEVT7OKCFH^YYC(mT!%iDg0MbdaPI zY}xpn_%Ol8BkVkSScD;&?Y{8yG+LOHd@yB0X8>FXWYHlsO|}2G8z5aQN}WfGvc^3I zx2(b|9n~s)KW69R9OeQ8%5;jjj!^n6&B+8iTxT{y(t5W`SrU0%_Mr_kS2>>C z-ZgTlYrLhwy5+>fiO;pweK=IVr zf;2VjeapPfY6T0yB{fJ$rN3Yb)wPhfyHLku#_iMfZ7jSKFj3TrBV35RN=Z5l10W0b zHf9KHb=61X(N_u}m|1|>9fqA!ouEJ?&bn?9?yg|1)e}?$HRc*`lW6{|Cvv5OEt7i2 zCoN*wf11|-IlW!7%MbP9ny2ui!n<1$lxn>)7I6#aAX%_il^Ta0qVs!9^Z!+ z3@vRPa`JzY&r#0RIlIO+2JX2_gREjnhqk6>(98Z9#+gu0EprddPh1W$;_MI>*u!Hu zNv8qTwsT{TP=O!$Q|JSDT}8);eU%Ft9pa)=i5|B?pVm;Q=n0>zqH%f zp=r3n9I3vE=;h}B4u*fcJIrBkRdkb&b|$ihiM`Hf>};N?>wm8b1=5dTQ{Fmad`^sL zGwd?ovsBx=Sk&;7!Vr4avuyj`aus{9+J{J={6USeqQdM$uarbtd%a7j_#tmIfl);U zkf1}C!x6Bu);>!W+ATGI?CGxyE}+^Mqux!%&i~tr z2mNZm>`Ceh*q>YBVWyA)nr;3UBsTyifAb2ew;m*aaS9mQP4o=)E?5US4|~?*U^vXg z;GT&oLclAGpxm@L)xh|BU8EvIJn9IIE#mO9!`iz9BU5nuN(D7@j&lenZ&{Y#jsQI;dc>E)Zl=7w$+fQ3Y&Ee2h#T3Y-#) z^9+ezZbjoO{Qz36xiSG5;t~IlTfdE&4-?}ug(hWP?4cVjttJ%S!&Tdk+AzPaZ9L7NVsyAjK zF9&ei)Htou3u>kKtPVLXv!;^)`Tc_@3BFSYz5wll_g&M(27VDfmwdQ0*1GVBIc12F zZq#-rCKlv`ud}@}GzR{1A-QN_(C^6renLHxL#~!SOMlo)$WNxr9=ORnQ+4Uc%I|+VDe(SUr`7`d6#MOmc>4eSp$3Rv7H5f&dz)9z4U0sC(o!d3(#T5ptkn$Io{lX- zF%<`8kt!v^>BL1g&;ou$YqlT&1A-KQ9aAyx=KNbi;gT+VzNvCNP`i$%&esCwGOh+H zi8k_YHl4sij^hp17<03Ix$H4Tk�OXP;%ONXA*_0p%H$#VxOL4-*3i)=D*ytcg&W;M2`mA4OXvr;o?Kly%^3@12c z{@_AF%h(ThCPV>=FmVs?)z0e{tp+5QJ#Kk!O%}pFxMRfH#>0M(o`Q*To6(~Xrf9GM z%IzrTiT0~*EpVwGfB*mh0A>i;000000000000Qto#W*H_0000004d)5kZGEmJU$5; z5Aw&a43<3Bi`I{~r)uGV^?DZ((e*+Ke^Hmv-a0xKlARJn9|*Q2_IDd?EIkI_tV(lA zOL7&!7N25_*?XE*ttm0U$FJPa!ayZ_9X>U}wrqu`{tE@Jx48b$tom<-&nDk5j~|a2 zsaIW*kZzQFW_sm?co*nCR6ESL&@Qcp+v%c#VndB!kg7BROgb$it4bLm=^Yv1k!J#X zd=}Xl%^(`n8$g^t$2B^sUkTfi9V;$~FvCWvDx{Y|Jmfpcb?i}B*>LneTMEAjtzF;6X(nHv;dsIk=)s z8)Aw)Hw95+O@J17*O4Laik$(~5g6GG60mmE>}ek6(&zVrQHNgE^j$XL_OTOg_3_VW8q@1NxW$tB z9ZRHJuxqv1$V&WML@tQi|ImddelDRLZ2V zicn*!sr`hhOVS|TVjI{Q5>YJ|2c*A%uq~L@>_d64zJvxig7NuC^)g%9Uy8_Z8dy#f zNd%r6$$TYh>dP!s=mUCv>s*>qqf1nTSsH22iJr)gR{Km_ z%~`3$XYr;gR);SdtnVnSY3X)f72$WBqXpdr?4+gXPX*q`2&)Tpx+&TQYU}u(e;}QT zK+E|zSINkD**#9T8)C{z7nU;o-eTq;Z%oD*iS- z{Ax!>sr}*wFynZZlYqLz4o_paU-V4!+gX=KkQKQ#erS}q8^ZyZMk3M(Up5%7{%F&Y zqSQ^Y6ov=+V6!M;`xagqevI`5Bd@A2)L+Q_`yB-H>~e*m@r*uQP2$%i7zg2kbH-z; z?sll-gz@$4&4mV80?dE1lrK9hJO>kHRddK#q2Kyn@{dRvG%o&M0q_pHRA-7k>1XxM zPAhm3-8bD~&YMRDPOgDvgoDq!_cPa~?CpSRf6L^=b0j@Oc;uE7Vlvy~TdqZ5b`(K< z6Bp^Hf+yOH{;pc!833Mpki;Pk_L|Oiq)$0L@>5(E&x1IEtNgH`x)!Tuf~c++a`FHK z?g4z+#?48N_4z`2H{~RV`4yYwq1yPb6HhS*0}g-Ft1{57#!IQ#889I!1XULD(WxdU%ioJuMMVB?|9CyWajyuW3G zlcT<+e^8hegTOi#S7+$=OjA(KzGqyV2Um)YEL-pxw@RUJJwzdG&#~BBaboi>J51R9 zqJL-n<<&-r^kbUA)zc~jni6fwvwBZmns+q`DqKlrEwRfyn=)|EDx>@SMB+-E%f2-{ zVUh#{BCGFyS{D6s1rz)X($REj6odE80_&XtK2z0b6+%zc0Z?$%KTZtk-^W7cO*$G2 zo&^a~JCP58A~fj(O*)eO&;e$Sis5o%lgTBYGR1c0dHj61kx^)!d4F?VJxz^BAl{|2 zH`tiYX|bw*Z1FZ|Hzki!2*i!8Q1uC93Xrd^zX3vX3-R$K&ozI1o^v;bVAHZCVab`FwXW!EUHz}L5Yl?hoDB_jUpwYCs#(@(rDYwy z^aE5Y79q*~{q^diRJc(Mdg+F+cRn`>g4^<@IAbSPU<*R!PdPk2Jeld0}kgBPCnu(N8dJ8fQ2@$@} z!X-Sz&zY=%Dt841s4^8sDd&_bHej)UXVp3LSDi59vWm*Hme>}$%ur$(cZ;NT^{FO! zp`v838opFHp)4G z;mRLeMU`p5mtAiZylF##0ZC@k#2A!LcvSt87xRo1%Cts{D82zvbfK%f=pX7sMiqJH zU?VUE=w8G)Q0lJ~iSJF&XI+*V6jXzxhi=X=pS--Vk<)E)9KeNxpJN{frEo)r@4uOI z-S&>Q$ZV7O3`130peW;3@AV5`5X;H`xbT=TvPB1C{H}h1WVNlN7Qz#WCqt; zC9qK|+$0*7fN&_wII!5~%3*#HZtjyJlNE)e#gulAbMYOPp?)}G$C}tFqQPVDS#Sn2 z4qv~WLbYmQ(L8JZvKJ1KZyLKDl>kqVFK_JSg6>e@gpzau=wfNVfdH}QgCW0n|15xU zYI<_gpiVmu_O~SaHm0!2=jXkKF$M_PpPR)<1-L_Snet#Q_Iy{6j{U7Mh~~vhQ(I@bLPy!3(9(H4O$jM ztZpjQk}pjQ;OT8WaaNM$+1Mmhe8}}C3UO8)mlSWWtI1-QlZr79PEi|4ONP=)2EQo` z1s5o@?cMQEFQ=qIDG3mB0iQQ@cJjAp_MIU(E$hR2H>%4h|Nd`&Hp}9wrso>I?vFs% zMw5ap&3VeoZy_q&7$O^oV@sUQuU?l@T`*$HN!n&*gkKu6Z~_iXOiH9);r zNn7lOnu^Lnn%fJwUS;j{eK#DB>ch&qr1PI4HN;N#l8{LUUkDi6_>`ABiutYJZ$FN5 z*rq&S1&iVyKU5ryNu;O|acr`RyO(x~%jsLfweQIWnW{pe<}pnT|4EmoSw>QC1JfsN zI;SKDJ%Nq&Nya>WG=huq^ZI8?%X{)crfQI=d5ll0>{``lI~TWtZ_7)71-@gf0$)xN z6a(ItQEMfLQT{!%3nRY>(?f6&kk^)jv#bZoA#_n&r06<;8FVsz?XWPEE9KtL&zd$; z?!)jErzo$O^LeLQ$<$82be#uK0&I9G$O)$8HE70ZJgoEgH_QPQ{vNifl&|nkEz6Nf zD66g!^0bo0u(wi-ur+@nEfJoT7VeMr9mY2o&;-*tteVip03TUWRSDZy<|N9q;^ppu zyxcV=c+fBBSevc7NJh!*aersOGo!$@AFiQJmpiI$7F;F4u4mZ^l_9<-M$p9KyHfy) zQ1LmiU=Xz}eO3DD+;r6V^2RaTp90WL^O$I=?T?+tU;j{&!({AQm?KSoHq2285lM!B zofHV0bTgTTqAV6Ob;ZBE!t%0S2_k>20uYws@AhGoW^uXc`!i|auJ+wBI#r4DPD2pXXk;;jF3f5#od zyYx7HWUT*;V>AmDh0k=@cn}1W)f5qdl9*0|;*2N_AykLB>Kgd*0)pQ%U4`O@h=YKk zx}dTP5wAivbpDJ&{3bn{+N`s7BQdu#Z${;#JOn-_U{iR|kQ6bF_v|)Xcfo1$wg)5mvMoh1cTWrJ?MM z`WlszT1c(R!V9T0(RXW&hX!DiU9-li@$g0%n-QljJ>1=HkE8_h$uN^Zw_VEBC zQ;F8A4weiEY@a$%63Bjrw-^&+byZO)O>}SW2Zs0+t9KYs+kgXJXBZ@DRw5-^Edt5> zAs86WAH$!by}w57OzCog>t7t%%l?e8f|N4{k|j_o!>la9K3^)TfD%aiPj0=E<5a(@P zX0#_R8{P=lUoAy&jm_yLfYOF(Zac=^(J5?FS#^$qAp8JqY0?(JGQ_-Q2VjtC9$>JR zi|wFPTtlE=^pua@03&a?QSy$IK1MaAU^6Mcv#@zfosTvTTAjqMx}>yWE%GUyNnI!E zGeY!dC((i6JUTq`{<;kIm%7c@%px!~io6|DhXhMgIhkYunLGn=vi@n$6y)b`c2Edb z`sVGNVn`8dM?Gtj)fp-D9FZ-X&F+^`%tl5+!Hg7+3y$Wk~!!^!!2&` zyPQ$o=Dm;m;RE$WHN3Y3VbjBGkzJu_w?=XJ(UPn4^>|*D4vxxV9s&~h97^3q=B<{b z(>JvhD_lnf$(;SHu_{}>#n^gX%`z?Et|?^zZ=GEvq6;;q2(g$rBc8*%qzmSrnv+UE)(=1lD<>hS5IJWSy52YP25gloN9zH-Fr zxNI(qIkY!c*!TLNFBf*_APShsW{r9NJ{4VT|5usbF2lK8%eSmE9?@xuVGjl&BknzI zHG`#uw#a{!1H0L4yKIb!goG^dIKcykd{>=C;WGqsyvHnM#?<9&a{vVP?l7jTI*SkD z@(jt_V^kiu(LY`5wi){IEQN(6Nh(6cgwxx{>_Wjo)QIP3l^?J!-0wg~Vg1??ooY8@ zw|&y7q~hwDRP1?eoOx|_4_6~;yHLPBL}&m5&07T+{$KPLQI%IsDe-da?=5Lrvtn-O z1rZIm3@Y5pw4UFP=m8lyZA@4Fgf3#o751Kv30aRLz91jHV_ALd0%?%@QvS_lBr2zL z000Qk18(8*b|<*PlzIeNC$k&yPTV75=bXTer{nmLm09>b~X^*7fSlAP+_b00005pfbQDs+GP4Z#?LfWVB`B z*7X1ozJ)<`T~RR6Wp9)er!aM6SdYiyOa2aYtkmLNbA`MU6=N&==!wdpFeoTlhnMm~ zcrHUoxs2;*AyMHUNvb#N*;W)T%~4T_c;s{gA+@DJ=P;r>!q~mPRPwy?Oy=|RC=<=q z25erO00000tuM)oDXd?Przi2*UepIw*ClV;r*l29=8v|zQheWo&j0`b00j9!00000 H00000%M8b% literal 18404 zcmbTdW0)mdwl%yHm3E~nZQHhO+qP}H(zdNi+cqj~+kWeGpWEFR&-detUwf|^u{1U2 zSP=^1A|mUe0D!8nfSigP8{W*H-v|&OSwIvMpkpAsll$lP-A&z<1d2ZzuGz!QZQfn9 z5Ut|aqx)`ND&A|Jb+vdIKXpHGMBscsA3g6suO7Zex>vfsz6QQto}ll(uD*KrZaWme zaQv~pULU9KPQTvotM;TBw!~|`ys18SzMhsbzj}&YOlOJ#koZynB!N@_Q7|1q97+$6 zfZPK_MD7PHrU(Fy`~NRixX%(U4|`xtH9l>C{zb@@N94#M{Eg(09m7^Zs-k{H3;YRg z;`M)yCAkb&dfaZT{JqYQ&&HG635NYkp5qJmJwV3ZPJA)+iA_e|m2RJzYvaM|b02xO z^~<5qpJ@E&QPVkBEQ4vg6Nn<=irm$iEgtG?)vrYKF}Ecf!#*2l5<>a?MnD8~H9={b zY6&bxfL*rqu4@0}zzBW5bvq;G;4u6rS8n2XDxoGMt9C~r2Q>Lm*O<)?K3Ybyi=6)m z7-fh;YN`FsD#qiFnXw^K5C;QSKV$l@gtCl{wlgCIH~DW5@~*U+4(2F5A=eoxhf2hg zvU(*FRXBN`N4m-qgHL^`om|GuYik=(8T3zQv|$FBK$)4f!&{Y|GJq)=rGU0L={@L* zKI`9tD8VGb-n|)tA1Gtrrzf-1FcyBX@;V#ez$p4L{`4iQGxMAvzFN{L0hp3f{neF1 zoL)ZPLAN34<^jmp*^&1AUnWF0)gC1CrE7L#sl=3K}Zw@8PDix7>dj4xnHTrL8 zFf-)I6xSmf|IuprUwtzX$9AiGq^WQGJx8^_BoBvrp888ue>MoFICbCGCGq$CeE? z%h8Ku&9~;VAJ?b;fenQH&p>}x^v_xBIiXDYcB{Lri=KC<8`A+e&W2fkxn-6LJObP% zi$ogz$jy-hRD*vUdd5xnE$KhLf3kw701JQ`IHy18{-AQzn`XD&;GS|$=5R0l*OvXI zj=!{&jQ(!NYdHTc3tb$Cyd;k0w!Gm{9h;ruHo9G5m-u0muG0AOxUWPuyR>l-V|83N zOB%*EHhE6pe=r7e(rkX7@g_)*B)k(79AaGhiFFu!hD4WXo~SpN$P>ceL$PJ(U&H*_ z?6RJJYGM-HQ*?#)@>doyQX9kN#z-!s8TMZ8JB-dX#wee)LAv>Q(T6Uh0*^Md3I?pP ztJH`W_P3@=xFJQUcQ^4zfI3jZw-FmgAift5t+pz2e44qIsPhcrMB47>BFn!m?Vn5g zd&~*OF`Uh}8F^ZIfFUz4YIz+c3YTFF?Ob1NAsKv5X0KA(@#p-f)1_gX{p)ui)#rKF1#{NLHG2L!4*H0uAhs@%uhFFWkjI1X^iE#2KjJTR zZ20^@B3x>FSP4`7$;!gherS$zIj^6}@y`BVHcPiLsRHiPbDo4k#MJ`%*eUt`-X|EjWu~tX3IZPyML#q<)>UlqCbQ4! zpOXHi+5hfi4nxwZY5NC3s8O{}^aGvbo)@C{uK#6r|6vdRe(Il*Bf3`=`9FR2pUa_G zw2HHLD!!TwBa1_eOr%ld?}DBSqKYtje|t_;U2uW>Saih6HV^>F>on8}bHp$zZ8qD1 z`9C!zqrdGg*R}tz5a94D)5~kWxq_qJjnN8rXQ*sHOh;xpZ9wN2LBC{GC7BNv3?J8H z2sb%p>RQ6j21}cTtBMEJhJ!_O+(W~-Bj3+$QyiNQdP>YhtmSg#8U9JpX}9rc-=eDp zC2QblzPV2iHfj!t2!9zf?HVm_CN~MhXj$BxP*E#;a;x? zqI|u`2WVLZ-MAzg(Xf4hXchgLxEO0`M>Kf_Z^j3QA6?m9G*xj|*K&!?I3!BHB&0j6Z> zYsgcN<BXBZ!*qVm6$m18d*Dt+CR^iX zej^aa$fMZya3~8L!zx?F%P!4jPx`o`{kk`e%zq9*S>G-mhQI}lDoO>++j>+rQ;}w` z{K*2aO(_ZHN>l;f@2WNOUo)~HwtNR@jDW_UF?)sB@cG9~4n@0E)8(x?LP&3t6oy~( zSA~lYHBc+$9=8|grEXL$n-}mC)U8I{932kwL6*BBSDtV7~X*v+bpaZeXP=5M6 zt@mQ*>b45Z$nu+|lWm!jh2wz9+;KI3eeu5I?y1>Hj7pQLhVTBW{a9w5w5h+c>bzl) zxa_ym9r$k*PJEJz=#hv4@B{+hND4(;WKK0Muy$*+O@?Nz%H->pbLoPAODO|r+ zq>P{G4Kf~@5}4;x(~WlH#0;ob^TlHHwtL!su){&qCLVA%q_!y_cv`u6dvng_50NPK z9||UhS9`h1KZ|mh3Kd;U00gn5xPgZn6EfhOHZ~F2q4vbca7uo0W`PUCr(8vX#;db| zlvh>?`Gve8qJ=L1+Lp-anCL-a&#R;bbp3@YUNI`hEBa&~ssy?vlO}n(y-+`6u7?XK z8IY7#HEFd-sA0Dp!DjMRHh0(J>CyLkn&=O08BweBeT7IFL=Al*uehaUH&t%{0t}zi zB_fZrP>f!oIpLMIeU<2g)udfHaou{p+wTPs`1eW18cdFKG zi>00*X_!n35m;L@+y_C&jey`#<95m{sv~7XFF~YSpgsR~d%t=L%+2Yd=GxfX^9orv zuP7TPjtOAnXUL2j$gIP!CiD{E`te51OIGI%b&sl6p26H|lSk?D!TiY}XT;JF9&6;2 zCW!Wj*~(^9fXQ|}eQ}nRMdjh=PRm9JFILTw%U|f~Pr@JBx;!y3Ex3Hep%h5@TKPL3 z-|tcS+MPB0K_CA+4RAmHSNghC{|i&y8cyB$8ua`JcKa{+f1srPoxhOONsKGp#B=Wo;%2apxU^b$5%bO1_xIOe zNtri{3C$}#`A<-d>t&w1#JzW;4RSGkpz0V#EzNw&_bCrl0dS6HEY=4RczK$QKhbqL zp+DrawD**FT3ZoC?EIjoh5f6L3S}6}g{tu5vuam?_&4nD-{t3qc#0tYwdwYQ*uHxU z7>D9l8%yyKpD3g;HV}p1&{<}+#boYxzo^SmaU6lG& z1XbKdDAq1felyZ$hf?zL+I~}PyM`M>?kwj7V4LlnV5yq_qckSYxu}h#UhntMMVy(S zqB?IV+{|z2V3m&|$(`wn-V~RfVxOXK7-V&ER^dpJk|f)vuB{-wrKT`eSUKGvQ`Z8* z%sI8$JM(w%xa}2Ge*A_*LiuJ{7${KgB*>;hQu#GL-Q8wf0CJ!HbmZdlBc@w*l~2|_1@7V8ml$u;05N_k7bCmg>CI#7I?6g>Ma{%@H7FIZ<1{Et#2 zdj3DaUwU`jUzJ?NeJ{^h649$B0X2g`6FDD8ang~V7gBH#pxou|5?>!$Ay_3WMPz#e zejGVPrqh*{DKo6Nc7vMk7Q{hj4_SZRi8Cftj~Ro4K;f^liEV@RKkBE7w~~LLzQ_k_ z)JcboaK1}Frl&aH>iM?j+8{Po zZgobBKD`YRK=$AOdE7Zu`E=OD?sY(jq2I674+8wWas`Y;Qn;bJUBRY2@RVtcypGLp zt>$I4(^QwTa-cfVjm@JBb$ku7rmzHj(CN?}f0Qs|cwY1mY#dKfda|$tVrM9) zM|laaD?_;%1Z!B0s95Wy4gkS%yM*NhZRx#f3SX+g0EW=+RG)He{D{Tthde98AE6yU z`oIi6(dlAC;d_k$C?2GX;9vzmkYT6PGrzUVuIQriP+oBF)WpI^=UBcR0FQ>jQm-IV zf6DGVQ_7yOEpdzOdDF&v-hGmJXq5iV@-|5O8chL30jL(P#XoEKsqWb zGs#nPW1mzF^-!+SxgWf#k-3Gza%5u@_WaFTtrf zgK4{M(SycRfN_O^>F+V(^HrC;ipi8Fz8B~sYZ;bwt5@e@J4TI9%M&Dfv>6DL@0f_D zRzcp*efYkuO4~E|9EjL_8F54Q{1x_f2It!BtLVZIkfUWr4mmJ~&UZu$D>nQA$Vy+i z6of-wdvmLcJ1~e}Lhdz#$YlzuB8$AN;ev1)D;V9Xl;%Hh#g%WEO|Gi>>#2ftm23&| z39~33d49aSW?QpLb9uq&7d7yB*)}pwD~JDhgP|Lhi;3HK!ssaLUE2^|Tm=m+D%E?dFwzgLnVn^i=U5G)g`>{E^L$ zmA>gXkS^q1QS|y9WEvIDcSMSaoSU?wsXeXy_~nlLxfcapFP(|kzIN~3hoeM9>2Uyj z5#(5e+`ebi^k_nfYLhp|i<&x>oy zA7(WB#|Y`~A^NrzP#j4EfSY{Y(SYB=;t4%^*lzwB{5PaC{2kR4Vn}ei830t0zoITa zcAWR?ByOkQ4N9Ffi6rSYT0~Sou3G8uo>O@$#5$2NxeR0U=mw3)RYZ(5*lNm<9m{>wFc!b>x%EKj|^_*pWL(e{~`}hl*ts_1zj} zzVcNb2~7cm;{-i8W-1uMiy8J2B>M~a(RiHj9VNB^>O9D+e%u=m7|{SRg|9b#T_CE@ zm*tSmsWAeaZMNRdA{)Jc+P^_lJ%U%~-~3l_a=|f#lcU9740pjll899_cs}Fq>_6GP zKV|=N@Z;b3hxJ1%i2^(A{tpc<@K3pL&?FHF+P_2W5F7jdro9|7oBl$6z0pgigMV{O ze^6cpCFj3svwzW0|77km$}AhmeLwvE%{I_${)_(lcPd{N`3E=Fj1C|o|C<~8BUD%a zztU3>0Py*~d%*|H&;tMvQl@~;!nLSIw%Wpwf5SqJ+_?;Dc1S(qjvnT9>Pi`vXfNIc zDjW=fS`?Fa!CXU>v3?-h)NWlm0; z3VSl6NG4f5J8wQGA-S2*^Xu~ECUq4yq<3c-Srvc7Jymh&_))cf(&+48 z1+336q>a-YCi+Cl5botl5*P(<=qy!+2yC_0O`|I11GMM8*36s(f_r-ZgfBj@f+-Cf zL*x%%yvE`sM@C^9I6BGJuktYJ_R34cProg!sBh* zyUAmjcDn54CtepKO)VXhiOT}uq9G2Nkfc8S5x7V2C@yzL7Z3*3+;j^hG4^e2XO!at z&ypVLJjrN5rbQMQ9iDV8MI_`tzO&eKeHvHYs&8rA8lzg2-dQ1>8I+2Nm!0ok|C46m zb6+cekYGqpp-WrHj4gJ+uluI=wqu~xkMj%h0iUI}fHmxikL#X_c3$nv&$KYBA zf1sT;*I`KiI(2Fru#&gNOcu7LJ^E(qcW7n9^O(^T|B-li+I?_1ZO_ulvpPJKkxzF^ zjx49)L7Lle{hbe9zNf=(1bt&xbEr7~v$3ICXxJ`yT{LKzJRjN zhWWRT7zJsVv(0a=_p8y2AB7xo1J?im5T|}oW1U($VOc7MV=AV%*O|lGqb=G_VrJeK zW?Du}b%LHy`tD`%$vP-YbEf{}rY)qBNDt=hfN;&aNg9r4NhG#NIvD*#$^8GBvqhq7aPJNf8hC{m-7Tt?eW+cUm&P?jB<#Gi&aS z7hst4e$Q*it?Xpi8V@RT=qgCP#WraF^mZJTNZGfX5`;EE$?zY2#9#@_ZlN+qRlHh= z4OPW+U6np(StdFVg#sWyw`zp-$@PSfd05Qr?-vNPYjlV2JUEM#8f=N^pg|^%5L2i~ z#QIX#Doc^xJ@qZ(T(QCuU;jK606DQ#76|Qrm6!%|6ehMaI9xIZREjMiPutEnR7EAO za2qM&X-s2Lw%O<(+?@DGC^eWVk4GyW5!`kr1<@oKADrxb9TmQ&$F=1p2iGKY)kZN3} z7DwcKtIx#%?6jmHD0@I9pO>`RzT44!RCIWi<<}_Mt;g5~D-}3c_{WiOamI{F_=>;p9Av!eQK*e}A`d(~oulvf8ScFnsO@2wsWx@g#OC zqnksA$ngOWVx=w3L9G@G0RX5fh^bRT;RC2}VA8|h3%n70z}Y=zHhGCBZNKElv-yPW=qG0e>zazKn$N1~}AU!%R7EYw(nRG~&!ltUiRVp#8YsHWYQfCyR&-)>R=)1kjHOp@J!Ys$W(S3aY&cUB^dv&HLCN}S4g+C=K25LTH>B;#I z?39#OZa3}*=;9YdyOb!4?-B0{c$DR@lYpQL`_)E-dGJWo1gAl)o4a+9Z9V`1mhg@R zEw-Gq(3zFNL)J>yIxm&{`WF`n#aWwB z3{U0$3&%`VgP0+S?%tF`?=ZaG$f;;Vy#9gT9~2#nOH08lgy*3?HLf9%^hAtp=OY*0 zkP)~`uD_V20OaLEZ}Kn&QWx)HBszE;b1X&s4dfGVi(ww$ zC-fM|8(+F%0tiI3$d%)})6E22SzC{@||#$qxu2%WiDe?vbqxN7G<+snzy zEIk3a9O^yOQzBZ=JCdOS|76B-V>zUwU@W&+p&)Y~m*feb<+znke>f)G)0l48HuhLp zV3|8-in*DypFj1Vl?-x3DGUBVRJE)!MU7Dwr=GRl8vwAHj~93Nb7tsZcH`%NsfLhi zx;W?Mf^;ug>X7!O$e8@nXO}r#_F6uSk~IKuS?N1xO6>9?ge$7O&-I&MZpfQB1kMi# z!CasM7rp#i9qgOjJSa|>-uYhhLXJ0>X~0rP53*=99QNF@iyO-_JV*CcW>t82zs^XEP^q8iDl#g0Q`(h>o?mSjLN=(kC%jYV7AX6$h^CF!!w>SW2ImYgc z1vQ!OKS_u$+oyK_Q2*v~#Si7xbw8mB%`qgur++~;t`SLVhh}JMTmp?{-pTZxC`?e2 zi8Q%qcFGoK8{{^CrZUYmADHU;eB4{pYgV?I9F4kj;yI9-De=BkcLY zj-{NmnyHr{HZb*KeuWMWC^L3WBN7~d6!u{MXn&~`Xr(zqtz1HCEYcV9S2A%)(`0C6 zTMotGt3viroAWd6pVWkSEz0RYAYz-f!V$?iaI@vkmKV3lH?!yRXdLhg^I zVl!SP6!(@=jUiZB5wM6EM$sAS-HpdRY@|O7ap8mdO+Pomttk08l4oyspCSi?Mvn4{ zKFpxGvRL+X!x=fK4!*1u2Ry<&TZPdR$nhQ*IJvI&*@Z&71C>>c7`LTsU-)oMp`5;< z`rgaXC05lC2-DBToqYEpyj|x3-|G2BZd}0a%)G(kq>|06==q)Ew6sS&}8o0|(U56}M$) z2PDh&eU&Y9un)fW+oX*&4}QTa5?-bzxONOFYM&AmJmwXbkE58*q?WF5As}&fgNb3T zb9w)C7%&IS+MavTckx!yNE(aQRs$A?p6nf=_(}lJ-E8sAh6aZImtaEV4_V=1`aqzN&qmo}XTJjFEKIt~>(@Nn)TS{4@iw36~$i_XT0Jr|qa=W6mG*;qbneTbEM@0)4ZBDCKSkfGVLp zZaFQNciQ;O!WGEw#9HOOOjtq60Ch5{m)V{^=bP~fAaBNi+cfzz@K&FyfMuj=q8>8h zM&#toKsm=1;jCxnz88KQ2B@dfqe@4*hB5Q=lvCiwW16J5jc94)H=d9m^_`ND9~5~t*D&}# z6UNwfQB`TH9>)qydMr*WwHbSaI2k(ag%LC+&v&#y^p5^=`D)QMM@SNWR-4h_#Ks!C z(r2_McKwbfr(7Yk0eIRKyhzuTl=)QUPdkH&$#CzFRvU54o3zEU?aP~(nYAV!M+Pjd z;IHID*7ZSjzpF&X8d1}qZa#0aWqf+FMVYM=-=kz&X81)1VhTK35OO%CDphw5D0|20 ze^<3MmfLGUMtmcF)XEI1eGQf3=CD35ely^nrHrE;Yx}u6-LE(^R)EopPxc|~Ej0jn z3;3jXT9EdM)Gz?SawWX)g=#G!+_POqbltoOnP?s_+#EO}$Gp`+#;qt?s8!vcMtwuG zonLSgJmG=f3o+(9r`IE^TmmgiIzlieTJ zm3X%RX%6{8!Nw3~f08X{EJ6#v3(4R_diW>Q$Uj3(;nu6n}QM4&%5h-`A zX~PfvX~mX;h}W*+@=gKj2Q==Y+kNHV2xSF{GrVB_>@_s1yYuXBL~PuCPnB66_3HAr zG9+^KTu(=)d5YAH1inH9!(Hr8zKX$eJzwx)yss;vQQom>Q`r&*K@MZpNHr%o`&bu* zZeSBgWr*DDuH4;<UiFO!lRg~*#7x}!{qvO@$JU>O*GKIZJ-hYW2wD712G#8Uh z`7+J5`wgW=CHsU?B(2S=fyC9%Wfy*ChY9G1Rry0H;PcKO-1PpjS}nVq@5fz@#z^GU z;g>$U3m{sOzqzQ&QhGD9+8d2s>5P#~NPb?g zXjqG+rk5W|^z?q?F?D(s;czxIY|1=T5|7#OdehSpUjrC6-!NnRb#;-mI z4wki13G-Hj;=|ug)t#)7EAE5WHzaQ`E)xP-o4q!*tV4`#v6)IA4KEm+TYPUa;s@c* z0}0|oJk5UyFWydXU?m+YKfjkL<2>+tgOZ)fSy)X7k8g9_M}}biBSd#A|2+09kivxEjkLTVLdRRYj^y$Vs!;rV7|hMXLmbUjW}<}Z z_EgGlAN*~VanGtzP8zi#RJzwi`e7<&4eCV1#tkea?uy2yh41K4pON-#V2n% z72uIaq=)Zz5W{9YwT?l(orXacbH)W2#%k`a(f%APTY~Cw2~LGN@tMpJUDkWMUJ9K+ zy~VAeRC7eWKn|7PL+U&)sbxee)w&`dZ}N!P*o~I&XBy2O*>;=5K6Ii= zXCsaCEcv-f=cxz7)_`(~R@?ZgGnqCQ@JsG{S3+8QCU@jpnu3unprJO#BHAin91C?b zrUdJpWOdiVXdt2Tm`{OK9$;|hy00fc0Ej{EtMJwJ=OsXcqr;J*4vk$>cT1qaH%;O* zQpfyx%XBqCk{yTKKlIkrSIM>?gO?Y4ew*JuX~p8J5o9i<^}DOnRm{rr0(Klj54OYuIpLj5j7XBv9P^*I-=wFL zA-W44OO2sPIz|*M#du6@!k{)zxG}4))R06ki`KU3iVFdznKy#2Wv7eNSLD@WugC>U z(n+YpbRxPu1(8BH!T`TGL0y&aF#P(fk{)=%ZWW-c+xl&Z>_M%K+X2~H?eO!ydre?r zcq{iU(44ncnsr;Gpu}GIjdP;ih(~!VgwP|?==uHUR;8!DTGYmRe>n^ zna9Wty&TrtjsxJ~-ty_F@lc5T(funbdPBFll)D=#991LeFu2BJ11GY%SHVrt6CBCr z`IF7g+YH*!Pi6CUy12$1$w?E~i)|uya0Nc21jg#mA@6rj3*l!FsmA6ZV!F;KKgN&i zw;N4{$3^LC!?q9Qvp7xOmkY-rN8&T;aeDgb?YX*BVa~)h$A}~G2Ph?=*hOxS&4g?i z;?c>2jVXk5K9EHk+^52bDIRbES~zt3lzd|F(RFJlv8P@i1I(Svdal!fKf7SRHo#j- zYfn3+KTa@%gk--(r_8qoAb4&3CLm;2s)ph2H!)^+mD$e01Jk+=KMNSrk?hd8aOQJW z-V&D1&%WM+J26UCddqN-_WY=*(1+(*Vzn*|Iax&LF7se3N_RvYI?>P>fuhm!|1&0@86ICUSc|f z<0Or3tr+~h)@vt@MLs-I)Eo~2gQ%pMmJ2I6=V_omBCtL?WF-?{@o|FLBnzrm-kAr- zI#I^{6ND{K=4Yx}v1LD}FS?K^RF5$ve?`%3o;o_N#Ose}v|l}W;uES*Eu&`?nl$n{ z`F@iX?3w+u4mU4Q60L>DoipE|v`aI&yNK%VG zm*c0_TM{za8GJc^gR$Jo<|Qi)#G$rMmC6@;LJoS*I?mKit&92$;;6eiLsLk}IqTCP zksjRlAyDK3MHrU2c|>V+dTw)TahvJh{oH$4(yW36Ik2d#2V=JvB$I+o`#LPPI1&bc z#g_)$fwRjIdGSY~SU-D}cVuyKj`hS1VPH$OgkQ+jH@gFA>iK6l$KMK{<_x(%sbkWY>5xU)cRyfqUE=_Ob? zwcRDGQ4}oa9r!87hAjTlPV)-PsF&FUrV$RQTUwRGd)}XZ4FJn}WVU_FkOu+n3#x^{ z7xd6nlL1*fYA>)FXp((S@xm?#{i}-m@7MRpUpL*pHAEZQTgp!DpTAj^&sp%cZr%8~ zRx;>=lt!dWDqS|sf8BSeQ{_%A+~!zl+YKW_Mc&S~J^$&5!S{=>Mp;bM>9s#u36ImH zfGbig+}I}{7|Vdw1ji{J6Jkrr_UFu)Y`zajIMsdkKC+S(!YS|1M^aoL1UC~t>$cc7 zqzT3lb?y|^qnNKX>ke3Wcw!`b_1OS5hE`&Zxr+Thl>!6|Shl>!rPb zmq%+9c&Y@$M%tW1Yy8yG)vx85U2Ri0K;(CR*pJs&|30BRQE=sRcO#5WZ+TO#5=Zj| zarH>|H)pOzXzgjk06(JO$zB=5T*!Ij#a;b>GH*EIE+LUo;bmGYvKN40Z17<7G$83eG|VkZE)&wi zGlg%>8Zr^h6CB_A7X2={iCg*RxEaWRD#o!?Jxw^Lw~W#K^BLa^SfWku)+dxa3MFG% z){xqr{D;Wx)+AXrLrfYBP=)Yj60*`&apQ-qAi}D%N)s^@{tG(4|BzYL+3Nf#XtkCN zeTyws`%?CaDMx?`RD}xaw2Xgr5djbAI5_dZ@vc&x4ZEEuE|36WSiUE(F~k^PO}BDy zrfAB>6YCJ&ORbmNndk%1NeRXl$5AQx96xGDbvZ-1#TG6l3k_LCR%owAv#>YvS@?>D zVKn2`i+IDD8MjSC-1vF^WL^|>;w}XS$8?qm8UF%zPTWfi9_3@RFj=!X+gcZG`Kx+8 z@GN<7TF+8Z`~?{{@3SG4pKkuq_E(e}{L!9M8%eehz63*{t;a`(%h80$=T_NQ!TevT{3$u-7>R z@H2uS_e`Mi;}xXG)9nb0t`}3z8!WNaD2ld3&o;$x{+ldv8ZuJT)QljsTQY*1Z?2i| z6;}K1?Ho3tgXyXoeg=eY*LjW<&2HWSvb%)nkh7h?>+BUJ^$Qd8#T*_krj@+l?DS8y zieFWHWhbVa=XRZABMqlb%|SuS&ve;h$M^CPxj@-5unr>|T;i=2GW*a?h(+6i6x}KC z8<%Ty-v?@_H_oeLv%4=mSdXcNmwZ8^2bD8EEXiwObnN{+@iF*d@oSn_LOK0k{Y=<9 zNCS-=@%LqE%AVyl97=)#0DMY2FlwAx-W=8+r-KZW)Va0^XB4>8xC%v?ZcKa3y&C)_ z;c1p2%l=bvEB-i-fzS(tE_;{yGIbBi6(6|Y&U6N{nPi-JPv=eWxnt#AlQhpVDb)62 z+@y76N{po*I`-a5HM+$M3gJNGa_~cYP@ls!n1SvbC&g*jVQJh|E+@L8 zmBi)UvP_cEz~UltMSDapJ21efM(n6Y4<@XZ-^{CQkMlL8GtN#vm_1vwmOQLm8%t$} zs1LpKJ0A>`{m5=v6?WjV`Oj1PP*cq^^?hCz1QG-rmB%#n03f@g<70U39oKilATS8b z15sbid_#Ih(5Y+yQIx=og$zqT+4JsQGTyyC zW&RA7VcM5nG3GJBMgpvP>8-D58Rurkw7@^4NeN0)Gxk8Osh;#~sD13ytekK09985y zJsSMn3r_u-FbF7BqyPXS7%z2j9KL7mn?NKZ_GQ7OS^|Kw3r8!SXAkw%pR=!Xsn!8# zT8nl%6v*sW#WJj29RSvK|KUH*q5jTz7NTSE$-@-X=yRYt^ z5!#pk-3lVA^r#p-3nk|+HumxQ?mJjNZ1>&#=``_5jX6EsfMxS?WUJb`yhn4f_bc=(Ip6C+~u2NmfLRwUcCXid4E{Ul%tGDzx(?2Ok#0x zwJq4_H0jshK9zy3U8=pauK^GMnUgx*P@Hxsx0JffdVGk;vZ1#rp3n)~j=f#twYRZf z;)pOk0066=^6RVxj+CGUYpGCUI>WdUwl^qUaz2y&3O2pC%dV#VJ-{48>zxGo*eCO{ z;H^+1!lpDs`WP0YTsH*Lw&xw&)=GX?4!9aZEZHg;kshSRMZ%yG{ybZ6-3?R0yr0$M z)<}Bi>M*;wTy3P=T|t9X+;I`nZW%@hw023W7iiN|8H3t*9J|c1@`cZQ&`N+eyByc+ zQPZUP7gJ{- z;O@a`tgpd%zfd!+V;_V-;-Pl!%zFGZt9`#)y&m{Ipfl;gd2FPYow1t>QQFR5Y#gxXN?uGf zlUakKW$&p+JL}fm!bxtQ*F%{X*BoF~ewT1mzK4n!PF2fkR{V1xk~if_uIb|$-uA;p zG65YP1>0)tc2~I@gm)8&e0$&%nVYt32SSC2x*L{Yl=Y03=Wg=JKz!NFu}Xb}0R*c= z0T4wACXUH8sACfDDh}iB2b9=gxcFik4q6VQuU?Pw)53O|#CICrUU3DR-}i$YX^e((T7sye@` z7w;=5FJvuH#v~Q-dl0QL{&0!vW^>(W#p7vE%n6rJZpu=&<4JgfQ|JQ{q04{~z2=7q zZV_4v19qLRsy=6-x=U5pv--lDdT`V+H;*vb2iS~5gUWE1eB)0waClV<~O&P+G(9>qK+SZKjT0$-;_)Kjhy9>3mlgDq03_v9fKqJzm_=&QQzS_g7K6k+^SjGo8{5|CS@PZ&>zO}&f)j?GZP&%%!{3!xk@fcz(PrblLWMtW z@LViBSEL=?{=(8NbJv`=qF>{at}R#@1YIH!?7j3o!p?10qlAM!xxJBQ#L`n+N=Up)eqnV77z6rHWLdOO1nAzd+RT)<*)2>O z0%#mtlpT1$r65aLQ>o*IHCTVAi*>lbblp%(BV8!$%h`%@Xlz`nJOF3 zHXQ3lyP?)6!FdemRp)IE6oYZDl}x&gySAC*GswW@X}ND0ACM8<*kN-l8AZ__;=3~@ z$i*%kev|+CUZOg#G-Szy$1_Ym`6qET)~Ru&m6PZ}G3MC}3Tb_%AgcCa@=7|wTuIvi zgm6`Ka`mhL0;@{k2EKMgNINlwWcU2*56zu^31fCPm)vd3fri3!d9eqA>$wa=C@&wHr78k-jU7a~T$VpQI}0kg={c%Y9^28c~1(5deizT7_$guMGL&Jk@B z=^sCpsem+pRjvtrrzx~zN@^fdcBTaJ#|k~%QfiwlXzzzy7czcY4&dUqedz& zZaQvn9ZPljQ!+?uo2Ge!Z3xnIhOXBka|QO-zZWT_sti+;$00_d{|(MsFtz#k1=oF6-cE5>6dWZ2Yw zPhLKZIhk@81(XjHLB`AlQ?*Pcgl|;;v{n-d{n*d?bsaVb&|NEu=9Xl9-b(!UBdr6# z0w27z)mes%Kpe8`09cywSTH69h)dD6g91Zt3m|P-FYh#d+G^E8pYhZYk*62 zvu7(AXgt+1%1DlO*_)R>!P4`n78$M*hF3)GFp7FHE!l-OJh98d>rks#$PSYq3xBffIogIfL z&9U_>xO%}cG>&=NgUmD3*Ur!nb7#lH>i_7! zDyLahSdFmbj(7;6h^V8EiEwvWicP~!(*aQLhf)y)GR_2=sa`Q^IS*RE24JU0_Zt4T z4Z{cH^*-92*uG^qicV%mKWM{FN>jrb>nm7#7tMdRn%4a*Wrl|_NLMg&hRY^71(x zU=Jy z==Dj%nP$DFSsV!gNXo28&&1{8fQJ*4(;FtpYMg*{uC1|mKn;eIMz432-cqTbADuUw z+k4!VK*m=W-@IYSR3q6uwKT;HcNmhx6lYZfF6y?G9$36?*<;n;A21WO+fynp=Q~jm zwm1P_OO`9#xSfQrG4_wffnO~& seaYYe004ZyXaE2J78?Kn000000000Cug(lo000000000000000080a-A^-pY diff --git a/ch04/01_main-chapter-code/gpt.py b/ch04/01_main-chapter-code/gpt.py index c08b118..a508786 100644 --- a/ch04/01_main-chapter-code/gpt.py +++ b/ch04/01_main-chapter-code/gpt.py @@ -168,7 +168,7 @@ class TransformerBlock(nn.Module): # Shortcut connection for attention block shortcut = x x = self.norm1(x) - x = self.att(x) + x = self.att(x) # Shape [batch_size, num_tokens, emb_size] x = self.drop_resid(x) x = x + shortcut # Add the original input back @@ -200,7 +200,7 @@ class GPTModel(nn.Module): batch_size, seq_len = in_idx.shape tok_embeds = self.tok_emb(in_idx) pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device)) - x = tok_embeds + pos_embeds + x = tok_embeds + pos_embeds # Shape [batch_size, num_tokens, emb_size] x = self.trf_blocks(x) x = self.final_norm(x) logits = self.out_head(x)