{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt # for making figures\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read in all the words\n",
    "words = open('names.txt', 'r').read().splitlines()\n",
    "words[:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32033"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n",
      "27\n"
     ]
    }
   ],
   "source": [
    "# build the vocabulary of characters and mappings to/from integers\n",
    "chars = sorted(list(set(''.join(words))))\n",
    "stoi = {s:i+1 for i,s in enumerate(chars)}\n",
    "stoi['.'] = 0\n",
    "itos = {i:s for s,i in stoi.items()}\n",
    "vocab_size = len(itos)\n",
    "print(itos)\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([182625, 3]) torch.Size([182625])\n",
      "torch.Size([22655, 3]) torch.Size([22655])\n",
      "torch.Size([22866, 3]) torch.Size([22866])\n"
     ]
    }
   ],
   "source": [
    "# build the dataset\n",
    "block_size = 3 # context length: how many characters do we take to predict the next one?\n",
    "\n",
    "def build_dataset(words):  \n",
    "  X, Y = [], []\n",
    "  \n",
    "  for w in words:\n",
    "    context = [0] * block_size\n",
    "    for ch in w + '.':\n",
    "      ix = stoi[ch]\n",
    "      X.append(context)\n",
    "      Y.append(ix)\n",
    "      context = context[1:] + [ix] # crop and append\n",
    "\n",
    "  X = torch.tensor(X)\n",
    "  Y = torch.tensor(Y)\n",
    "  print(X.shape, Y.shape)\n",
    "  return X, Y\n",
    "\n",
    "import random\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "n1 = int(0.8*len(words))\n",
    "n2 = int(0.9*len(words))\n",
    "\n",
    "Xtr,  Ytr  = build_dataset(words[:n1])     # 80%\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])   # 10%\n",
    "Xte,  Yte  = build_dataset(words[n2:])     # 10%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11897\n"
     ]
    }
   ],
   "source": [
    "# MLP revisited\n",
    "n_embd = 10 # the dimensionality of the character embedding vectors\n",
    "n_hidden = 200 # the number of neurons in the hidden layer of the MLP\n",
    "\n",
    "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n",
    "C  = torch.randn((vocab_size, n_embd),            generator=g)\n",
    "W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5) #* 0.2\n",
    "#b1 = torch.randn(n_hidden,                        generator=g) * 0.01\n",
    "W2 = torch.randn((n_hidden, vocab_size),          generator=g) * 0.01\n",
    "b2 = torch.randn(vocab_size,                      generator=g) * 0\n",
    "\n",
    "# BatchNorm parameters\n",
    "bngain = torch.ones((1, n_hidden))\n",
    "bnbias = torch.zeros((1, n_hidden))\n",
    "bnmean_running = torch.zeros((1, n_hidden))\n",
    "bnstd_running = torch.ones((1, n_hidden))\n",
    "\n",
    "parameters = [C, W1, W2, b2, bngain, bnbias]\n",
    "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n",
    "for p in parameters:\n",
    "  p.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0/ 200000: 3.2342\n",
      "  10000/ 200000: 1.8947\n",
      "  20000/ 200000: 1.8914\n",
      "  30000/ 200000: 1.9489\n",
      "  40000/ 200000: 2.1701\n",
      "  50000/ 200000: 2.0639\n",
      "  60000/ 200000: 2.0728\n",
      "  70000/ 200000: 2.3965\n",
      "  80000/ 200000: 2.4142\n",
      "  90000/ 200000: 2.2257\n",
      " 100000/ 200000: 2.2824\n",
      " 110000/ 200000: 1.8584\n",
      " 120000/ 200000: 2.1613\n",
      " 130000/ 200000: 1.9009\n",
      " 140000/ 200000: 1.8430\n",
      " 150000/ 200000: 2.3324\n",
      " 160000/ 200000: 2.2026\n",
      " 170000/ 200000: 1.6905\n",
      " 180000/ 200000: 1.9502\n",
      " 190000/ 200000: 2.0909\n"
     ]
    }
   ],
   "source": [
    "# same optimization as last time\n",
    "max_steps = 200000\n",
    "batch_size = 32\n",
    "lossi = []\n",
    "\n",
    "for i in range(max_steps):\n",
    "  \n",
    "  # minibatch construct\n",
    "  ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)\n",
    "  Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y\n",
    "  \n",
    "  # forward pass\n",
    "  emb = C[Xb] # embed the characters into vectors\n",
    "  embcat = emb.view(emb.shape[0], -1) # concatenate the vectors\n",
    "  hpreact = embcat @ W1 #+ b1 # hidden layer pre-activation\n",
    "  \n",
    "  #hpreact = bngain * (hpreact - hpreact.mean(0, keepdim=True)) / (hpreact.std(0, keepdim=True)) + bnbias #batch normalisation layer\n",
    "  #----------------\n",
    "  # BatchNorm layer\n",
    "  #----------------\n",
    "  bnmeani = hpreact.mean(0, keepdim=True)\n",
    "  bnstdi = hpreact.std(0, keepdim=True)\n",
    "    \n",
    "  hpreact = bngain * (hpreact - bnmeani) / bnstdi + bnbias\n",
    "    \n",
    "  with torch.no_grad():\n",
    "    bnmean_running = 0.999 * bnmean_running + 0.001 * bnmeani\n",
    "    bnstd_running = 0.999 * bnstd_running + 0.001 * bnstdi\n",
    "  #----------------\n",
    "\n",
    "  h = torch.tanh(hpreact) # hidden layer\n",
    "  logits = h @ W2 + b2 # output layer\n",
    "  loss = F.cross_entropy(logits, Yb) # loss function\n",
    "  \n",
    "  # backward pass\n",
    "  for p in parameters:\n",
    "    p.grad = None\n",
    "  loss.backward()\n",
    "  \n",
    "  # update\n",
    "  lr = 0.1 if i < 100000 else 0.01 # step learning rate decay\n",
    "  for p in parameters:\n",
    "    p.data += -lr * p.grad\n",
    "\n",
    "  # track stats\n",
    "  if i % 10000 == 0: # print every once in a while\n",
    "    print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n",
    "  lossi.append(loss.log10().item())\n",
    "\n",
    "  #break #Add this while experienting so you dont have to print all the steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------------\n",
    "\n",
    "Used for lecture : [00:12:59](https://www.youtube.com/watch?v=P6sfmUTpUmc&t=779s) fixing the saturated tanh "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.hist(h.view(-1).tolist(), 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.hist(hpreact.view(-1).tolist(), 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.figure(figsize=(20,10))\n",
    "#plt.imshow(h.abs() > 0.99, cmap='gray', interpolation='nearest')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "End of Used for lecture : [00:12:59](https://www.youtube.com/watch?v=P6sfmUTpUmc&t=779s) fixing the saturated tanh \n",
    "\n",
    "----------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x21a782d9720>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGdCAYAAADJ6dNTAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUyxJREFUeJzt3Xd4FOXaBvB70xNIIQRSIBA6hJJAkBCUppGAqGBFRUFUVJRjiaIHCygWOMJBLAgcFftR9Ij6qYhC6BDpofcWWhJaEgiQ+n5/hCzZZMvM7uzOu9n7d125rmR3dvadnc3Os295HoMQQoCIiIhIEl56N4CIiIioOgYnREREJBUGJ0RERCQVBidEREQkFQYnREREJBUGJ0RERCQVBidEREQkFQYnREREJBUfvRugREVFBU6cOIHg4GAYDAa9m0NEREQKCCFw/vx5xMTEwMtLeX+IWwQnJ06cQGxsrN7NICIiIjscPXoUTZs2Vby9WwQnwcHBACoPLiQkROfWEBERkRKFhYWIjY01XseVcovgpGooJyQkhMEJERGRm1E7JYMTYomIiEgqDE6IiIhIKgxOiIiISCoMToiIiEgqDE6IiIhIKgxOiIiISCoMToiIiEgqDE6IiIhIKgxOiIiISCoMToiIiEgqDE6IiIhIKgxOiIiISCpuUfjPWT5ddQhHz17EPT1i0T6KBQWJiIhk4NE9J79vPYHP1xxG9pmLejeFiIiIrvDo4ISIiIjkw+CEiIiIpMLgBIDQuwFERERk5NHBicFg0LsJREREVINHByc1CSHw9HebMX3RXr2bQkRE5LEYnAAQV8Z1Nh45h1+yTuD9jH36NoiIiMiDeXRwUnNQp7isQpd2EBER0VUeHZwQERGRfBicEBERkVQYnADgYmIiIiJ5eHRwUnMlMRcWExER6c+u4GTmzJmIi4tDQEAAkpOTsW7dOovbfv755zAYDCY/AQEBdjeYbCuvEFi+9xQKLpbq3RQiIiLVVAcn8+bNQ3p6OiZOnIhNmzYhISEBaWlpyMvLs/iYkJAQnDx50vhz5MgRhxots4JLpSjRedXPF2sOY+TcdRj60Wpd21FXlFdw2I+IyJVUByfTp0/H6NGjMWrUKMTHx2P27NkICgrC3LlzLT7GYDAgKirK+BMZGelQo7UmNLr2nLlQjITX/0LfqUu12aGd/m/LCQDAodNFurajLvhm7RF0nLgQ6w+f1bspREQeQ1VwUlJSgo0bNyI1NfXqDry8kJqaiszMTIuPu3DhApo3b47Y2FgMGTIEO3bssPo8xcXFKCwsNPlxBkPNWSYOTjrJPHgGAHCy4LJjO3KRkrIKLNh2EmcuFOvdFGm9/NN2XC6twNPfbta7KUREHkNVcHL69GmUl5fX6vmIjIxETk6O2ce0a9cOc+fOxS+//IKvv/4aFRUV6NWrF44dO2bxeSZPnozQ0FDjT2xsrJpmkkIzl+7HE99swm0frdG7KUREqpwsuKT7EDo5j9NX66SkpGDEiBFITExE3759MX/+fDRq1Ahz5syx+Jjx48ejoKDA+HP06FGntjFjdx5W7jvl1OeQ0cLtlQFl9tmLOreEiEi5bccKkDJ5CW79cJXeTSEnURWcREREwNvbG7m5uSa35+bmIioqStE+fH190bVrV+zfv9/iNv7+/ggJCTH5cYorwzj/23gMD3y6zmTi49miEjz/wxasO8S5BkREMvk56zgAYHfOeZ1bQs6iKjjx8/NDUlISMjIyjLdVVFQgIyMDKSkpivZRXl6Obdu2ITo6Wl1LXaB6cPLGbzvxv43HcPccy3NpiIiISHuqh3XS09Px8ccf44svvsCuXbswZswYFBUVYdSoUQCAESNGYPz48cbtJ02ahL/++gsHDx7Epk2bcP/99+PIkSN45JFHtDsKjRiqZWU7fIYrXZQQQmDuqkP4+8pkYHKtC8Vldj+2uKwcxWXlGraGiEgbPmofMGzYMJw6dQoTJkxATk4OEhMTsXDhQuMk2ezsbHh5XY15zp07h9GjRyMnJwcNGjRAUlIS1qxZg/j4eO2OgjRVcKkU/j5eCPD1trntsj2nMOm3nQCAw1MGG28/f7kUQz5cjRs6NMbLg3mu1Tpw6gK+XHMYY/q1RlSo+aSFP2w4inH/24qXbmqPR/u0UrX/svIKJL6+CN5eBmyZOADeXsyPTETyUB2cAMDYsWMxduxYs/ctW7bM5O93330X7777rj1P43Tnikr0boJTOJK2pfByKRJe/wvBAT7Y9lqaze2PWOhh+n7DMRw8XYSDKw8xOLHDkA9X40JxGbYeL8BPT1xrdptx/9sKAHh7wW7VwcnZohJcKq3sNblwuQyhQb6ONViFSyXlCPD1MumpJCKqzqNr6+zLu6B3E6Sz/VgBAOD8ZfuHCwCggllVHVI1XLPtyvmoK7YfL0CHCQvxzx+36d0UcmMMa+s+jw5OanL0Da9VpllXEazGrFhd+5a/86RzEhvaMnNp5Sq9eRucmx6AiNwbgxMiD/QPZrwlIokxOKmG/Qhkj+3HC7A/z73yLRSXMrMmEcmLwUk1ZeXyf2ALdxs7cpDsc1fOFZXg5g9WIXX6Cr2bYre5qw/p3QQiIhMMTqp5+IsNejfBqgvFZeg7dRle/Xm73k1xiW/XZaPTa39i3aGz0ubjcLTIY0lZBfIv6rtq7L2Mfbo+P5FadWwKGJnB4EQje3LOO30c/6fNx5F99iK++vtIrfvKK4RJhtu6YPz8bbhYUo6752Si3SsL8X0dnER5/b+XIXHSIuQWmg9y6tYZJSJShsGJBj5ddQhpM1zQrW9hSKeiQuDG6ctx47vLK4dB6ujQzwtX8nrUJcfOXQIArNjrvMKT/12bjeVO3H+VxTtz8eL/tuJyqZy9XETkPuxKwuYJNmfnK9puy9F8vHElQ2pNJWUV8PNxfvx3pqgEB09XJkM75+AQQd0Mayzbn3cB89Zn47G+rRBR39/kvmPn3L9a8/bjBXjpp8qcItUz+DrDI19WDovGhgdi7PVtnPpcRLIquFSK0EDXJTWsq9hz4oDV+09jyMzVZu/LPnMRbV/5A+N+2GJ1H/ZO+Py/LSfselx11TtYdpwowNGzrrsY784pdPjb/In8Sxj8/kr84MBwz03vrcTHKw/heTPn6YMMy5Wz3YWlOTH25Lg5evYi0udlYZeNHCk5FoaoqG4rLa9Q9Hl29OxF7DjhWHJBWfMOzd90DAmv/4V3F+3Vuyluj8GJCsVl5Th9odj49/BP1lrcdkZG5Zvzh43HTG7PK7xsXHHz5m87kfTmIuRZ+TAvLivH/E3Has1JeErj+S2D31+F3u8s1XSf1gycsRIj567D3lx1S3Crr1Z68/ed2HGi0JjG3Zzv1x/FpF93WlzlVHJlhdbWOpaJ1Rke+2oj5m8+jps/WKV3U2zi0JJrFZeVo+fbGRj6kfkva9X1fmcpBr+/yuI8K3f2z/mVvZScZO44BicK7DxR+U0xdfpydH9zsaIehvmbjte67Zes4+jxdoaxm/2TVYdw7mIpPl550GS7l3/ahvTvswAAHy7Zj/Tvt2Dw+ysdPArnOFUtWLPHfpUlBNYfPmf8vajY9gXohR+3Yu7qQ1i937Rq8tZj+Zi17ICq53aF/Isl2KcyYHOm/Xnn8dXfR1BWXmEMJGWfeD3lj91o/+pCrGWlbJd5P2MfzhSVqAryD55i5XelLpWUo6RM/lQXWuKcEwWmLNyNLx/qgaNnKycvLtmdh5G94lTvZ+qfewAA3647iqduMD8mf7m0HN+szQYAjEtrh8W78gAApy/IWaRw5lLXXuDtXXZbcKnU5O9bP7T9DU8P3d5YhOrXfmfltSktV7ZfY/4WFe3Qez727OWV78m3/9iNX540XzSRtOXqzwFZOSOAuFxajg4TFiIsyBdZEwZovn9ZsedEAWespOg1ZYnZ2yuqfbLX+oYq6TirWodO2/+N6dGvNmLh9hy7H7/rZCFe+78ddj/eWQSAb9YegdadEhUaRQpZRznsRaSHI2cqe+rzL5ba2NJ+Mib3ZHCiUKkD2WPzzl/GpZJy47JRQP9vl85QWl6BIR+uMju5tMqJ/EvoP22ZQ8/z+Ncb7X7soPdW4vM1hxVtWz3xW1mFdt+IiorNV3x++Sdtk+sJIfDYV+pfq+3HC6zOgyLSW934miaHy6XlSJuxAv/8Ua5UDQxOFGrz8h92P7bHWxnoMGGhxfuFqLyQ/OPbzfj3X3tr3Gc7inEkcKrpbJH9w0drD57FlmMF+F+NScDVuWriaXmFwNI9eQ7tI7/aUNBFBfNblPhrRw46TvwTMxY7fzb/BQtBkDU7TxTi5g9WocfbGU5oEdFVrIouh4xdedibewHfrZcrySWDE0lsPHIOv245gU9Xqa9zsmS36UW42IFxTyVZbr/6+wjeW1x7NrpWQwha+GbtEYz6bL3i7c8WldjVtan2A/blK6UHZph5/WQwXYMlkPK8C8iawsulWLDtpMMrm1yZgoC0J9PndnUMTuzgjKkf5gKKmu+ZC5fNfxOuOTdld45zV3u8+vN2vLt4Lw6eUrfSBnDdP8If29TPS7lQXGZxQtv54jLc+5+/LSbcA4AfNx5Djpm8IpdLy1WvStKLq8ee5yw/gMe+2uAWRTfrmke+2IAnvtmESVbe00o8asfQYV2x8chZjPl6Y51I2CgbBid2ulTi+jwK/1q42+ztek1msraU11KPgrObWlEhkHU0365Cgf9ZcRBtX/kDaw6cNnt/5sEzVnu2nvthCwa+V7uMQWXV4uVYuc++idVni0rw1d9Haq04MkcI4bL5Iv/+aw/unLXG4msthMAf207iyBnLE6An/7Ebf+7IxR8OTHIm+6w7dBZAZeIwR9hKyleX3TErE39sz9E87xQxOLGLAcAXmYc125+W12tnZE7cnH3O7AXvZMElM1vr69NVhzB05mpsUlh+oLoPllRmhK2amLrjhPkP3S+sTKg1N6O+qtfklyz1WX0FgIe/WI9Xf96O9HlZNref8sdu9Hg7A1//fUTT98KPm46hvEZk+cGS/dhw5Bx+3XLS7GP+3JGDMd9sQt+py2zu/0JxGe6ek2l1MjWRkWQzYrPPOu+zsI4s0lSNwYmdHK1ho4X3M/bV6jWxlMrd3t6VrcfycdtHa8xOkJSxO9fSShx7Jt+dOm8+wdxEZyxFttK8qjpPGbttT/Cds6IyoZ+14Sd7WXoL1ZyQXbXdhmoJ82zJys7HukNnrU6mVstDP9Pdh5xTHUgSDE7skH+xFIWX1K+EsMRa3GDtvumL9iJjl+kFa/IftYd+1h48g6Q3F+O3req/uVd1/QLApuxzimti2PP6bMo+h8l/7FI8ZOaKSrt6k3Sumg2Vja6+dN6Wmr0ydd2F4jJ8uGQfDtgxb4uAOcsP2t6I3BqDEzv8e9FefLsuW+9mAKjMOmvrY33E3HU4W1SCsf91bFz09o/WYPD7luuqVM8FUr34W/UaGjWHgqonZLv9ozWYs/wgZi61XXBvw+GzJn/bW0DR1WS/BmvZhXxUxSRB2V8XrU1esAvT/tqLG/69XO+m4HJphd3zoVyhvELgrtlr8ML/Kof88s4zB48nYHAiAWtDDraGI/bknsfrv1rvwnfVCpmHPt9g/L36Sp7ktzNQeLlyLsabv+8yeczUP/dgd47p3I59ebZXG905O9P0ub9QvmxYK3tyzuOiDhOjlVIbZxw9e9Fk0u0vWbXrQ1VRMjm3OhkzUNqyYNtJ9J+2THUFXSGExUR7VeyZE1WluKxc9fLfwsulVldEPfDpOrvb42zrD5/F+sPn8P2GyiE/DfMhGq3adxq3fbQae5y80tFR7vh/ZC8GJxKYt/6oxWJvjr4XN2Wfs1hHZZ+C5a325jCo+eF72ErK+hV7Tzn8T7dsj2u/+SW9sQhpM1bgrhpBUk1aJ1sTQmDHiQLNa3jkFFxG73eWmhRWtJbz5K3frwbExaXlJit2VuytvdppyMzVbtO7VeWJbzbh0OkiPPqlurlVY/+7GR0n/qm64rYSFRUC3d9YjM6v/ak4+WJOwWV0ee0vaetJ2VL9fbO+Ro+pVu7/dC02Z+fjkS9d/yXHFk+dO8XgRAIXS8rxmo3eD3vd9/HfZm/fpjBT6xeZRzRpR26h9erFB6pVKN2XdwFn7Kh2/O6ivaq/0dvrjMJMutWTrWUecLxK7merD2Pw+6vwxDebrG6nNmtw1tF8VdtXz6Xz2q87kfTGYuPfx/NrzzXZeqwAp4uK3XL45nj+JVWvz+/bKlcvfbb6sOZtKS6rwPniMpSWC7M5dcxZtLNymfZOyZb82vNWUBqc5F8ssSsYPitpgVVnkvVfksGJxIpKHJ90e7m09kXqcmk5vlSwFHrjEe2+pTz3fZbV+6tfTA+eKkLSm4utbG3eexn77ErZ7gxFxWW1hqvMXbSrnFW4+uuTlZUTARfvyjW5/e0Fu/B9tZVaFcJyDR9nqPm6L1NQOmCLyoBIrayj+Zo9x6Rf5SsWSeZtP16AxEmLcMP05aqTHxbZPUzrvEu8p00Wr8LgRGIDZ6zUfJ8bj5xD+1cX4gcFSzbvmGV9yEKNQgvZbeuqge+tUHX+pphZZWXOCQvflv+z4iBe+J9p4a7bPlpTazshhEvGrR/8bD1O2+j92nb8au+ds+qsDJnpnkMZSphbrVZcVo5pf+7R9IuF3mq+M2xN2q764nXodBFSpy9HvgRpHxzxnxXOXZkk67ARgxPJaf2R/S+FF0FXMsDgcH0PW05Y6bWoSYv5HEedmJRJqZo9NeUVAi3GL0CL8QtqbWsub09VqXZ7OVJEUmu5hZfx/YajTn+fOVv1C/MrP9euYv3pqkP4cOl+Tb9YOIurOgScMbzmSvM3XZ2Y7kmdKAxOJKf1SpscF6U2V6O4rNzst3wtvb1AeVB2PP8SzjlwYf1rh+VU7LZ6E5zJWprx8fO3qdqXq6pLa6HgYilu/XAVXvjfVryrQWFDLTjr26q71HACKiehqi10umb/GdVDt44UQtXarpOFmPTrTuw6WYjXf91hV30yT8HgRHYaR8rZElYQlfED9V4LE4mVkDFzrh7MrRJTk/tEKwmT/jJOyP5t60m7VqB50BdWl1KbyXjV/tMYIfGyZ1sGvbcSc1cfwqD3VuKz1Ycx1MawY82irp6EwYnkDlpZgltXOKMekDlp79YuymeJsys7A3D7sXB7WKpXZO3qn1d4GX/uyMGinbn4vy3qsxxXdzz/Enq/s1R1sbrNduQlKbhUgtMXil0yMflCcZlUQf724wV49MsNmrSpZuexuYnlZeUV+HDJPqzZf9qYD8WWE/mXdF/ebm0uXll5BfpNW+rC1sjFR+8GEP202XKyLy3tcULeCUf87KLjriLTxQtQ3hvRf9oyk1UUyS3CERkSYLJNcVk5zlwoQUxYoKJ9/rUjFx2iQyy3TYPh1AXbcrBgW+UQ3+EpgxU/bseJAmSfuYhBnaMVP6bf1KU4faEELRvVU91OrV0oLsOtH65Chahcvrzqxesd2p+tydL78y7g74NnMO0v80N25h6/cPtJPP71JgzqFIVZ9yeZ3FdcVo7PVh9G37aNrL5H1Nh1slB1uY1Dp4ukmLumF/acEOnEWbltLHlGQVVjV1q8M9f2Rqi9vNNc5eeb3luJXlOWYPtx++bCHDh1AS//tA3Hrgw7/WwlO641Wky4Hfz+Koz5ZhM2Z5sWTnzxf1sxwELv3+kr+TkOnrLd02otU6wWKoRAVYeEmvpK9kqdvlx1+v1ZV2rz/LG99vywT1YewpQ/dmPQe9qtlhz03krFK/KUuFRSjh0nCup0xlj2nJDL1OV/JLLtqxoJ/Sx9kywrr4CPt/XvTRUVAk/+dxNaNqqHcWntjUn8ftt60q623TU7E2eLSrDxyDksfKZPrYKaSg37j/W5SkfPXlQ85yXraD66Nmtg/HuehYrjatzywSpsO16A27s2cXhflugxVLLrpHa9okoTVOqh6pUdOnM19uSex0fDu+EmFT1s7oQ9J+Qy10tQ5Iz088ESywUdqyeastQ9X926w2fxx/YczFx6QJNCcFXLnq3NNfpx4zGbgYWtpG+931mK+z5Zqyhbq6Pp7//YdrJW1eOq3DLznTik2Psd+edJqJ0HVHCp1KSAaU1afO+6XFqOEXPXKVrBVDVEPX+Tsvk17ojBCbnMIQ+Y3Ev2+SXr6kTXL9YcNv6+9qD5lP/Vl4fe/8lap7Wruud+2GL1wnvMgZVIl0rK8f36ozh1Xrul5mO+2aRL1ePzkidc/PrvIybzr7Yey7f5mITX/0Ly2xlOncT+/YajWLH3lOoVTI6StT+bwQkRScvWMAkA7M29eqFRmmn2vYy9GDhjhaZ5Z5TWuqnpyJkivPn7Trzw41bcNbt2vp/j+ZeszhPRajXQ2wt2OSXvxvrDZ/HjlYzUX/2tvlaXw70SNR5fM3mdtaKhNZfyVn+v2fKfFQcUbZdTcBlzVx0y6ZlR22tWVFyGv3bkmMx5OpF/CddOWYJZyw7g2XlZ0uT4UYrBCRFJ5ZILsrhWiMohnPcz9tne2MmGzlyNv65MDj5cLSvv3twLWL73FK6dsgQj5lrO7aG2yKMl/1lx0JjuP6/wMt5esAtHzjje23nX7Ew898MWbM4+h1fNZLWdu+qQxfloW47mI19BMU8tyx+cKboasLZ6aYGi4O/85TLcON20l0pp4seekzMw6bedmLn0ajAzbI66DL9PfLMJj361ERN+qXx9i4rLMO2vPTiefwn/WrgbP20+jvckeK+rweCEiKTjqsnT9pQq+DLzMN76fSeEEDhXVFJrXoda58ysPgIq62B9eWWIa40GFa2VqBqSeeKbTfjPioO4/aM1uFxajm3HCjB+/rZaPU1qzpOlBJCTftuJFftO17p99f7TGDJzNZ76drOKI6htzoqDqmoNrT9sukrKWs9KlZLyCuzTcKm+pfeEpde7anL59xuO4bPVh9Bx4p8mae/dEVfrEJFbKS4rR7GF3pU5y51bJA0AJvxSWaH4loQY3PphZU/D8nH9rD7m4c/Xo2F9P9XPlaWgqrLWi2MWbj+JDUcqL9BnikrQ/tWFxvtOnS/GJyO7QwiBY+cuGatkK2Gt5IG5icb2rpgy58G567Ht9TSz9x3WoHfIGew9ra+rTFHgaPDnLAxOiMitTPhlBx7r01KTfZnLSyKEULQkuXqNl03Z5xDbIMjithm7rV9oLU2EPaOgxtM3dszjsObxrzdZvG9/3nk8Oy/LrsSJSlahXC4tx6nzxYgND8L6w9pVVj5vZWhGbQ9DbuFljJ+/DSNSmiva3t5hN0s5bTwFh3WISEoFFrq2s47mY/WB2sMA9vg5q3Y6/FX7tdm3q9ibMM4e5UI4NaNz2owV6P3OUmw/XmBc9iybV37ejiW78/DgZ+ttbvvC/7ag44Q/XdCquofBCRFJ6aPllvOifP13tmbPU3NIwZ2WvDs722tNzkqnvu/K6pQjVyYE/77NvmR6rrBIYWZjoHIOSInG58jRFVVbjua7xXucwzpEJJ3LpRU47oLU58DVBGxVrCWLk83NH6yyXTjTRYU1HfFF5hH0ah1h/DvTCROAN2efM8m4a4+7Va6i0dqAGSvgyNk8kX/JuCJLTb0nPbDnhIik0+PtxXanoneUlonQnM0V1bNd5bGvNhp/VzIRWK3bPqqdQ8bdHDxVZCzVYA936DGpwuCEiKTjyiyjNQvsuRtb1aaV1vLxBJ5e34vBCRGRm7C3OnSFa6d72O0/K5y/vNpdPCtZZW4tqKnEXTM7bk0yBW8MToiI7PD+kqsZNzdn5+vXEFLM3OosW2TNg1Ll5g9WabYvLRPJOYoTYomI7LDu0NU8HF9mHoG/D7/r6cVZq4gAYOqfe5y2b0eVltvf0/H3wTPo2bKhyW0SdZyw54SISAt/bM/RuwnkYarS1tvjHgVFNfXE4ISISAPHXLT0mcgTMDghIiIiqTA4ISIi8kA3Tl+udxMsYnBCRETkgWRanVMTgxMiIiKSCoMTIiIikgqDEyIiIpKqRiSDEyIiIpIKgxMiIiKCRB0nDE6IiIhILgxOiIiISCoMToiIiEgqDE6IiIhIKnYFJzNnzkRcXBwCAgKQnJyMdevWKXrcd999B4PBgKFDh9rztEREROQBVAcn8+bNQ3p6OiZOnIhNmzYhISEBaWlpyMvLs/q4w4cP4/nnn0fv3r3tbiwRERE5h1vnOZk+fTpGjx6NUaNGIT4+HrNnz0ZQUBDmzp1r8THl5eUYPnw4Xn/9dbRs2dKhBhMREVHdpio4KSkpwcaNG5Gamnp1B15eSE1NRWZmpsXHTZo0CY0bN8bDDz+s6HmKi4tRWFho8kNERESeQVVwcvr0aZSXlyMyMtLk9sjISOTk5Jh9zKpVq/Dpp5/i448/Vvw8kydPRmhoqPEnNjZWTTOJiIjIjTl1tc758+fxwAMP4OOPP0ZERITix40fPx4FBQXGn6NHjzqxlURERCRTjlgfNRtHRETA29sbubm5Jrfn5uYiKiqq1vYHDhzA4cOHccsttxhvq6ioqHxiHx/s2bMHrVq1qvU4f39/+Pv7q2kaERER1RGqek78/PyQlJSEjIwM420VFRXIyMhASkpKre3bt2+Pbdu2ISsry/hz6623on///sjKyuJwDREREdWiqucEANLT0zFy5Eh0794dPXr0wIwZM1BUVIRRo0YBAEaMGIEmTZpg8uTJCAgIQKdOnUweHxYWBgC1biciIiIC7AhOhg0bhlOnTmHChAnIyclBYmIiFi5caJwkm52dDS8vJp4lIiJyJzLlOTEIIYTejbClsLAQoaGhKCgoQEhIiGb7jfvn75rti4iIyJ1lPNcXrRrV13Sf9l6/2cVBREREUmFwQkRERFJhcEJERERSYXBCREREUmFwQkRERBLlh2VwQkRERJJhcEJEREQwSJTohMEJERERIa/wst5NMGJwQkRERPi/LSf0boIRgxMiIiLC9uMFejfBiMEJERERSYXBCREREUmFwQkRERFJVZaYwQkRERExCRsRERHJRaKOEwYnRERExJ4TIiIikgwzxBIRERFZwOCEiIiIpMLghIiIiDjnhIiIiMgSBidERETEpcREREREljA4ISIiIi4lJiIiIrl4yRObMDghIiIiwCDReh0GJ0RERMQJsURERESWMDghIiIiqTA4ISIiIg7rEBERkVw4IZaIiIikwp4TIiIiIgsYnBAREZFUGJwQERER09cTERERWcLghIiIiCRaq8PghIiIiCTD4ISIiIi4lJiIiIjkIlFswuCEiIiIuFqHiIiIJOMlT2zC4ISIiIjkwuCEiIiIINOsEwYnRERExNU6RERERJYwOCEiIiKJBnUYnBARERE4rENERESSMUjUd8LghIiIiKTC4ISIiIikwuCEiIiIOOeEiIiIyBIGJ0RERMSeEyIiIpILV+sQERGRXOSJTRicEBEREeAl0bgOgxMiIiKSqeOEwQkRERFxQqw0Zg3vpncTiIiIqAaPDk4GdY7WuwlERERUg0cHJ0RERFRJolEdBidEREQkFwYnRERExKXEREREJBl5YhP7gpOZM2ciLi4OAQEBSE5Oxrp16yxuO3/+fHTv3h1hYWGoV68eEhMT8dVXX9ndYCIiItKeW6evnzdvHtLT0zFx4kRs2rQJCQkJSEtLQ15entntw8PD8fLLLyMzMxNbt27FqFGjMGrUKPz5558ON56IiIjqHtXByfTp0zF69GiMGjUK8fHxmD17NoKCgjB37lyz2/fr1w+33XYbOnTogFatWuHpp59Gly5dsGrVKocbr4X46BC9m0BERETVqApOSkpKsHHjRqSmpl7dgZcXUlNTkZmZafPxQghkZGRgz5496NOnj8XtiouLUVhYaPLjLEMSY5y2byIiInch0XxYdcHJ6dOnUV5ejsjISJPbIyMjkZOTY/FxBQUFqF+/Pvz8/DB48GB88MEHuPHGGy1uP3nyZISGhhp/YmNj1TSTiIiI3JhLVusEBwcjKysL69evx1tvvYX09HQsW7bM4vbjx49HQUGB8efo0aOuaCYREZHHkqjjBD5qNo6IiIC3tzdyc3NNbs/NzUVUVJTFx3l5eaF169YAgMTEROzatQuTJ09Gv379zG7v7+8Pf39/NU0jIiKiOkJVz4mfnx+SkpKQkZFhvK2iogIZGRlISUlRvJ+KigoUFxereWqnkWmMjYiISC8yJWFT1XMCAOnp6Rg5ciS6d++OHj16YMaMGSgqKsKoUaMAACNGjECTJk0wefJkAJXzR7p3745WrVqhuLgYCxYswFdffYVZs2ZpeyRERERkNy+J0rKqDk6GDRuGU6dOYcKECcjJyUFiYiIWLlxonCSbnZ0Nr2pHWFRUhCeeeALHjh1DYGAg2rdvj6+//hrDhg3T7iiIiIjIQfL0nBiEEELvRthSWFiI0NBQFBQUICRE27wkH684iLcW7NJ0n0RERO7mvuRmePu2zpru097rt0SdOEREREQMToiIiEgyHh+cSDQ5mYiISDcyXQ49PjghIiIiuTA4ISIiIqnynDA4ISIiIqkwOCEiIiKpMDghIiIiqTA4ISIiIqlWr3p8cGKQ6WwQERHpRKaroccHJ0RERCQXBidEREQkFY8PTmTqxiIiItKLTNMcPD44ISIiIk6Ilda1rRvq3QQiIiJdGCQaS/D44OSOpKbG3zs3CdOvIURERASAwQlCA331bgIRERFV4/HBCREREXHOibRkOjFERESuJNMlkMEJERERSYXBCREREUmFwQkRERFJNbWBwQkRERFJhcEJERERMX29rITQuwVERETE4KSGHi3C9W4CERGRR2NwUsNtXZvo3QQiIiKPxuCkGoNBriQ0REREriLT9Y/BCREREUkVnTA4ISIiInhxtQ4RERHJRJ7QhMFJLQG+3no3gYiIyKMxOKnhps7R6Nu2kd7NICIicimJRnUYnNTk5+OFLx7qoXcziIiIPBaDEyIiIpIKgxMiIiKSCoMTAD3iKlPW39Gtqc4tISIi0odBovU6Pno3QAbfPtoTFy6XITTIV++mEBER6YITYiXj7WVgYEJERB5NotiEwQkRERHJhcEJERERSYXBCREREUk16YTBCREREUmFwQkRERFJhcEJERERSYXBCREREcFLniknDE4smfdoT9yaEKN3M4iIiFzCixNi5ZfcsiHev7er3s0gIiLyOAxOiIiIiBliiYiIiCxhcKJSVQVjIiKiukSiKScMTtTq2ZLBCRERkTMxOCEiIiKpMDghIiIiGCQa12FwQkRERFJhcGLDj2NSnJ7v5JnUNk7dPxERkTthcGJDUvNw3JoQg4SmoQCAWxObaP4c9fx8NN8nERGRu+JVUaEfx/RC4eUyhNfzU/W4h69rgYXbc3A8/5LFbQSEo80jIiKqM9hzopCPt5fqwAQAXr05HgmxoU5oERERkXYkmg/L4ISIiIgAg0QJ7BmcOFG7yOBat41La2d225AAjrAREREBDE6c5qkb2uCPp3sDAIZd0wwA0CKiHp7s3xpP9GuFYH/TYCSivr/L20hERFSFwzoewNfLAC+vyjPdt20jLB/XD38+0wcA8MLA9tj62gDjtoLzYYmISGdeDE48T/OG9eDnc/XldkUmvkBfbyx4qrfTn4eIiEhLdgUnM2fORFxcHAICApCcnIx169ZZ3Pbjjz9G79690aBBAzRo0ACpqalWtyftdI9rgPiYEL2bQUREbsCtJ8TOmzcP6enpmDhxIjZt2oSEhASkpaUhLy/P7PbLli3Dvffei6VLlyIzMxOxsbEYMGAAjh8/7nDjSZkx/Vrp3QS7JbdgFWgiIk+jOjiZPn06Ro8ejVGjRiE+Ph6zZ89GUFAQ5s6da3b7b775Bk888QQSExPRvn17fPLJJ6ioqEBGRobDjafaMp7rW+u26nNaOjVxr56Uh69roXcTiIjIxVQFJyUlJdi4cSNSU1Ov7sDLC6mpqcjMzFS0j4sXL6K0tBTh4Za/ERcXF6OwsNDkpy7Tcj5sq0b1rd7v7+Ot4bM5n0xVMomIyDVUBSenT59GeXk5IiMjTW6PjIxETk6Oon28+OKLiImJMQlwapo8eTJCQ0ONP7GxsWqa6XSDOkUBAK6Ja2Bxm4hg5UuDuVqHiIj0JtN3QZdm/poyZQq+++47LFu2DAEBARa3Gz9+PNLT041/FxYWShWgvH9vV+zLvYDssxex/vBGk/tuaN8YkaEBuDOpqU6tIyIicm+qgpOIiAh4e3sjNzfX5Pbc3FxERUVZfey0adMwZcoULF68GF26dLG6rb+/P/z95U1K5uvthfiYEBw9d7HWfTd1jsYddgQm7DwhIiI9yTSMrmpYx8/PD0lJSSaTWasmt6akpFh83DvvvIM33ngDCxcuRPfu3e1vrRuQIci4rWsTAO69Soe0wbIIROSOVH9ypaenY+TIkejevTt69OiBGTNmoKioCKNGjQIAjBgxAk2aNMHkyZMBAP/6178wYcIE/Pe//0VcXJxxbkr9+vVRv771yZueQmgc0ky/OwETb4lHWJD6KspUt8SEBaIw57zezSAiUkX1UuJhw4Zh2rRpmDBhAhITE5GVlYWFCxcaJ8lmZ2fj5MmTxu1nzZqFkpIS3HnnnYiOjjb+TJs2TbujIBMGg8HhwMTZS457t4lweB8P9GyuQUvIEXMeSLK5TZOwwFq3pXWMNLMlEelJnkEdOzPEjh07FkeOHEFxcTHWrl2L5ORk433Lli3D559/bvz78OHDEELU+nnttdccbTtpLLVDY7SNrI8n+zt/OGjiLR0d3sfzA8xXeCbXGRBvO8h49eb4Wre1bsxeUyKyjLV1HBDXsJ7LnqtHnPMzpdb398Ffz/bFuLT2Tn8ufx/H33qhQb4atMR9jUzRv+fI1gS6Rc/2wcBO1ifLExHVxODEAe2igjH7/iT8/OS1xtv06BYb3CUa/ds1wuz7bXexu5s+bR0f/qmrokJrD5fUNLp3Sxe0xLLGwZZTBhARWcLgxEEDO0UhMTZM8/2+MaQjpt+doGjbhvX88NmoHg5/Q332xrbG3+v7O3eVR80v3N4WanW7W0ZbV3qwV5zNbexZ1k5EnslXgx5trcjTEg9mLkPsAylxaBHh3GGjj4Z3w8cjuqN9VDB++8d1aF5tmOpfd1jPRaM1P2/nvhVbOuG19LEQULmKFkNjjri/ZzNdn5+ItDW4c7TeTTBicOLBAny9cGN8JBY+0wedmoSa3Ne8YT3FK3Z+eNw0x829PVx/0ercJBTto4It3v/HM71d2Bq53HONc7IrN1CwIkzrZfJE5Dy+3vKs12FwIjGTyYZW3jN61+a5Ji4ct3erTPzWq1VDTL69s8vb8NqtHbHwmT4W7/fk4SE9zkdNqR24dJiIlGNwojFnxQnyxLPmvTW0Mz68rytmK8h7Yc7oPlcnbt7fsxlCAnzw77uUzbmx5a9nLQctdVFMqOkkVBlSUn94X1fj71EhnCRLRNYxOHFDw5P1H+uvOQk40M8bN3eJQUiAfct7n76hjfH37s3DkTVhgGaTOdtGWh7uqW5oYgzeuydRk+e0xtq47h3d6uYE1gDfqz1XfhJNuiOiq2T4IlOFnxJkl6YNbC9jVaPmah0vOyebvjjQ/hwtM+7piiGJTex+vFK9Wje0eN8bQx1PTmeLVj1S9moWHoTBnWN0bYOn6N+ukd5NILILgxMPI1RMUHnq+ja2N7KTsyJ0NcUOJ5jJXKqGM4bwgvycu4S7WXiQ1R6piPrWJ7k+fF0Lq/d3axZm8b4fx6Tg5i7RmHZXAuJjnFsegSq9PNix9ziRXhicSEJN0KDVdd3Hy/rpH9AxClPvdN6SYkv1dexZ4WHromrOQzYutHWRreDCFnOp6JVKah6OD+/rhqhQzjlxFZlWXxCpweDEAzxwJc35kETTrvRrW9vOvtpE4+Gb6topnAtizWcPXoNpdyWY5GixpPq8lpqU5FkJr+e+VZ4furYFWjeuj7u6az+nJal5A833SUSejcFJHWCrp6FJWCD2vDkQM4YlmtxuKSurErckVAY69vRYVHn2xrZ4rG9L/FIt/b81T13futZt/ds3xp0KJ85Wz4Bb0709rOcCSW4RjuYNgxQ9jy0GHdZeTbglHovT+zpl2OjHMb003ycRuZ5M/WwMTiSgZEjHNOWJ+reQv4+3ZvM8Vr7QHwPiI/HjmF7ISO+n+HHBNVLi1/P3wfhBHZCgMP1/aw16Wuxla8XP4C6uz6xoadKwlvN5rM0hIfl5WXgvvDG0k4tbIr9HPHCYtyaJFuswOCH1fLwNMBgMSGreQFVl4J/HKushkZGt1Ukf3tvV6v3O4sjqJPNMP53eu0fdcTEfrFwsvW9DApw78dodveLgBHln+nXsdR43fMrgREI7J6UBkKuLTQvNwq8Oi4QG1g5qqpJz9Wrl/ErEAzs6ViSxJmu9FXe5cfE9f19+RLgzg8GguIAoyatz09A6dz2whZ88EnL2clK9+Hp74dex1+GnJ3qZrXq8/IV+2PTqjYjUIYNoSivLuUeq+NpZnPAZK3NdZNOykWMFEg1wLI+KHl3rehdQdDa9y1u42pP9racT0Ltgp7087DQyONFa5xoF9Ooie7PAApXfALo2M9896e/j7fIVMZnjr8fcB7sjTUFPytu32VejpklYIGYN72Zzu3QVQUzV6qKq5djVJzc3DvFX2cKrRlxZ2eWI6ywsEVdCSZCotXFp7RBsxzBHdw/rZncX49La49vRPfGnhVpbji6nt6RvW+sJ75TOrdOTHpP1LambX9F1sPKF/sg7fxntrFTGtUTqbzY12vafB5JQz0yvh0ua4oQXKjo0ENGhypZLt25cHwG+XrhcWqH6eWxddBNiw/CUlaXO1Y26Ng73Xan83KlJKBY81RvRoQFYf/gslu45hWviGmBzdr7qNgK2c994El9vA0rLZf7nJEv0CHIHd4nG8r2nzN7XMqIefnnyWsT983e79++Mzz+Z8ZNII7HhQUhqHu6S59JzRrU7RP/OVDVXpb3KIDQsyA8bXknF9tfTHD5/E2/piDbVVg/Fx4SgQT0/DOgYhcm3d9a9dk31Ojr2DoU5m5Ke/Yj69vdAVfecGw3ryer+ns6tJ3adgpxPjnBkMYC9PbbuTs5PDqrFWnebK4eSwhSuzmkRUTl3oWaFXEe5sqJtl6a1X9c3b+uMybd3xtePJKveX0R9f7NzbWRm7X3XzcLwXGigL6be2QX/vitBt142ax65rgV2ThrosueLdKOMuEr/v13tzaGd0bOl87783Rgf6bR9A44Nhd8nQaFXPTA4cROmeU5M3ZlkPYGYVja9eiP8fbxtbwjgr2f7YMuEAWiscTDh6l6jqh6SQZ0q85jU9/fBvT2aIaK+Px7sFQfA+R9saikZN65ZVVqN5eP6Yc4DSehnpajcXd1jFVeVrr6KyxW8vAwmvTvOpubC5OiEZEep7REEgFsTXFPE8f17utqcE+XI+9oR2i/p1wfznJBm7u7e1KFMr2qomazq6+2lKgeKq3krrDny2z+uQ9aEG9HMTHbYfw5qj/+OTsYHOuU4cUT/do3tfmzzhvWQ1jFKs2RvSnZzTVztXpqbu0Tjpyfkzk47rHssBigMXu/v2Qw/K8yWLJP37kk0Zox2hqrVVI1DAjBpiPXkcS0j7AvuBnVyLLWAs+a4uHPJDEcxOJGArF2pddErgzsgNjwQ4wcp+6bj4+2FsCDzHxC+3l7o1SrCqd/C37sn0eTvVhp9s7anuKKezI27D+wUZXHllyz+dWcXi5l86wqDweDWBQZ3TkrTvIdXJj1aXB0OaxcZjKXP93OLJHwMTnQ0/e4EDEmMwbBrmuGFK92CD/RUt5Szdxvry9fI1CO9W2LlC9ejaQPthxJ+eDzFYqVlew1JbGL8PTo0AAstLI90pmtbu37lQ01O7262I1ZzVRf4h/e5X8+cO9Eir5TSlTS6DDtVa5rBUDkfcOtraa5vh0oMTnR0e7emeO+ervDz8cJNnaOx8ZVUTBrS0ebjqnen3+zkmi7u9f1aG4FXekLUZqq9Ji4cXz2sfqKsUiEBvrqsfukR59zgxJHJgvZw9soMR5i7xg3urOx/PLmF+QmjgX7On19jT36MmvV9nDnh1YSOnTyunl9VU4iZzNyyYnAikYb1/U0CDyXfzLQs8ia7RsHaLO20ZVF6H7w5tBOeSVWWd0StaBurN4Ymmh+/d7ehGKUSYkMxpl8rTLnd+UsmR6Y0Rx8bybJ05cApbmBh+FHpnBdXaRIWiL1vDqrVS/zdoykIckEg5SyuXEmoRqNqSRkfujbO6rYyXU7kH3jyYNW/jcj0pnG12fcnYV/ueaS0dM3wQtMGQbhf5fCaGn3bNsK4tHaIjw7BqM/X17q/jYurL//3kWTc98lalz5ndQYYjKsdMnblOvW5kjV6D3V1o2rNPhLmmlGTi8ddQvLebSLw3I1t0T46BKO/3KB3c4xu7BCJIF9v+Pt6KcqELQsGJyS9gZ2iMNDB2fR66NkyHJ+OvKbW7QaDAU/2b+2051UbyPbScJjDXS4kSlh7GRvW80PBpVKH9j9jWCKemZfl0D5IHgaDAf9QmOXZlQwGYKoD9a70Il9ITVRHRIYESJmETA1P7rFzNi0LXNbTaDikbY1eu+EemgBMS9byAdli70IvrSfm64HBiRvSo24EKffm0E5oHxWMfypcrky2hQa6Pt+DtflcnZqEal4kzZHSKdFhyupDmXND+8ZI6xiJ0b1bYFxaO/sbIaGnbfRk/OLkvDIN6/lhrpneU6XmPNDd+Luat0dPFw2BOxODEzc0ID4Sn47sjjX/vF7vppAZ9/dsjoXP9FFcUFApV9T96hBde77LXd0rM71e397+xG3WWIoBqmcrddVk6OpGXRuH2PBAPNGvVa37Hu9b+zYtmJv0bGvSe9+2jTDngSS7n/PTB6/BnAe64+XB8Qh28coprT1XI7h69sa22PbaAIvbO7tWWMP6fg7luUmys/K1vXPmZKpKzODEDRkMBtzQIRIxDnxbUsrDCmF6rNX/vB6/jr0OzRvWTvIWExaI3W8MxKcju5t5pP7WvXyD4m0tvZ97mFmG2yDIDyvG9TfmIKpOSeK9uQ9af720GjL74qEeaNWoPppHaL9MVWkbZSjh0CQsEE3MfCY6Wmn77u7KyjDIJNSNlgxbwuCE6rTf/nGd3k1wC03CAtHZTKHDKgG+3rovW//yoR546abagULj4AD4OJiFdepdCcaK09XZe8ypHSJxffurF+wXB7ZHbHigSc6S7s0bKC7a2bxG+QRzmYKfur6NzdozzpLWMRLfP5aiy3NXsXSqavZGqe0daBFR394m6fLlzpH5JjLNMWNw4iZG9WoBoG5MdHKV27s2QScXVmyWRQs764vIrk/bRni0j3OGU8Lr+eHZG9s6Zd8AMKZfK6x84XqTSbA+3l74v7HK5jz88FiKsdAkUFmlt6Z6/j42a884YuIt8RbvMxgMZnuf6gIt803Z2qx5wyCHErVNvCUeXz7Uw+7Hy4TBiZtIjY/Eyhf647MH7Z9cpQetvzhIFNhLq0N0CGbfn6T4wqe36udUpm9urqA06WLjkADF8wiq6jFZCybsMeraFpruTwuP9mnptH2P7t0CzRsG4T4HViyp/fxbPq6/Q3O7fLwMDvVwyvTv597rHD1MrM6pj8k1lNbpsMYd88LozV9FYjCZDUlsggHxUS5JWz/1zi4W72sbWR97cy/Uuj3AV5vXedKQjhiREof/rDioyf5qenlwPF66qYOii70W/7Nq3N61CeZvPu7S53S1uvHfWEd52rdIoupsfd7bs7IgKtTyqp+4iHomQyfV/flMH80qQltj7ZiqT6uxVcqgZmASHx3iULvM2f3GQNzVPdbi/TXrTM28rxtaRtTDh/d1s/gYJWd03qM98dT1rXFfD+fnYNF7npUl04cl6t0Ep2NwQuSB+l6pL2Ort0DOj2b7dG0WhqTm1udFvHbr1cKb1a9L7aKCjSn2nUUIIDTI8iqLFhH1cGN8JO5Maqp6Bcq3j/Z0tHm12FqxVDPJ3OAu0VjyfD90UBso1YjDkls2RPqAdlKm5ZeRu6645Nklq+pqsTl3pOWZ6NqsAX5/6jqsfUn5MlxnaRel/bd6c2ScM6GGwWDAxyO6Y5odqchrLi21Vp12+t2V+3/rNudNrtVS1UTce64x34uj58XZ1cM9denTmnNOiDxUxxh9VzL99o/r8PfBMxhm4aKiRrdmDbDu8FkNWmWZrF389ujbphGGJzdDfEztwPD2bk1xU+doBPh645Wft+nQOnU+e/AabDmWj+QW7p8VVW8yvcfZc0IkGUtftlz9LUyN4Cs1hPq0UV5HpFOTUDzSuyW8LeQoUdNr9+F9XTG6dwu7EsXVhYRVanl5GfDWbZ0xPNn8CiAlSeacwZ53eD1/H/RqFWHxfVSTRNdfswZcSWjXsJ7rSzbIhD0nRDpqEOSLcxdLcX0756SGd5VF6X2xav9p3JoQo8vzNw4JwMuD43HgVO3VIVVqfthPvbMLMg+ewW1dmzi7eXYbkqjP6+kqz6S2wYzF+/DGUPcYQnJUr1YN8UvWCevbtI7Ab/+4Ds0aevbqTPacEOlo4TN9MGNYIsaYqd/iTqJCA3BnUlP4Sbocd2z/1uhVo2DmXd1jMf3uRLsnVr4wsLKOi6UVPo56/96ueO+erk7Zt1JVPWGOZuC15JnUttj4SioesLMWjFpv31Y7eZ0r3ZUUiw/v62ozIO7UJBQhLqpzNKha2gGZOpXk/CQh0oC1SX+yiAwJwNCuTaS9qNcVz6e103w8vXtcOHZOSjNZ4VNlaFfzPR739KicX3Nta9fPj+jURP3E4xvjI/HNI8lYM955RUYb1nddUce+7ZQPOzqDl5cBN3eJQdMG6uuiOWtUt/qKPZmGvPiJSHXOu8MScF3rCJvl0mUl78wSz9b9SoXY8GrDQ0F+piPjDev54dex15nU0KmubWQwtkwcgK8eSjZ7vzPZUwDPYDDg2tYRaBxcuSy4Zu+TUgN0LAyo5v+paugvOjTAxpaOPbdMK8dk/bzhnBOqc27r2hS3dXW/SqJkqmNMqNkMo3ppUM8PWyYMQIBf7Yu8t5cB5RUCvVpHWC2gCFiegNu0gfxzDHq0CMeaA2dUP06Wb+S2mjHvsRR8tGw/xvZvrc0TWrjyh7twsqu7poNgcEJWSbxAxOO46lT84/rW+GDJfgD6Li187ZaOaBTsL9WEVUtJ0v58pg9+3nwco3urr/Xy9cPJWLwrFw9dF6f4MWFWkrU5U134PGhg5bVr3bg+pt+d6LrGVOOseT3uisGJxDpEh6B9VDAaBbtuTFZ2U+9KwPBP1uKlm5ybrdOTPZDS3Bic6Ck0yBcv3dRB1WMCNVwCqyYua924Pp5Pa2fX81zXJgLXmak2bu3p20YGY/yg9rWysNZFWn/+jUiJw+bsfKR20G6oyVeDbLUDOuoz9CVrwMngRGLeXgYseKq3NF2iMri2dQT2vjmIE0jJrJiwQDyT2gb1/Hzw1oJdejfHqR7r6z4rvJRmAa7+UffD4yl4d9FesxOOHRHg641Z9ydptr/buzZBRzPJ7FwtMsQfQx3sZWQSNlLMy8ES2HURAxOy5pnUthjdR/3wCmnvlyevxVPXt8YTdiyVvyYuHP8d3RNtI4Od0DLtTB+WaPEz2lWdEo/3bYXMf96gavnxPdfEIjjAB/e6oICiPdhzQkR1UuNgf+SdL9a7GR4tITYMCbFhejdDSvf2iMXfB8+irKICR89ewpBEx3o9vFTOWZlyRxe8dVtnHLSSuFBPDE6ISBoBPtrNGfn6kWRMXrALz97YVrN9kvsJqjYPKcKFOVVsmXx7FwghcKm0HIdPX0SHaOf0EFmbU6I05b8eGJyQU8VHB2PL0Xy9m0FuomfLhhjcJRptGzv+Qd02MhifjeqhQavqBlcMMdyV1BQ/bDyG/jonO6vOy8uAXZMGokII6YaEDQYDgvx8zBZgdJUWEfXQtEGgdDWmGJyQVXEN6zn0+PE3dUBwgK9uNVfckayz513By8uAmfd107sZZKc3hnbCjfGRuLZ17dVHegr007aQ4Q3tGyNjdx5ul2iZu718vL2wfFx/qVLXAwxOyIZmDYPw30eSEV7fvqRBIQHql4OSeX4aLFdUwr/a0Ips3zRJXzFh1pcuB/h6Y0DHKKvb1AX392yOV2+OR7Nw/RLnPdanJeasOKjJvmQc3mFwQjb1kuxbUF1XM6Pju8MSMH3RXrw7LNElzx8a6Is3hnaClwGo78+PCLrqjm5NceBUEVJaur42kGziImz3KjszE+z4mzoYgxNrWWDdtSeWnzxEktMjHb+rqsQ6Ii5C/nTvjpAxg4CPtxd7QmE7Jfzno67BnOUHMeUOfasguzMGJx7G3vLwRLL4v7HX4kT+JXSMsV7DxlEG6UbhyV30a9cY/do11nSfN8ZHIutoPiJD5Flx5EwMTjzM27d1xsi56zD2eo0KWxG5WJemYejSNEzvZpATXNMiHMv2nJJu5YgMHu3TEnEN66FHi3C9m+ISDE48TOvG9bH6n9fr3Qwiolr+fVcCPl9zGHcmsap4Tb7eXhjcJVrvZrgM+/iJiEgKDev747kB7dDcwRQGWmOhUddjcEJERGTFo31aYcuEAXo3wzI3XZFjDYMTIsm469I/orosNOjqPBj+jzqfXcHJzJkzERcXh4CAACQnJ2PdunUWt92xYwfuuOMOxMXFwWAwYMaMGfa2lYiIiDyA6uBk3rx5SE9Px8SJE7Fp0yYkJCQgLS0NeXl5Zre/ePEiWrZsiSlTpiAqqu5nDiQikhK/7pMbUR2cTJ8+HaNHj8aoUaMQHx+P2bNnIygoCHPnzjW7/TXXXIOpU6finnvugb+/Z6zPJiJyVOcmzs3jQiQzVcFJSUkJNm7ciNTU1Ks78PJCamoqMjMzNW8cEZFe9KoUu+6lG7Dwmd7SrVgh95LWMRJhQb4Y0DFS76bYRVWek9OnT6O8vByRkaYHGxkZid27d2vWqOLiYhQXFxv/Liws1GzfRETWLHu+H3ILL6NtZLAuz984JACNQ6wX2COyZfb9SSivEG6bFVzKVk+ePBmhoaHGn9jYWL2bREQeIi6iHpJZ2I6saFhfrikKvmYCEIPB4LaBCaAyOImIiIC3tzdyc3NNbs/NzdV0suv48eNRUFBg/Dl69Khm+yYiIrLH7Pu7Yfyg9kiMDdO7KQCA8YPao2NMCEb3bql3UzSnaljHz88PSUlJyMjIwNChQwEAFRUVyMjIwNixYzVrlL+/PyfPEhGRVAZ2kit9/GN9W+Gxvq30boZTqO7zSU9Px8cff4wvvvgCu3btwpgxY1BUVIRRo0YBAEaMGIHx48cbty8pKUFWVhaysrJQUlKC48ePIysrC/v379fuKIjqkLSOlb2QTcICdW4JEZE+VBf+GzZsGE6dOoUJEyYgJycHiYmJWLhwoXGSbHZ2Nry8rsY8J06cQNeuXY1/T5s2DdOmTUPfvn2xbNkyx4+AqI6JjwnByhf6o1Ewew+JyDMZhJA/M09hYSFCQ0NRUFCAkBB9lvcREbmzIR+uwpZjBQCAw1MG69wa8hT2Xr/ddyovERER1UkMToiIPID0XeRE1TA4ISIiIqkwOCEiIiKpMDghIiIiqTA4ISIiIqkwOCEiIiKpMDghIiIiqTA4ISIiIqkwOCEiIiKpMDghIiIiqTA4ISIiIqkwOCEi8gDXto4AAAT48mOf5OejdwOIiMj5nr6hDZqEBaJv20Z6N4XIJgYnREQeIMDXG/f3bK53M4gUYf8eERERSYXBCREREUmFwQkRERFJhcEJERERSYXBCREREUmFwQkRERFJhcEJERERSYXBCREREUmFwQkRERFJhcEJERERSYXBCREREUmFwQkRERFJhcEJERERScUtqhILIQAAhYWFOreEiIiIlKq6blddx5Vyi+Dk/PnzAIDY2FidW0JERERqnT9/HqGhoYq3Nwi14YwOKioqcOLECQQHB8NgMGi238LCQsTGxuLo0aMICQnRbL8yqevHyONzf3X9GHl87q+uH6Mzj08IgfPnzyMmJgZeXspnkrhFz4mXlxeaNm3qtP2HhITUyTdcdXX9GHl87q+uHyOPz/3V9WN01vGp6TGpwgmxREREJBUGJ0RERCQVjw5O/P39MXHiRPj7++vdFKep68fI43N/df0YeXzur64fo4zH5xYTYomIiMhzeHTPCREREcmHwQkRERFJhcEJERERSYXBCREREUnFo4OTmTNnIi4uDgEBAUhOTsa6dev0bhImT56Ma665BsHBwWjcuDGGDh2KPXv2mGzTr18/GAwGk5/HH3/cZJvs7GwMHjwYQUFBaNy4McaNG4eysjKTbZYtW4Zu3brB398frVu3xueff16rPVq/Rq+99lqttrdv3954/+XLl/Hkk0+iYcOGqF+/Pu644w7k5ua6xbFViYuLq3WMBoMBTz75JAD3O38rVqzALbfcgpiYGBgMBvz8888m9wshMGHCBERHRyMwMBCpqanYt2+fyTZnz57F8OHDERISgrCwMDz88MO4cOGCyTZbt25F7969ERAQgNjYWLzzzju12vLDDz+gffv2CAgIQOfOnbFgwQLVbVFzfKWlpXjxxRfRuXNn1KtXDzExMRgxYgROnDhhsg9z53zKlClSHJ+tYwSABx98sFb7Bw4caLKNu55DAGb/Hw0GA6ZOnWrcRuZzqOS6INNnp5K22CQ81HfffSf8/PzE3LlzxY4dO8To0aNFWFiYyM3N1bVdaWlp4rPPPhPbt28XWVlZ4qabbhLNmjUTFy5cMG7Tt29fMXr0aHHy5EnjT0FBgfH+srIy0alTJ5Gamio2b94sFixYICIiIsT48eON2xw8eFAEBQWJ9PR0sXPnTvHBBx8Ib29vsXDhQuM2zniNJk6cKDp27GjS9lOnThnvf/zxx0VsbKzIyMgQGzZsED179hS9evVyi2OrkpeXZ3J8ixYtEgDE0qVLhRDud/4WLFggXn75ZTF//nwBQPz0008m90+ZMkWEhoaKn3/+WWzZskXceuutokWLFuLSpUvGbQYOHCgSEhLE33//LVauXClat24t7r33XuP9BQUFIjIyUgwfPlxs375dfPvttyIwMFDMmTPHuM3q1auFt7e3eOedd8TOnTvFK6+8Inx9fcW2bdtUtUXN8eXn54vU1FQxb948sXv3bpGZmSl69OghkpKSTPbRvHlzMWnSJJNzWv1/Vs/js3WMQggxcuRIMXDgQJP2nz171mQbdz2HQgiT4zp58qSYO3euMBgM4sCBA8ZtZD6HSq4LMn122mqLEh4bnPTo0UM8+eSTxr/Ly8tFTEyMmDx5so6tqi0vL08AEMuXLzfe1rdvX/H0009bfMyCBQuEl5eXyMnJMd42a9YsERISIoqLi4UQQrzwwguiY8eOJo8bNmyYSEtLM/7tjNdo4sSJIiEhwex9+fn5wtfXV/zwww/G23bt2iUAiMzMTOmPzZKnn35atGrVSlRUVAgh3Pv81fzgr6ioEFFRUWLq1KnG2/Lz84W/v7/49ttvhRBC7Ny5UwAQ69evN27zxx9/CIPBII4fPy6EEOKjjz4SDRo0MB6fEEK8+OKLol27dsa/7777bjF48GCT9iQnJ4vHHntMcVvUHp8569atEwDEkSNHjLc1b95cvPvuuxYfI8vxCWH+GEeOHCmGDBli8TF17RwOGTJEXH/99Sa3udM5rHldkOmzU0lblPDIYZ2SkhJs3LgRqampxtu8vLyQmpqKzMxMHVtWW0FBAQAgPDzc5PZvvvkGERER6NSpE8aPH4+LFy8a78vMzETnzp0RGRlpvC0tLQ2FhYXYsWOHcZvqx1+1TdXxO/M12rdvH2JiYtCyZUsMHz4c2dnZAICNGzeitLTU5Dnbt2+PZs2aGZ9T9mOrqaSkBF9//TUeeughk6KV7nz+qjt06BBycnJMnic0NBTJyckm5ywsLAzdu3c3bpOamgovLy+sXbvWuE2fPn3g5+dncjx79uzBuXPnFB2zkrZooaCgAAaDAWFhYSa3T5kyBQ0bNkTXrl0xdepUk+5ydzi+ZcuWoXHjxmjXrh3GjBmDM2fOmLS/rpzD3Nxc/P7773j44Ydr3ecu57DmdUGmz04lbVHCLQr/ae306dMoLy83OUkAEBkZid27d+vUqtoqKirwzDPP4Nprr0WnTp2Mt993331o3rw5YmJisHXrVrz44ovYs2cP5s+fDwDIyckxe2xV91nbprCwEJcuXcK5c+ec8holJyfj888/R7t27XDy5Em8/vrr6N27N7Zv346cnBz4+fnV+tCPjIy02W4Zjs2cn3/+Gfn5+XjwwQeNt7nz+aupqj3mnqd6Wxs3bmxyv4+PD8LDw022adGiRa19VN3XoEEDi8dcfR+22uKoy5cv48UXX8S9995rUiDtqaeeQrdu3RAeHo41a9Zg/PjxOHnyJKZPn+4Wxzdw4EDcfvvtaNGiBQ4cOICXXnoJgwYNQmZmJry9vevUOfziiy8QHByM22+/3eR2dzmH5q4LMn12KmmLEh4ZnLiLJ598Etu3b8eqVatMbn/00UeNv3fu3BnR0dG44YYbcODAAbRq1crVzVRl0KBBxt+7dOmC5ORkNG/eHN9//z0CAwN1bJlzfPrppxg0aBBiYmKMt7nz+fNkpaWluPvuuyGEwKxZs0zuS09PN/7epUsX+Pn54bHHHsPkyZOlSgluyT333GP8vXPnzujSpQtatWqFZcuW4YYbbtCxZdqbO3cuhg8fjoCAAJPb3eUcWrou1DUeOawTEREBb2/vWrOHc3NzERUVpVOrTI0dOxa//fYbli5diqZNm1rdNjk5GQCwf/9+AEBUVJTZY6u6z9o2ISEhCAwMdNlrFBYWhrZt22L//v2IiopCSUkJ8vPzLT6nOx3bkSNHsHjxYjzyyCNWt3Pn81e1L2vPExUVhby8PJP7y8rKcPbsWU3Oa/X7bbXFXlWByZEjR7Bo0SKbZeWTk5NRVlaGw4cPW2179XbreXw1tWzZEhERESbvSXc/hwCwcuVK7Nmzx+b/JCDnObR0XZDps1NJW5TwyODEz88PSUlJyMjIMN5WUVGBjIwMpKSk6NiyymVmY8eOxU8//YQlS5bU6kY0JysrCwAQHR0NAEhJScG2bdtMPkyqPlDj4+ON21Q//qptqo7fVa/RhQsXcODAAURHRyMpKQm+vr4mz7lnzx5kZ2cbn9Odju2zzz5D48aNMXjwYKvbufP5a9GiBaKiokyep7CwEGvXrjU5Z/n5+di4caNxmyVLlqCiosIYmKWkpGDFihUoLS01OZ527dqhQYMGio5ZSVvsURWY7Nu3D4sXL0bDhg1tPiYrKwteXl7GoRCZj8+cY8eO4cyZMybvSXc+h1U+/fRTJCUlISEhwea2Mp1DW9cFmT47lbRFEcVTZ+uY7777Tvj7+4vPP/9c7Ny5Uzz66KMiLCzMZCazHsaMGSNCQ0PFsmXLTJa0Xbx4UQghxP79+8WkSZPEhg0bxKFDh8Qvv/wiWrZsKfr06WPcR9WSsQEDBoisrCyxcOFC0ahRI7NLxsaNGyd27dolZs6caXbJmNav0XPPPSeWLVsmDh06JFavXi1SU1NFRESEyMvLE0JULkFr1qyZWLJkidiwYYNISUkRKSkpbnFs1ZWXl4tmzZqJF1980eR2dzx/58+fF5s3bxabN28WAMT06dPF5s2bjatVpkyZIsLCwsQvv/witm7dKoYMGWJ2KXHXrl3F2rVrxapVq0SbNm1MlqHm5+eLyMhI8cADD4jt27eL7777TgQFBdVapunj4yOmTZsmdu3aJSZOnGh2maattqg5vpKSEnHrrbeKpk2biqysLJP/yaoVDmvWrBHvvvuuyMrKEgcOHBBff/21aNSokRgxYoQUx2frGM+fPy+ef/55kZmZKQ4dOiQWL14sunXrJtq0aSMuX77s9uewSkFBgQgKChKzZs2q9XjZz6Gt64IQcn122mqLEh4bnAghxAcffCCaNWsm/Pz8RI8ePcTff/+td5MEALM/n332mRBCiOzsbNGnTx8RHh4u/P39RevWrcW4ceNM8mQIIcThw4fFoEGDRGBgoIiIiBDPPfecKC0tNdlm6dKlIjExUfj5+YmWLVsan6M6rV+jYcOGiejoaOHn5yeaNGkihg0bJvbv32+8/9KlS+KJJ54QDRo0EEFBQeK2224TJ0+edItjq+7PP/8UAMSePXtMbnfH87d06VKz78mRI0cKISqXR7766qsiMjJS+Pv7ixtuuKHWcZ85c0bce++9on79+iIkJESMGjVKnD9/3mSbLVu2iOuuu074+/uLJk2aiClTptRqy/fffy/atm0r/Pz8RMeOHcXvv/9ucr+Stqg5vkOHDln8n6zKW7Nx40aRnJwsQkNDRUBAgOjQoYN4++23TS7seh6frWO8ePGiGDBggGjUqJHw9fUVzZs3F6NHj64VxLrrOawyZ84cERgYKPLz82s9XvZzaOu6IIRcn51K2mKL4cqBExEREUnBI+ecEBERkbwYnBAREZFUGJwQERGRVBicEBERkVQYnBAREZFUGJwQERGRVBicEBERkVQYnBAREZFUGJwQERGRVBicEBERkVQYnBAREZFUGJwQERGRVP4fr/NzQnir1AEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(lossi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # calibrate the batch norm at the end of training\n",
    "\n",
    "# with torch.no_grad():\n",
    "#   # pass the training set through\n",
    "#   emb = C[Xtr]\n",
    "#   embcat = emb.view(emb.shape[0], -1)\n",
    "#   hpreact = embcat @ W1 # + b1\n",
    "#   # measure the mean/std over the entire training set\n",
    "#   bnmean = hpreact.mean(0, keepdim=True)\n",
    "#   bnstd = hpreact.std(0, keepdim=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 2.037672996520996\n",
      "val 2.107128620147705\n"
     ]
    }
   ],
   "source": [
    "@torch.no_grad() # this decorator disables gradient tracking\n",
    "def split_loss(split):\n",
    "  x,y = {\n",
    "    'train': (Xtr, Ytr),\n",
    "    'val': (Xdev, Ydev),\n",
    "    'test': (Xte, Yte),\n",
    "  }[split]\n",
    "  emb = C[x] # (N, block_size, n_embd)\n",
    "  embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)\n",
    "  hpreact = embcat @ W1 #+ b1\n",
    "  #hpreact = bngain * (hpreact - hpreact.mean(0, keepdim=True)) / (hpreact.std(0, keepdim=True)) + bnbias #batch normalisation layer\n",
    "  hpreact = bngain * (hpreact - bnmean_running) / bnstd_running + bnbias\n",
    "  h = torch.tanh(hpreact) # (N, n_hidden)\n",
    "  logits = h @ W2 + b2 # (N, vocab_size)\n",
    "  loss = F.cross_entropy(logits, y)\n",
    "  print(split, loss.item())\n",
    "\n",
    "split_loss('train')\n",
    "split_loss('val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(3.2958)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#The initial loss value that we expect\n",
    "-torch.tensor(1/27.0).log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mora.\n",
      "mayah.\n",
      "see.\n",
      "mel.\n",
      "rylee.\n",
      "emmadiejd.\n",
      "leg.\n",
      "adelyn.\n",
      "elin.\n",
      "shi.\n",
      "jen.\n",
      "eden.\n",
      "estanar.\n",
      "kayziquetta.\n",
      "noshir.\n",
      "roshiriel.\n",
      "kendreth.\n",
      "konnie.\n",
      "casube.\n",
      "ged.\n"
     ]
    }
   ],
   "source": [
    "# sample from the model\n",
    "g = torch.Generator().manual_seed(2147483647 + 10)\n",
    "\n",
    "for _ in range(20):\n",
    "    \n",
    "    out = []\n",
    "    context = [0] * block_size # initialize with all ...\n",
    "    while True:\n",
    "      # forward pass the neural net\n",
    "      emb = C[torch.tensor([context])] # (1,block_size,d)\n",
    "      h = torch.tanh(emb.view(1, -1) @ W1 + b1)\n",
    "      logits = h @ W2 + b2\n",
    "      probs = F.softmax(logits, dim=1)\n",
    "      # sample from the distribution\n",
    "      ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n",
    "      context = context[1:] + [ix]\n",
    "      out.append(ix)\n",
    "      # if we sample the special '.' token, break\n",
    "      if ix == 0:\n",
    "        break\n",
    "    \n",
    "    print(''.join(itos[i] for i in out)) # decode and print the generated word"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}