File size: 39,398 Bytes
9470d95 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 |
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Importing the PyTorch and Matplotlib utilities as before"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import torch.nn.functional as F\n",
"import matplotlib.pyplot as plt # for making figures\n",
"%matplotlib inline"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Reading all the words"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# read in all the words\n",
"words = open('names.txt', 'r').read().splitlines()\n",
"words[:8]"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"32033"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(words)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Printing the vocabulary of all the lower case letters and the special dot token"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n",
"27\n"
]
}
],
"source": [
"# build the vocabulary of characters and mappings to/from integers\n",
"chars = sorted(list(set(''.join(words))))\n",
"stoi = {s:i+1 for i,s in enumerate(chars)}\n",
"stoi['.'] = 0\n",
"itos = {i:s for s,i in stoi.items()}\n",
"vocab_size = len(itos)\n",
"print(itos)\n",
"print(vocab_size)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here we are reading the dataset and processing it. In the end of this cell, we are also splitting the dataset into three- Train, Dev and Loss split"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"torch.Size([182625, 3]) torch.Size([182625])\n",
"torch.Size([22655, 3]) torch.Size([22655])\n",
"torch.Size([22866, 3]) torch.Size([22866])\n"
]
}
],
"source": [
"# build the dataset\n",
"block_size = 3 # context length: how many characters do we take to predict the next one?\n",
"\n",
"def build_dataset(words): \n",
" X, Y = [], []\n",
" \n",
" for w in words:\n",
" context = [0] * block_size\n",
" for ch in w + '.':\n",
" ix = stoi[ch]\n",
" X.append(context)\n",
" Y.append(ix)\n",
" context = context[1:] + [ix] # crop and append\n",
"\n",
" X = torch.tensor(X)\n",
" Y = torch.tensor(Y)\n",
" print(X.shape, Y.shape)\n",
" return X, Y\n",
"\n",
"import random\n",
"random.seed(42)\n",
"random.shuffle(words)\n",
"n1 = int(0.8*len(words))\n",
"n2 = int(0.9*len(words))\n",
"\n",
"Xtr, Ytr = build_dataset(words[:n1]) # 80%\n",
"Xdev, Ydev = build_dataset(words[n1:n2]) # 10%\n",
"Xte, Yte = build_dataset(words[n2:]) # 10%"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Almost the same MLP, but we have cleaned it up to add those hard coded values into variables so we just have to modify them there"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"11897\n"
]
}
],
"source": [
"# MLP revisited\n",
"n_embd = 10 # the dimensionality of the character embedding vectors\n",
"n_hidden = 200 # the number of neurons in the hidden layer of the MLP\n",
"\n",
"g = torch.Generator().manual_seed(2147483647) # for reproducibility\n",
"C = torch.randn((vocab_size, n_embd), generator=g)\n",
"W1 = torch.randn((n_embd * block_size, n_hidden), generator=g)\n",
"b1 = torch.randn(n_hidden, generator=g)\n",
"W2 = torch.randn((n_hidden, vocab_size), generator=g)\n",
"b2 = torch.randn(vocab_size, generator=g)\n",
"\n",
"parameters = [C, W1, b1, W2, b2]\n",
"print(sum(p.nelement() for p in parameters)) # number of parameters in total\n",
"for p in parameters:\n",
" p.requires_grad = True"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here we are optimizing the NN. Same as before, just those hard coded numbers (or magic numbers as Andrej sensei calls it) have been replaced with variable names for more readability"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" 0/ 200000: 27.8817\n",
" 10000/ 200000: 2.8244\n",
" 20000/ 200000: 2.5473\n",
" 30000/ 200000: 2.8961\n",
" 40000/ 200000: 2.0967\n",
" 50000/ 200000: 2.5020\n",
" 60000/ 200000: 2.4999\n",
" 70000/ 200000: 2.0510\n",
" 80000/ 200000: 2.4076\n",
" 90000/ 200000: 2.3172\n",
" 100000/ 200000: 2.0199\n",
" 110000/ 200000: 2.3338\n",
" 120000/ 200000: 1.8767\n",
" 130000/ 200000: 2.3989\n",
" 140000/ 200000: 2.2102\n",
" 150000/ 200000: 2.1937\n",
" 160000/ 200000: 2.0843\n",
" 170000/ 200000: 1.8780\n",
" 180000/ 200000: 1.9727\n",
" 190000/ 200000: 1.8222\n"
]
}
],
"source": [
"# same optimization as last time\n",
"max_steps = 200000\n",
"batch_size = 32\n",
"lossi = []\n",
"\n",
"for i in range(max_steps):\n",
" \n",
" # minibatch construct\n",
" ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)\n",
" Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y\n",
" \n",
" # forward pass\n",
" emb = C[Xb] # embed the characters into vectors\n",
" embcat = emb.view(emb.shape[0], -1) # concatenate the vectors\n",
" hpreact = embcat @ W1 + b1 # hidden layer pre-activation\n",
" h = torch.tanh(hpreact) # hidden layer\n",
" logits = h @ W2 + b2 # output layer\n",
" loss = F.cross_entropy(logits, Yb) # loss function\n",
" \n",
" # backward pass\n",
" for p in parameters:\n",
" p.grad = None\n",
" loss.backward()\n",
" \n",
" # update\n",
" lr = 0.1 if i < 100000 else 0.01 # step learning rate decay\n",
" for p in parameters:\n",
" p.data += -lr * p.grad\n",
"\n",
" # track stats\n",
" if i % 10000 == 0: # print every once in a while\n",
" print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n",
" lossi.append(loss.log10().item())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here we plot the loss"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[<matplotlib.lines.Line2D at 0x28412485fc0>]"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGdCAYAAADJ6dNTAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAATylJREFUeJzt3XdYVfXjB/D3ZYMKqMgSFPcWEZNwDxzkT9uaWpqVpek3v2FllCPtm1iZ2TAtc7QdZTbcouTCheJIRXGBynAxlf35/QH3ei/33Mm4B+779Tw8j/fcc879HC7e876fqRBCCBARERHJhI2lC0BERESkjuGEiIiIZIXhhIiIiGSF4YSIiIhkheGEiIiIZIXhhIiIiGSF4YSIiIhkheGEiIiIZMXO0gUwRklJCW7cuIF69epBoVBYujhERERkBCEEsrOz4evrCxsb4+tDakQ4uXHjBvz9/S1dDCIiIjJDcnIy/Pz8jN6/RoSTevXqASi9OFdXVwuXhoiIiIyRlZUFf39/1X3cWDUinCibclxdXRlOiIiIahhTu2SwQywRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREclKjVj4r6qs2HcZyXfu4Znu/mjrzQUFiYiI5MCqa042nbyB1QeuIOn2PUsXhYiIiMpYdTghIiIi+WE4ISIiIllhOCEiIiJZYTgBICxdACIiIlKx6nCiUCgsXQQiIiIqx6rDCREREckPwwkRERHJisnhZM+ePRg+fDh8fX2hUCiwceNGo4/dv38/7Ozs0KVLF1NfloiIiKyEyeEkNzcXgYGBWLJkiUnHZWRkYNy4cRg4cKCpL1nlBHvEEhERyYbJ09eHh4cjPDzc5BeaNGkSxowZA1tbW5NqW6oSu8MSERHJT7X0OVm1ahUuXbqEOXPmGLV/fn4+srKyNH6IiIjIOlR5OLlw4QLefvtt/Pjjj7CzM66iJioqCm5ubqoff3//Ki4lERERyUWVhpPi4mKMGTMGc+fORevWrY0+LjIyEpmZmaqf5OTkKiwlERERyYnJfU5MkZ2djaNHj+L48eOYOnUqAKCkpARCCNjZ2WH79u0YMGCA1nGOjo5wdHSsyqKVwx6xREREclGl4cTV1RWnTp3S2PbVV19h165d+PXXX9GsWbOqfHmDOEEsERGR/JgcTnJycpCYmKh6fPnyZcTHx6NBgwZo0qQJIiMjcf36dXz//fewsbFBx44dNY739PSEk5OT1nYiIiIiwIxwcvToUfTv31/1OCIiAgAwfvx4rF69GikpKUhKSqq8EhIREZFVUQgh/ynIsrKy4ObmhszMTLi6ulbaeZ9edgBHrtzFsme7YmhHn0o7LxEREZl//+baOuAMsURERHJi1eFEwTliiYiIZMeqwwkRERHJD8MJERERyQrDCREREckKwwk4PywREZGcWHc4YX9YIiIi2bHucEJERESyw3BCREREssJwQkRERLLCcALOEEtERCQnDCdEREQkK1YdTjhYh4iISH6sOpwQERGR/DCcABCcho2IiEg2GE6IiIhIVhhOiIiISFasOpwo2COWiIhIdqw6nBAREZH8MJyAk7ARERHJCcMJERERyQrDCREREcmKVYcTBeeIJSIikh2rDidEREQkPwwnAOeHJSIikhGGEyIiIpIVhhMiIiKSFasOJ5whloiISH6sOpwQERGR/DCcABCcIpaIiEg2GE6IiIhIVhhOiIiISFasOpywQywREZH8WHU4ISIiIvlhOCEiIiJZYTghIiIiWWE4ISIiIlmx6nCiAHvEEhERyY1VhxMiIiKSH5PDyZ49ezB8+HD4+vpCoVBg48aNevffsGEDBg0ahEaNGsHV1RWhoaHYtm2bueWtEpwgloiISD5MDie5ubkIDAzEkiVLjNp/z549GDRoEDZv3oy4uDj0798fw4cPx/Hjx00uLBEREdV+dqYeEB4ejvDwcKP3X7x4scbj+fPn448//sBff/2FoKAgU1+eiIiIajmTw0lFlZSUIDs7Gw0aNNC5T35+PvLz81WPs7KyqqQsaVl5AIDs/KIqOT8RERGZrto7xC5cuBA5OTkYOXKkzn2ioqLg5uam+vH396+SslxIzwEAzNp4ukrOT0RERKar1nDy888/Y+7cuVi3bh08PT117hcZGYnMzEzVT3JycjWWkoiIiCyp2pp11qxZg5deegnr169HWFiY3n0dHR3h6OhYTSUjIiIiOamWmpNffvkFEyZMwC+//IJhw4ZVx0sSERFRDWVyzUlOTg4SExNVjy9fvoz4+Hg0aNAATZo0QWRkJK5fv47vv/8eQGlTzvjx4/HZZ58hJCQEqampAABnZ2e4ublV0mUQERFRbWFyzcnRo0cRFBSkGgYcERGBoKAgzJ49GwCQkpKCpKQk1f7ffPMNioqKMGXKFPj4+Kh+pk2bVkmXQERERLWJyTUn/fr1g9Azperq1as1HsfExJj6EkRERGTFuLYOERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREcmKyeFkz549GD58OHx9faFQKLBx40aDx8TExKBr165wdHREy5YtsXr1ajOKSkRERNbA5HCSm5uLwMBALFmyxKj9L1++jGHDhqF///6Ij4/Hf//7X7z00kvYtm2byYUlIiKi2s/O1APCw8MRHh5u9P7Lli1Ds2bN8MknnwAA2rVrh3379uHTTz/FkCFDTH15IiIiquWqvM9JbGwswsLCNLYNGTIEsbGxVf3SREREVAOZXHNiqtTUVHh5eWls8/LyQlZWFu7fvw9nZ2etY/Lz85Gfn696nJWVVdXFJCIiIpmQ5WidqKgouLm5qX78/f0tXSQiIiKqJlUeTry9vZGWlqaxLS0tDa6urpK1JgAQGRmJzMxM1U9ycnJVF5OIiIhkosqbdUJDQ7F582aNbTt27EBoaKjOYxwdHeHo6FjVRSMiIiIZMrnmJCcnB/Hx8YiPjwdQOlQ4Pj4eSUlJAEprPcaNG6faf9KkSbh06RLeeustnDt3Dl999RXWrVuH119/vXKugIiIiGoVk8PJ0aNHERQUhKCgIABAREQEgoKCMHv2bABASkqKKqgAQLNmzbBp0ybs2LEDgYGB+OSTT/Dtt99yGDERERFJUgghhKULYUhWVhbc3NyQmZkJV1fXSjtvwNubVP++smBYpZ2XiIiIzL9/y3K0jiWUlMg+oxEREVkFhpMy8dcyLF0EIiIiAsOJSg1o3SIiIrIKDCdlmE2IiIjkgeGEiIiIZIXhhIiIiGSF4aQMW3WIiIjkgeGEiIiIZIXhpAw7xBIREckDw0mZy7dyLF0EIiIiAsOJyk+HkgzvRERERFWO4aTMjYw8SxeBiIiIwHCicisn39JFICIiIjCcEBERkcwwnBAREZGsMJwQERGRrDCcEBERkawwnBAREZGsMJwQERGRrDCcEBERkawwnBAREZGsMJwQERGRrDCcEBERkawwnBAREZGsMJwQERGRrDCcEBERkawwnBAREZGsMJwQERGRrDCcEBERkawwnBAREZGsMJwQERGRrDCcqLl6O9fSRSAiIrJ6DCdqxq88bOkiEBERWT2GEzVXbt+zdBGIiIisHsMJERERyQrDCREREckKwwkRERHJCsMJERERyQrDCREREcmKWeFkyZIlCAgIgJOTE0JCQnD4sP4huIsXL0abNm3g7OwMf39/vP7668jLyzOrwERERFS7mRxO1q5di4iICMyZMwfHjh1DYGAghgwZgvT0dMn9f/75Z7z99tuYM2cOzp49ixUrVmDt2rV45513Klx4IiIiqn1MDieLFi3CxIkTMWHCBLRv3x7Lli2Di4sLVq5cKbn/gQMH0LNnT4wZMwYBAQEYPHgwRo8ebbC2hYiIiKyTSeGkoKAAcXFxCAsLe3ACGxuEhYUhNjZW8pgePXogLi5OFUYuXbqEzZs345FHHqlAsYmIiKi2sjNl51u3bqG4uBheXl4a2728vHDu3DnJY8aMGYNbt26hV69eEEKgqKgIkyZN0tusk5+fj/z8fNXjrKwsU4pJRERENViVj9aJiYnB/Pnz8dVXX+HYsWPYsGEDNm3ahPfff1/nMVFRUXBzc1P9+Pv7V3UxiYiISCZMqjnx8PCAra0t0tLSNLanpaXB29tb8phZs2bhueeew0svvQQA6NSpE3Jzc/Hyyy/j3XffhY2Ndj6KjIxERESE6nFWVhYDChERkZUwqebEwcEBwcHBiI6OVm0rKSlBdHQ0QkNDJY+5d++eVgCxtbUFAAghJI9xdHSEq6urxg8RERFZB5NqTgAgIiIC48ePR7du3dC9e3csXrwYubm5mDBhAgBg3LhxaNy4MaKiogAAw4cPx6JFixAUFISQkBAkJiZi1qxZGD58uCqkEBERESmZHE5GjRqFmzdvYvbs2UhNTUWXLl2wdetWVSfZpKQkjZqSmTNnQqFQYObMmbh+/ToaNWqE4cOH44MPPqi8qyAiIqJaQyF0ta3ISFZWFtzc3JCZmVmpTTwBb2/S2nZlwbBKOz8REZE1M/f+zbV1iIiISFYYToiIiEhWGE6IiIhIVhhOiIiISFYYToiIiEhWGE6IiIhIVhhOiIiISFYYToiIiEhWGE6IiIhIVhhOysnNL7J0EYiIiKwaw0k5sp/Ln4iIqJZjOCnnRHKGpYtARERk1RhOyhn77SHcyS2wdDGIiIisFsOJhPTsPEsXgYiIyGoxnBAREZGsMJwQERGRrDCcEBERkawwnEhQQGHpIhAREVkthhMiIiKSFYYTCflFxZYuAhERkdViOJHwa9w1SxeBiIjIajGcSMjNZ80JERGRpTCcSFCwPywREZHFMJwQERGRrDCcEBERkawwnBAREZGsMJwYcPJaBn7j6B0iIqJqY2fpAsjRpZs5qn+P+HI/AMDHzQk9WnpYqkhERERWgzUnEo4lZeBcapbGtkS1wEJERERVh+FEhz3nb1q6CERERFbJqsPJyG5+li4CERERlWPV4aSdj6vO54SoxoIQERGRilV3iLXRMxVs1JZz8HR1rMbSEBEREWDlNSeOdvov//W1J6qpJERERKRk1eGEiIiI5IfhhIiIiGTFqsOJKX1euVAxERFR9bDqcEJERETyw3BCREREsmJWOFmyZAkCAgLg5OSEkJAQHD58WO/+GRkZmDJlCnx8fODo6IjWrVtj8+bNZhXYks6nZWPr6VRLF4OIiKhWM3mek7Vr1yIiIgLLli1DSEgIFi9ejCFDhiAhIQGenp5a+xcUFGDQoEHw9PTEr7/+isaNG+Pq1atwd3evjPJXq8Gf7gEArH35YYQ0b2jh0hAREdVOJtecLFq0CBMnTsSECRPQvn17LFu2DC4uLli5cqXk/itXrsSdO3ewceNG9OzZEwEBAejbty8CAwMrXPiKGtbZx6zjzqZoLgr45voTePn7oxCcVpaIiKjCTAonBQUFiIuLQ1hY2IMT2NggLCwMsbGxksf8+eefCA0NxZQpU+Dl5YWOHTti/vz5KC4urljJK4Grk73Zx97JLUDfj3dj4bYErI+7hu1n0nDl9r1KLB0REZF1MqlZ59atWyguLoaXl5fGdi8vL5w7d07ymEuXLmHXrl0YO3YsNm/ejMTERLz66qsoLCzEnDlzJI/Jz89Hfn6+6nFWVpbkfpa0Yt8lXL19D1/uTlRtKy5hzQkREVFFVflonZKSEnh6euKbb75BcHAwRo0ahXfffRfLli3TeUxUVBTc3NxUP/7+/lVdTIMWbj+v8bi4xEIFISIiquVMCiceHh6wtbVFWlqaxva0tDR4e3tLHuPj44PWrVvD1tZWta1du3ZITU1FQUGB5DGRkZHIzMxU/SQnJ5tSzCqReb+wUs+Xk1+E09cz2U+FiIioHJPCiYODA4KDgxEdHa3aVlJSgujoaISGhkoe07NnTyQmJqKk5EFVw/nz5+Hj4wMHBwfJYxwdHeHq6qrxIzfSCxobHzSGfb4X//fFPuw8m15pZSIiIqoNTG7WiYiIwPLly/Hdd9/h7NmzmDx5MnJzczFhwgQAwLhx4xAZGanaf/Lkybhz5w6mTZuG8+fPY9OmTZg/fz6mTJlSeVchE7vP3TR636tlnWf/PnmjqopDRERUI5k8z8moUaNw8+ZNzJ49G6mpqejSpQu2bt2q6iSblJQEG5sHmcff3x/btm3D66+/js6dO6Nx48aYNm0aZsyYUXlXYQHnUrQ76Z64llH9BSEiIqplTA4nADB16lRMnTpV8rmYmBitbaGhoTh48KA5LyVLmfeLsDtBu5bEnN4jXFCQiIhIE9fWMcOnO89Lbt90MgXn07JxOydf8nkiIiIyzKyaE9JNOcX9lQXDLFwSIiKimonhpJoJIfDsikOqxwrpYT9ERERWi806VUQ5f8m6o8kYungPrt0tHZ2Tnp2P/Ym3LVk0IiIiWWM4qSLd50cj+mwa3vr1JM6lZmPeX2cAACXlJl1LTM+xRPGIiIhki+GkitzMzseL3x1VPc4rkp7v/tT1TMntQgikZ+dVSdmIiIjkjOGkGhWXCEjNVv/MN7GI2nIWp69not/Hu7HlVAre/u0Uun8Qjc2nUow697Z/U/Ht3kuVXGIiIqLqpxA1YHGXrKwsuLm5ITMzs9Knsg94e1Olnk+fhnUc4FffGSeuSdeWSGnlWRc7IvpqbBNC4PPoRLTyqotHOvkAeHAdf07tic5+7pVWZqq9Fm1PgJuLA17s1UznPkIIdtomIrOZe/9mzUk1up1bYFIwAR5Mc6/u0OU7+HTnebz60zGtOVVucY6VSlNQVIJF2xMQd/WOpYtS6a7ezsXnuxLx/t9ndO5zMzsfPRbswifbE/Se61jSXZxIzqjkEhKRNWM4kbmC4hJcSMtWPc64V4CozWdVj4P/t1Nj/7d+PVVtZavtvo+9gs93JeLJpbGWLkqly80vNrjP0piLSMnMwxe7EnXuk5NfhCe+OoBHl+xHgY5+VUREpmI4qQH2Xril+vf0dSe0al/Oq4WXWzn5uHwrt9rKVptZciTVyWsZePn7oxZ9L8uPLJOSdb9Q9e+CYoYTIqocDCc1TPS5dK1tyllplV5YfaRCr3Eg8RbWHU2Gru5I+UXFyM0v0touhEB+UTFKSszrxnTmRhbCP9uL3WrXeOlmDm5mG99UJYRQlXvfhVt4ftVhXM+4jxPJGUhMzzZwtH6HLt3GkE/34PDlqm/mGfHlfmw/k4YXv6vYe1nZlL/bL3ddwE+Hrlq4NGTNis38nKGageGkBlh3NNmk/dW/bd/NLcBnOy8g+Y5235WUzPsY8eU+/BZ3TbUtPSsPY749hLd+PYnNp1IBABfSsjF93QlcvV163of+txMd5mzDvQLNgPL8qiNoM3MrRizZZ1J5lV7+4SjOpmRhQlm4SsvKw4BP/sFDH+w0cGQpIQSeXHoAo74+qJqJNybhJl5YdQSPLtmPsEV7DJ9Eh30XbmHUNweRkJaNkV9XXzOP1PtmrLzCYizcloBjSXe1njOnj2vsxdsIen8Hvtx1AQu3n8e7v5826riSEqEzsK7YdxlTfz7GGw2ZJCXzPjq/tw1z/jDub5BqHoaTGuBcajZOXsvAin2XjT5mxb7LSEjNxpu/nsSnO8+j90e7cSDxlsY+8/46g5PXMjF9/QnVNvWmjN0JpTUYTyw9gN+OXcP4lYcBAFl5paHk1LVMjZvOP+dLV2o+fT3LxCsspd5EAAA7z6aZdHx6dj6OJWXg8JU7qjICQIJas9fPh5LMKpv6kgPVSVGBdau/2XMJX+5OxBNfHTDvtcu99OjlB5FxrxALt0svfClFCIHHv9qP4V/ukwwo7/99Bn+fTMHW06lmlZGs0zd7LiG3oBjfxbL2rrZiOKkhRny5X+/IivLe//sMhizegz1lgQEAxnz74AZ79XYutkjcENRvHyVCIDUzD9llN/or5UYOjfrmIIZ+tkdvM86ZG1n4PPoCMssFD2OU/2aenpWHPedvIuNeAX44eBUZ9wpMPuc7v5/Cl7suoMiI/hHVOcj+eNJdvLD6CC7erLx+Lhcq2GdG/frNnXEg414hTlzLxL83svSOJMvJN/3vg6zHllMpOHSJy35YEy78V8vp6qTY9+MYye3qtRcbjl3HhmPXNZ5XrhGkdD4tB+nZ+fB2c9I611cxifhoa0LZftn4ckxXpGbmYfYfp/FEVz/sOJOGx4J80btVI4PXUVhcgtAFuzSq/zefTMEvLz9s8NjyFm4/jxIBvDawlcnHmirzXiGy8grh38BF736Pl9VuXL6Vi91v9HvwhJkVJ4cv38FfJ26Yd7CEqm51kf9sS7XT7Zx8zN98Ds9098dDAQ0sXRxJV27lYvJPx0r/XbbaO/9eaj+GEyuz5VQKFu+8oLX9+VWHce3ufYMjVHp9uFtr28NR0Vrbyk9ud6isE2nkhpPYnXAT28+UNtn8dqy0v0s9RzuNWpvyNRut3t2i9RqxZd+k/oi/jvjkDLzcp7nqudM6lgVQ+jXuGl4b2AqpmXnYcjoFTwX7oZ6Tvd5jdPnuwBW4Otvh8SA/recC520HAOyb0R9+9fUHFABao3PUs0nm/UIs2p6Ax4IaI6hJfb3nMaVfzLu/n8IHj3cyen9jJN+5hx1n0jCko7dR+/NeYxlz/zqDP0/cwG/Hrqlu/HKTkml9y3jcyLiPiHXxeKFnMwzuYNz/odqG4cTKKL+BlBeTcFNye2VRzoGxW8frZJcb/bPh+HXJ/cpbdzQZb/16EgDQRK12Yuy3hvuIvLn+BNaXdQZevPMCDr87EI52tjh65Y5RH4ifbE8oa14qrW1q4+WK9r7SMyAeT8owKpyUl1/2e7t6OxfDPt+HnPwifBd7FVcWDMOq/ZexaMd5bJnW26xzK/10KAnzHu2I72OvILRFQ9R1tEM9R3uNPifxJk6yNvjTPbhfWIyT1zSPS8/Owys/xGFM9yZ4upu/avu3ey9hdPcmZl8DmUfZyZ3kZebG0zh46Q4OXroj29BY1RhOqFpk3i80qp+H0kwjR4IogwkAkzpVCghVMFGWb/6ms5j7aEc8tay01iHQz03vOcpPTvbI53vx4ZOdMOqh0pusvn4a+qaFv5ur2Zfmx4NXMXOj5u9j3ZFkzC1b6brXh7s1PsDyiwxPsFZY7r1o8c5mrX3GhjwIC9/HXpE8j64aqvuFpWWILddPYOG2BBxPysDxpAyNZoSLN+V5k/wj/jqW/XMJy57tiqYN61i6OGQlbuea3p+uInYnpMPVyQ7BTeXTtMcOsVRtWko0zehizoReh0yYfyT5zn2tbWuOaA7ZvnZXex+lz6O1m8YAYMZvp3A3twALtpzDO2oB63rGg3PtTkhH4Nzt2PavdJgKen+HxuPywQQA3vrtpNY2ANj+byrazNyqtT2vsBjJd+4hryw0GDPi5icjRja9/EOc6t8JqVn48eBVjQ7SGn1VFMC6ow8CYb+FMQbPr27V/sv4TKJJsipNWxOPsylZRg+bVjp46Tam/HQM6VnybpK4Y0an8uomleFrwJJwNUZqZh4mrDoiu5mwWXNCVCa/qAQvqk1gl69nOvZFO3Tf3Pt/EqNq6lFasOUcXuzVDPa2NpiwqvQ1XvkhrtKrbP/zy3HJ7TEJ6Zj0Y2mT3gePd9QYxWUMY5q5lB9utjYP7ibqE+jtPX9L6xhTKGuKnuja2GAH48qWW6A96aA+z3xzEEBpLda34x+qiiKZZPOpFDTzqIN2PprNjlIhncxXXCJwOycfnq7aAwTkKk2mAZo1J0Rq1GfgzZGYBdcY5YOJ0g865mSojsUa/1QbuWNqLQAAk2bFjdwgvb6T+nw6plIfiq5sMqoJ9NW+VZfDl+/g1Z+OIfyzvVX2GmuPJOHJpQdwx4jmiNiLt/H3yYqNJNNXb7J8zyUs++dihc5vrudXHUb3+dE1atizXOugGE6Iqsm8v89g6GLNWWrvFxRj3l/Gz1+jS1ZeIf675rjO2h7lbL81VeDc7Sbtn51XiOdXHcavav2KLEEOrQ/nUs2bFNEUM347hbird7F4p+HmwtHLD2Lqz8dxxch1o8q36ggh8L1a0FdvMs3NL8IHm89iwZZzWn23jLXrXBq+ikk0q+lIuQ7a9wcrNjmc+VMv1h4MJ0TV6Fyq5vo+7WZv1ajVMFfn97ZjY3zlzWtiCVtOpSD2ovY3TkNT28ddvYPnVhzSWDvpmz2XEJNwE29UoLbGWInp2Xprls6mZGHEl/uw94JxTWmHL9/BL4fNm8m4OuXmFyFyw0mNJkL12sYfD17F/M1ndd7kU81sTkjL0qxp7LlgF8I/24vUzDwUqf2thERFSy7/kJNfhD/iryM7T7qG84XVR/HR1gSNBVcrW3ZeIT6PvqA16WJK5n1ErI3HKQNTIRhyJ7fA6HAl1/47DCdEJAuTfzqG0csPam0vP7KovCeXxmLvhVuqNZkAYF+i9I3l6u1cbDqZUqkfyGGL9mDk17GqG2H5c7+4+ghOXsvEcysOG3W+kV/HInLDKZOaBjadTMGb609IjtSqqnvPkt2J+OVwMsatfHBd6sstzNx4Gt/suWTyMHRznE3JQtSWsxrbCopKME9iVu3/ronHtDXxeH2t/uCaWoH5VeKuaK9npe6DTWexaMd5DPzkH43t09bEY8Px6wYDeWJ6Nlbsuyz5fv9+/Bq6vr8D8zeflThSmzyjCcMJEcmMen+ET3ecR5d5mk06um626jeT40kZkvv0/TgGU34+hk2nUipczvIulTVTqK8Kfq+wSO+w0PsFuvvPXDVh0ccpPx/D+rhrWBZzSbXtRHIGHluyH3P+/Fe1be+Fm1i13/g1uvQxdnScrr5bxoYm9SH3+uZlkfpdStVoKdfsMrR2l6jAbdtQrVDcVc3w8svhJOw8k4ZLRi5fEbZoD97/+wy+3XsZmfcK8cPBq6r+Pu//XRpKlu+Vfp9v5+Rj1sbTBieqtDSGEyKSlak/Pxhx9Fn0BeQVGjesXHmzk7pJXb6Vi+nrHnxTnvrzcXyzR7vTZF5hsWStiq6wo045hFp9osHMcp2j/4h/MLng97FX0G72Vt39YiTujflFxdhz/iYe/2o/jl7RvvF+uvO8avjyY1/t16q1eG7FYcz96ww2HDO+L86i7QlYsjtRa3v5GyxQWms0f/NZ/KBjXhx1EevisVutA7oxPtl+HsU6Us32M2n46ZBmXw9Da3o9ufQAcnWEp8+jEyVv4PcKihC26B+8pxb6DLl4M0e1kGp5ienZiNxwCi99f9To8ynFJ2fgtTXHMWvjaUyUOP6vEzfQIyoa+9SaqN75/RR+OHgV//dF6erxMm3VYTghInnSNcTxzxPSswcr+xuM/Va7aWjs8oOqpRKU5m8+pzHUOSXzPtrO2qqq+biRIT3SJiXzPiI3nEKfj3ZrBIQJq4/gXrkhx1l5RRoZY9qaeMRdvVu2xlTpze2N9Scw9tuDWpMU3si8jwVbzqHb/3ZiZdmK5P0/jsG4lYdxPClDNVlgeTFlfUD03XQi1p3Aj+U6bUZtPotd5zRrE9Kz8vD5rkR8vC1B69qkbDh+Hd/suYRZf2jfuDPvFWLQogfNGCmZeRpNcbqod5z988QN/Odn6VmuAajW8jJW3NW7Wr8HpesZ91U3cHW/H7+OxPQcrD5wBYDhPlEAMPCTfzBh1RHJmpx0tb/BWznStWxFxSU6myKVq8HHXb0LIYTGiKn//HIcNzLzNFZVTyjX702uGE6ISHbm/XUGIfO112wCgCW7L6K4REiuSl1cInBMopbjho7+A5n3S89xIS0boVG7AJTWfAS8vQk9FuySPKbXh7vxy+EkJN25pxUQJJeBKHdP2Xo6RWs9qv2Jt7G3XD+ZxTsvYNk/F3ErJx/z/j6DvMJireu4LhGgikuEarI9fcpP7vf1nkt4YfVR/BZ3DV/uuoCCohKN0V/ryybQS8vKM2kUlHK9qM+iL5i8UvaOM2laEw5Kvb/6LPvnIrJ0dH4FgKgt5wwO5//75A20fncLguZt15gI8KdDV9Fu9lbEJKSj14fSfy/qRn4di6LiEo2J5Zb9c0n3AQC2/ZuKlu9uQbNI7Vmcy1u43bRwdulmDiI3SE/oaGlWH076tzG8Ii4RVa+VBvpFtHhnM7rM26E1rb5Ufwr1UTzlfbOn9MbwRNmq0PqcupaJf29k6v2mrN50pFR+tmNdfVAOJN7SOxQ3uNzMwQAwUqL2JHLDKXScs03neQyZvv4EFm4/j9UHNH+XBy6WhqeQ+dEmjYKa/ce/KCkROHU9Q+c+WXmFWiueA9BZq2GKBVvOqebe0RXaDA2BnvrzcRQUl+DuvUKNmo53fz+NgqISPL/qiNacNvt1dMpu+e4WnE97ENIMTYj4itoszOWVH3K8ZLdp87uM/DpWoyzG1AJVF6sPJ89wsTGiGmt2ueaD/23SHqGgq/kDKJ1Ov7hEaC08KWX4l/sw7HPtan51xkwQt+GYdLPU8r2XJVcMV8qV6EsjVXMCQGNIrbk2HLsuOXW8OR75fC+O6BnBEjh3O3p9uFvjeopLhKrJoqJ2lq2CPuxz6Yno7heUIDe/yOQ+MPqM/fYQbufky26ornptWPlmpPNp8mny4fT1RFSr6ZqxV2ncSsMrWFujc6nZ6PXhbtXjE8mZuG3mbMbl5/dRJ4RQ9Y+Ju3oXnvUccelmLg5LdPg1V35RCY4n3dW5wORvx65p9UlSWrDlnNmvG/y/nXi4eeUupqfeSXf7Gf0jjtSlZ+fBs56T3qUoKiuMVgaGEyKyavsTa85U41JMWe27IlKz8hD8v52Vfl71vhSLd5zHazrWh6qox41oupNS0anwD16qvJC1ZHei2bMed/8gGj+82L3SylLVrD6cyKzGjYjIJKas9i13l4yc0t5afbzNtA6v5RmaCFAho4nzrb7PCREREQElMvq2znBCREREiDYwa251YjghIiIinZPAWYLVh5PWXnUtXQQiIiJSY/XhpHkjhhMiIiI5MSucLFmyBAEBAXByckJISAgOHzZuKfA1a9ZAoVDgscceM+dliYiIyAqYHE7Wrl2LiIgIzJkzB8eOHUNgYCCGDBmC9HT9M+tduXIFb7zxBnr37m12YYmIiKj2MzmcLFq0CBMnTsSECRPQvn17LFu2DC4uLli5cqXOY4qLizF27FjMnTsXzZs3r1CBiYiIqHYzKZwUFBQgLi4OYWFhD05gY4OwsDDExupev2LevHnw9PTEiy++aNTr5OfnIysrS+OHiIiIrINJ4eTWrVsoLi6Gl5eXxnYvLy+kpqZKHrNv3z6sWLECy5cvN/p1oqKi4Obmpvrx9/c3pZhERERUg1XpaJ3s7Gw899xzWL58OTw8PIw+LjIyEpmZmaqf5OTkKiwlERERyYlJa+t4eHjA1tYWaWmas8ilpaXB29tba/+LFy/iypUrGD58uGpbSUnpIlV2dnZISEhAixYttI5zdHSEo6OjKUUjIiKiChA1dfp6BwcHBAcHIzo6WrWtpKQE0dHRCA0N1dq/bdu2OHXqFOLj41U/I0aMQP/+/REfH8/mGiIiItJi8qrEERERGD9+PLp164bu3btj8eLFyM3NxYQJEwAA48aNQ+PGjREVFQUnJyd07NhR43h3d3cA0NpORERElpN8976li6BicjgZNWoUbt68idmzZyM1NRVdunTB1q1bVZ1kk5KSYGNj9RPPEhER1Sj3CoosXQQVhZBTI5MOWVlZcHNzQ2ZmJlxdXSv9/AFvb6r0cxIREdUkHz3ZGSMfqtzuFubev1nFQURERLC3U1i6CCoMJ0RERAQFGE6IiIiIJDGcEBEREQTk0wWV4YSIiIjYrENERETywpoTIiIiIh0YToiIiEhWGE6IiIhIVhhOiIiISFYYToiIiEhWGE6IiIgIclppj+GEiIiIoJDPNCcMJ+raetezdBGIiIisHsOJmv5tPdHKs66li0FERFTt2KwjYyHNG1i6CERERFaN4aScNt6uli4CERGRVWM4KaeBi4Oli0BERGTVGE6IiIhIVhhO1LTx0hytU8/JzkIlISIisl4MJwD+nNoTc4a3x4hAX43t+94aYKESERERVS85jdZh1QCAzn7u6OznrrXdzcW++gtDRERk5VhzUo6rM/MaERGRJfFOXE6vlh4YF9oUbcuGFLf2qovzaTkWLhUREZH1YM1JOQqFAvMe7YgxIU0AAJP7tbBwiYiIiKwLwwkRERHJCsNJBXw6KtDSRSAiIqp1GE7MNCLQF48H+Vm6GERERLUOwwkRERHJCsOJAV2b1Ld0EYiIiKwKw4kBTRvWwc6IviYd83Kf5lVUGiIiotqP4cQILT3ram3r1cpD5/7vPNJO53OLRhrXiTa8o7dR+xEREVUGGc1ez3Biqie7+mHZs8F4qqv+zrAuDraS23u21A41Ye08tbYtfTYY7w1vr3rsYMe3ioiIrAPveCbycXPC0I7esLFR6N2vQR0HrW0KBeDqpL1ez+NBfvCr76y1/aFmDVT/fjakqRmlrdneHNIGzT3qWLoYRERUzRhOqsG347rB29UJv0x8GM4Ottjxeh+81KuZ6vkGdRzgYKv/rRjRxRfP9wgw6XWf7xGA6YNaw9leuhZHzj58shOm9G+JXW/0M+k4dy7WSERU43FtHRMp9FeYSApr74Ww9l6qx6286qGFWj+Wh5s30Gjr2zejv9Y5uvi7o0WjOigoLsHxpAycTcky+LrvjegAAJjYpznyCovRZd4OjecD/d2x/pVQtJ65xcQrqlqfPB2IJ4PNm0NmzvD2eH3tiUouERFR7SeEfHqdsOakknVt4g4A6B5Q2iTjZC/9K+7TuhEAwKOuIxTlEo9ffRfJY+o52WP+450QotbcozS6u7/GY896jqp/O9nbwt3FAWtfflhjn+mDWmv1ZbG3NSN9meGFns0M72Skjo1dVf/u1Nit0s5bEU+bGa6IiIg1J5Vm4dOBSEzPwQs9AwAA7z3aAc086mB4oK/k/o3dnXHk3TDUcyp9CyqSWD9+qjNsbRT45XAygNJOu1P6ay9YGNK8IQL93HDiWmbpa0qcq2NjNxxPyjC7LMaYO6IDxvcIwMr9lyWf93J1Mul8K8Y/hPf/PoMhHbzR0rNeZRSxwqaFtcL6uGuWLgYRUY3EmhMT6apX8K/vjLfD28Kz7Mbq6mSP/wxshQA9HTob1XOEUwX7g6ye8BCeCvbTqAH5ZGQgmjfSHv4MwGC7lF99F+x4vU+FyvRIJ2+ce3+ozud11SYp9WzZ0KTX83J1wpdjuuoMgkREVLOYFU6WLFmCgIAAODk5ISQkBIcPH9a57/Lly9G7d2/Ur18f9evXR1hYmN79yTAftwc1C/3aeGo1CxlLWVvz66RQjW2tvOqhrqP5lWrdAxrAyd4WG6f0NOm46Ol9ceTdMLOvp7aYUFb7VlWOvBtWpecnopqplZc8ap4BM8LJ2rVrERERgTlz5uDYsWMIDAzEkCFDkJ6eLrl/TEwMRo8ejd27dyM2Nhb+/v4YPHgwrl+/XuHC1yamTJM/vkcAxoQ0wYrx3VTbBrT1hI+bEx7ppH/yNqnbfrcA7T4shhgaXQSUduINbmr8dbVoVBeN1PrKGKNvWd8dda8NbGXSOSrilb6VPxtwHQfTg+HF+Y8Yva9HXe1h7kREnWXSZw8wI5wsWrQIEydOxIQJE9C+fXssW7YMLi4uWLlypeT+P/30E1599VV06dIFbdu2xbfffouSkhJER0dXuPAWUUXf6ueM6IDXBrQ0aqp8J3tbzH+8Ewa2ezACyMXBDvtmDMCSMV31HtukwYPOtlJ9ThqWzc8y9uEmes8jNY+LqdSHUxvjtQEtNR7Pe7QDPn8mSGu/iEGtK1QuXdQ73ipFhkvPBlzfxfzfj7F/YmNCHrxHtgbm3VEa1tmnSmqmeuuZMZmIyFQmhZOCggLExcUhLOxBtbCNjQ3CwsIQGxtr1Dnu3buHwsJCNGig+9t6fn4+srKyNH5qOzdne0QMbqMxVb6pTSu2NgqDN545arPOqls6tivC2nnh9bIb+xuD2+DHF0OwdKz+sKOk3o/EVq1WRV9H39bexlUhfjW2K6YNbIXXB7VGY/cHk9WNCw2Am4F5TTa82gO/Te6Bda+E4q+pvbRGLAHAo10M91Vxd7E3qrZIqY6jHba/3gfR07XD5qhu/hJHmE5q1JYhlRlLAho+CLorn38IP70UYvSxUu8DVb49b2pPS0DVq72P9pcaMsykcHLr1i0UFxfDy8tLY7uXlxdSU1ONOseMGTPg6+urEXDKi4qKgpubm+rH379yPswrok7ZdPT92mg3I1SVpg3r4M0hbTD/8U6Vds6GdaWbTcI7+eDb8d3gXvaN397WBr1aecBZxzT85bXyrIcp/VugY2NXg1P7qxg5QOmRTj54fVBrKBQKyXWOpByfNQjb/tsHXZvUR3DT+ujerAE6+bkhpHlDPPew5my73ZrWx3/UamUm9W2BH17srrHPr5N6GFdYNa296qGFRMfktj4PQplUs5e+ALFrel98/FRn/P2fXiaXxxjG9nd5/7GOGNbZR/XY3tYGzRvp7vxd/n0LaW5ap2cyj5V335KFmrROmpz+Xqp1tM6CBQuwZs0a/P7773By0j1cNDIyEpmZmaqf5OTkaiyltAORA7FlWm+dfUPqSUxLXxmm9G+pUX1fqYwIB7pqYtSbhwBg6bNd8eaQtvj7P701Ak1lT+nz8dOdMaqbP/6aqv/mXL+OA9roqJmZ+X/tsHrCQ6rHXq5OmD64DY7PGoQz84bg7fC26N1KM4SWbzbxdnXCvEc7mFT2hnUcsHRsV42h0j+9FIKJvTWbt3R1SmvmUQfNG9XF09380bGxm8n9c4xhSh+h8lz09JX52YRaFao8fvWdq7zJ7auxXc2qxTNF1ybu+HfukCp9DbmytVHgyoJhBvsT1jYmhRMPDw/Y2toiLS1NY3taWhq8vfX/4hYuXIgFCxZg+/bt6Ny5s959HR0d4erqqvFjaW7O9mgnUT338VOdETGoNdr7Wr6MVc3BzgZ/TOmJ717ojqZqVfrn/xeuc+I4KRW5AXrWc8KHT3VGJz/zO2452tmiXxtPrHy+G6YNbIVBZbP31q/joPcGqy42cgDGhQYYtW+Hsr+Nxc90QXgnH43nnOxt0cFX81r+r7PmPrqENm+I18Na46uypre3w9sCgOp6pBia3l9hQsNP+X3dnO3x5RjNPkBrX34YK5/vphpir+6vqb0wc5juFbyri3pTYW2jUCjww4vmB8OHmxsOHY908qnQa6jTtdTGque7o04FRhAaw9TlQYxV0dqIxA/CAQBzR3SshNLUHCaFEwcHBwQHB2t0ZlV2bg0NDdV53EcffYT3338fW7duRbdu3XTuVxM93c2/WkeHWNLJOYMR6O+Ovq0b4SG1ET76VkxW73JydGYYtkzrrZooTVh4ge4Bbb1UzUWm0nfM/x7T/BDZ8GoP7H6jn6o2xtCrKRQKVaDR2C6x37SwVnikLPBM6tsCVxYMw5NdG2sd+97w9ujdygPTB7UBAEzp3wKPB2nvV9EP0v/r7IuPnyr98jG6exOENG+IAW2lw1InPze81LvyRzuZqk9r/TULbs6WXa/J2KbMqrDmZd2f6+qMWTV9+bhuuBylf1SZrr8/Q33LlMLa6Q7mUrqpfVFSLvdR2SraAV15fGXUlDb3qIOHAuqrZijX9VpyYHKzTkREBJYvX47vvvsOZ8+exeTJk5Gbm4sJEyYAAMaNG4fIyEjV/h9++CFmzZqFlStXIiAgAKmpqUhNTUVOTk7lXQWZxdQPPfUJ454K9sOnowLxz5v9jD7eo66jZO2TXJlaHd7Y3Rkn5gzGs+X6tDja2aKZnsn4jA1pLo6mT9h3eu4QHH53IJ7v2Qw/vBiC+mWjrN4c0hafjuqitX/5j6axIU3g6ybdBCu1kjZQGthjIwdg/uPS3/TKNwmqk5rZuLK8MVh6FJdUvyB1P080vlagldr/qfKjywwZHuircbNUerlP5Qc4ZQ1XHSP7lAHAExJhVp9L5Ya3D2rvpXHzU86ObYo1BjpSDw80rtZR6ZORgejU2M3gKEdjSf1tS013YMiyZ0vL079cH8cu/u5mlUvJzlaB9ZN64PsXuls09BrD5HAyatQoLFy4ELNnz0aXLl0QHx+PrVu3qjrJJiUlISUlRbX/0qVLUVBQgKeeego+Pj6qn4ULF1beVZBJ9r7VH39O7Ql/PTcJJV1/wDY2Cjwe5IemDXXfdGu6xeVu3o2NaLoy5lu2vi8n6nPXKC0Z0xVtvOpplccYdR3t4FlPd/+uyf00w4B62Z4IaowPHu8kGWKA0oD6St/m+O6F7lrP+bg5a30Le6XsJvuunqacqlwbqWnDOiaNgPvk6UBcWTBMq9nNWG7lhpOPD22q0depvAYu9vh1snbH64o0g+ryf519cWXBMOydMUBju76O1o9L1MjpY2NgeLspo7uUHtbTkXrTa70wItBXYxSZPs8+3ARNG9bBX//ppdG5uyJGd9fuH9ixsRt2vN4HcTMNT37Yxd8dP08MwdCOPriyYBhWTdD8v7X2lYqNcrO1eXDL3/xa7wqdq6qZ1Yg3depUTJ06VfK5mJgYjcdXrlwx5yWoCvk3cDEqmACltQF/Te1lsK+COcI7+WDGb6cq/byVpXwnZ+Uw7NEPVe7osT5qnW8HSlRLD+vsY9KHZ1BZp21jamh91fpb+NV3Rr82nqrHdgYWgbSztdE5z4uUyEfa4dX+LbUC3OD2Xth+Jq0smEi/5rMPN8GPB5OMfi1VGW0UKCp5UDO18OnOmPTjMcwY2hYfbj1n8vmU3JztUVhcgnsFxUYfM/dR/X0GdNWfqdfsNPeog0u3cnWeY0BbT+w6Jz0hphT1sHZs1iCD8xfNe7QDZv/xr+rxkA6mNaOok5qbpyKNCsoQGT29HwqKSlBQVILAedu19nvu4aZYH5eMV/uZVrNVEcpO7m296+FcarbO/QzNqu1oV7HlTtRHJRrTFGdJ8i4dyUInPzejw0x5ygnRyq+aDJSuP9S9inv5V4Sd2oenl6sjPOo64ovRQejRsqKjHzQ/ghvWdcTpuUM0ZnmtyMrlXq5OOBg5EKfeM210w543+2s03SnLINURUVeTjiFSNUsLRwbi/cc6YvWEh3QGqvF6Oh//oeMDvVNjN/w778HvIKBhHQzt6IOz84Zicr8W8C7rpDugrafk8fps+28fyX5BpvJQG9pvzHs+VaKpaITamlKLn+li0us72Nngl4kP44cXuxsMJgooMC40AJ890wX/19kHnz3TBYtGmvZ66nzdDP8NuTnb49VytXuH3xmIn14K0dnMZGujgLODrc5+Ku8/1hGn3xuiEcz1+bFcZ9+4mWGSv6vmjepgTPcmqr8rKa2raXr4+jquXV8Tk9yWteCqxFSl+rRuhBOzB8PVWfpPrWcLDxy+fKeaS2UcGxsFjrwbhuISYfQoHnNVZC0jKd46+onoo6savoOvK57vEQC/+s5o7+OKMylZ6GdGO7ourk72WnPPmCLQ3x0nZg/W+pa8flIoHO1s8furPZB8975qhJdyqHvMm/2Qca/QrN+VrmPUw5UQAm+Ht8WCLeckZy32dXPC/rcHoFnk5tL9JepOjJlG4PPRQZg+uDUKi0vgqmNKg4+e7Iy3fjsp+VxoC82mEgdbGxQUl+h8vUe7NMajXYxr4unerAEOX76DtmrD+mPe6If7hcWq/k/6HJ81SOvv0tPVCZ6uTngooAE2HDdvGRQ7IyZUjJ7eF4npOehVru9Zw7qOWjU8gX5u+HJMV7i52OPA2wPQ/J3Nkues6v6mE3s3Q0FRCcb3CMCAT/7ReO7MvCF6P8eqYmqCimA4oSqnr6f9pH7N0bCug2ynP5fbf1hLUCgUGiMZKl5zpJuNmZ/eUn9jynMFNamvaupS52RvC2833dXklXEjmdS3dFSUl8S3aYVC/4zOX4wOMnqlbUN9v0Y+5I9f467h8BXDXwTWvvIwZv/xr9Ywb1N+H8qRYF+N7Yq1R5LxVPCDiRnVV2p/KtgPv8ZdUz3u2NgNh9S+rOjrt+JgZ6MKP1WhRaO6OjtLl/9dzBnRQVW7bKivjS7Gzsatj7ebM17UsSyIVDAJaOiCK7fvVfh1qwKbdciiHO1s8ezDTWt0x1plj3pj50mQ0Wg9tDNyCYHqIvWr6R7QwKhmxY+e6oxAtflvzLxHGKQc9SM1J0xnP3etbVLBxBhSQa0izQLGjgoLalIff/2nl9YsvoZ+nftm9Mfycd3w2+RQfPhk6XByj7qOmNK/pc7fwcKnAzWaCD8fHYR6JnZa7tmyoWSn7Kql+dswZWmL8iLD22Lh04FacyCZw9Q/+V/KRj+Zus5ZdWDNCVEFLXsuGAmp2eho5qgOS+oW0ADLng2WHOps2VloSv02uQfa+7jCyd4WJ+YMhp2NAh3mbAMAjOzmh/CODz7QR3bzx8hu/vjzxA0429saVXWvj3o22DKtN36Lu4apA1qqlnhQt/31PvjrxA1M7NMcxSUC0WfT8KSxyziU6eKve1TOtv/2wbW799CxsRue7xGA1Mw8zBreHi46Ji2zBL/6LiZNxqg0sps/Fu04j65N3OHl6oQ5IzrgjfUnjDrWv4ELfnpJ9wiWnRF9cezqXZ1NWuX1bd0I/5y/adJ8KY928a1Q/6NX+po2fP6fN/thz/mbmKXWMVnJ1C8+Pm7OuLJgmGkHVROGE6IKcrSzlfzGrEtTI4c6VpehMlr7w6ZcnlAfRqvsTPvrpFAk372Hx4Okb/4jjGwKUffb5B5YtCMB+xNvSz7fzscVM/9PetFMoLRGY/rg0gnuPh3VBUXFJTrDkXJ00ktlyxZETy+9geqbR6SNdz3VcgzmThZWkU7WACp31Ug1r/Zrga5N6qNLE/dKP3dLz7po2tDF6HDyxZgg7DqbjjA9sywDQFg7T6w5kgz/Bs74TGJldCV9w5ob1nHQmrDRGE0b1sFzoXXQt7Un+ny82+TjawqGE6Jq1tbbFUvGdIWPu3nV/bVZT7X+LG11NDl1C2iAbgGVO8oruGl9zBzWHuGf7TX6mHeGtcNjS/ZjksQ3X321NkvGdsXFmzloU9ZEU75vw+D2XjienIH+bSt3kVE51IRJsStbZLSqmBLKXJ3s8ZgRk83NHt4enfzcMEhHDcv40Kb4LvYq3hraVmN7gFrz9dGZYRWakbVJQxcM6+yDTScfzCsmdbauTdwxvoqm5q9KDCdEFlBZkz5VpQp/0zaD+jwOM4fprqmQgy7+7kj431CT556wt7VBW2/dzQBfPxeMEiE9D4gx3n+sI2ZtPI1p5ZbVEJZ4Q83QvZKDpzpTF+vUxcXBDmNDdI8we29EB0wLa6015HhyvxbIzS/Smi3XXJ88HYingv0wYdURnfuMCWlq9OgqOWE4ISJZ2TilJ86nZVfpt+nKUtFJsaQoFAoYmP9Or+cebopHOnqjYV3NkWYVb9Wpnp7cTRq6YPcb/dBAom+POdQ7AvdpZV5t1LBOPth0KkVj1JE+CoVCci4UJ3tbvc2DpnKyt0V/tYkTze18LUcMJ0QyNP+JTnh62QG8UdaPwZp08Xev8Boi5pDTKKqKKh9Mahp9a1FVhLnv8ScjA/FUNz+E6pk+35KWPdsVx5MzMKSDfPqPVRTDCZEMdfF3x9l5Qys84oTMV9fRsqsRVzZzW3Ua1XPEzex81SR2NY36dZtb+1O+hkJuhnb0wdCO8m8qNgXDCZFMWSqYjAj0xZ8nbuiczKm2Ul9L6elgPww0Y1p7OTO3WWf/jAEoLC6RXMagpqlNtWPGcpT5Gjq61Py/NiKqVJ890wVRT3SqFTcjUzR2d8Z7w9ujrpO90X0LahQzq04c7Gxkv0icPjWkH3Clmz6oNY5evSurqQJMYV2fPkRkkEKhsLpgovR8T+uqLaLa6z/lRmvVNDU3DhMRkdFaVdOKuHJj7LT9JC/W+fWIiMjKzBrWHnUd7fBE15o350VlscY+JzUVwwkRkRVwc7E3e+r7msxa+5zUdGzWISIiq1AZs7JS9WA4ISKiWstebUh+XSvt6F0T8Z0iIqJay8HOBt+/0B3FJUK1sjXJH8MJERHVan1aV+4Kz1T12KxDREREssJwQkRERLLCcEJERESywnBCREREssJwQkRERLLCcEJERESywnBCREREssJwQkRERLLCcEJERESywnBCREREssJwQkRERLLCcEJERESywnBCREREslIjViUWQgAAsrKyLFwSIiIiMpbyvq28jxurRoST7OxsAIC/v7+FS0JERESmys7Ohpubm9H7K4SpccYCSkpKcOPGDdSrVw8KhaLSzpuVlQV/f38kJyfD1dW10s4rJ7X9Gnl9NV9tv0ZeX81X26+xKq9PCIHs7Gz4+vrCxsb4niQ1oubExsYGfn5+VXZ+V1fXWvkHp662XyOvr+ar7dfI66v5avs1VtX1mVJjosQOsURERCQrDCdEREQkK1YdThwdHTFnzhw4OjpauihVprZfI6+v5qvt18jrq/lq+zXK8fpqRIdYIiIish5WXXNCRERE8sNwQkRERLLCcEJERESywnBCREREsmLV4WTJkiUICAiAk5MTQkJCcPjwYUsXCVFRUXjooYdQr149eHp64rHHHkNCQoLGPv369YNCodD4mTRpksY+SUlJGDZsGFxcXODp6Yk333wTRUVFGvvExMSga9eucHR0RMuWLbF69Wqt8lT27+i9997TKnvbtm1Vz+fl5WHKlClo2LAh6tatiyeffBJpaWk14tqUAgICtK5RoVBgypQpAGre+7dnzx4MHz4cvr6+UCgU2Lhxo8bzQgjMnj0bPj4+cHZ2RlhYGC5cuKCxz507dzB27Fi4urrC3d0dL774InJycjT2OXnyJHr37g0nJyf4+/vjo48+0irL+vXr0bZtWzg5OaFTp07YvHmzyWUx5foKCwsxY8YMdOrUCXXq1IGvry/GjRuHGzduaJxD6j1fsGCBLK7P0DUCwPPPP69V/qFDh2rsU1PfQwCS/x8VCgU+/vhj1T5yfg+NuS/I6bPTmLIYJKzUmjVrhIODg1i5cqX4999/xcSJE4W7u7tIS0uzaLmGDBkiVq1aJU6fPi3i4+PFI488Ipo0aSJycnJU+/Tt21dMnDhRpKSkqH4yMzNVzxcVFYmOHTuKsLAwcfz4cbF582bh4eEhIiMjVftcunRJuLi4iIiICHHmzBnxxRdfCFtbW7F161bVPlXxO5ozZ47o0KGDRtlv3rypen7SpEnC399fREdHi6NHj4qHH35Y9OjRo0Zcm1J6errG9e3YsUMAELt37xZC1Lz3b/PmzeLdd98VGzZsEADE77//rvH8ggULhJubm9i4caM4ceKEGDFihGjWrJm4f/++ap+hQ4eKwMBAcfDgQbF3717RsmVLMXr0aNXzmZmZwsvLS4wdO1acPn1a/PLLL8LZ2Vl8/fXXqn32798vbG1txUcffSTOnDkjZs6cKezt7cWpU6dMKosp15eRkSHCwsLE2rVrxblz50RsbKzo3r27CA4O1jhH06ZNxbx58zTeU/X/s5a8PkPXKIQQ48ePF0OHDtUo/507dzT2qanvoRBC47pSUlLEypUrhUKhEBcvXlTtI+f30Jj7gpw+Ow2VxRhWG066d+8upkyZonpcXFwsfH19RVRUlAVLpS09PV0AEP/8849qW9++fcW0adN0HrN582ZhY2MjUlNTVduWLl0qXF1dRX5+vhBCiLfeekt06NBB47hRo0aJIUOGqB5Xxe9ozpw5IjAwUPK5jIwMYW9vL9avX6/advbsWQFAxMbGyv7adJk2bZpo0aKFKCkpEULU7Pev/Ad/SUmJ8Pb2Fh9//LFqW0ZGhnB0dBS//PKLEEKIM2fOCADiyJEjqn22bNkiFAqFuH79uhBCiK+++krUr19fdX1CCDFjxgzRpk0b1eORI0eKYcOGaZQnJCREvPLKK0aXxdTrk3L48GEBQFy9elW1rWnTpuLTTz/VeYxcrk8I6WscP368ePTRR3UeU9vew0cffVQMGDBAY1tNeg/L3xfk9NlpTFmMYZXNOgUFBYiLi0NYWJhqm42NDcLCwhAbG2vBkmnLzMwEADRo0EBj+08//QQPDw907NgRkZGRuHfvnuq52NhYdOrUCV5eXqptQ4YMQVZWFv7991/VPurXr9xHef1V+Tu6cOECfH190bx5c4wdOxZJSUkAgLi4OBQWFmq8Ztu2bdGkSRPVa8r92sorKCjAjz/+iBdeeEFj0cqa/P6pu3z5MlJTUzVex83NDSEhIRrvmbu7O7p166baJywsDDY2Njh06JBqnz59+sDBwUHjehISEnD37l2jrtmYslSGzMxMKBQKuLu7a2xfsGABGjZsiKCgIHz88cca1eU14fpiYmLg6emJNm3aYPLkybh9+7ZG+WvLe5iWloZNmzbhxRdf1HqupryH5e8LcvrsNKYsxqgRC/9Vtlu3bqG4uFjjTQIALy8vnDt3zkKl0lZSUoL//ve/6NmzJzp27KjaPmbMGDRt2hS+vr44efIkZsyYgYSEBGzYsAEAkJqaKnltyuf07ZOVlYX79+/j7t27VfI7CgkJwerVq9GmTRukpKRg7ty56N27N06fPo3U1FQ4ODhofeh7eXkZLLccrk3Kxo0bkZGRgeeff161rSa/f+UpyyP1Oupl9fT01Hjezs4ODRo00NinWbNmWudQPle/fn2d16x+DkNlqai8vDzMmDEDo0eP1lgg7bXXXkPXrl3RoEEDHDhwAJGRkUhJScGiRYtqxPUNHToUTzzxBJo1a4aLFy/inXfeQXh4OGJjY2Fra1ur3sPvvvsO9erVwxNPPKGxvaa8h1L3BTl9dhpTFmNYZTipKaZMmYLTp09j3759Gttffvll1b87deoEHx8fDBw4EBcvXkSLFi2qu5gmCQ8PV/27c+fOCAkJQdOmTbFu3To4OztbsGRVY8WKFQgPD4evr69qW01+/6xZYWEhRo4cCSEEli5dqvFcRESE6t+dO3eGg4MDXnnlFURFRclqSnBdnnnmGdW/O3XqhM6dO6NFixaIiYnBwIEDLViyyrdy5UqMHTsWTk5OGttrynuo675Q21hls46HhwdsbW21eg+npaXB29vbQqXSNHXqVPz999/YvXs3/Pz89O4bEhICAEhMTAQAeHt7S16b8jl9+7i6usLZ2bnafkfu7u5o3bo1EhMT4e3tjYKCAmRkZOh8zZp0bVevXsXOnTvx0ksv6d2vJr9/ynPpex1vb2+kp6drPF9UVIQ7d+5Uyvuq/ryhsphLGUyuXr2KHTt2GFxWPiQkBEVFRbhy5YresquX25LXV17z5s3h4eGh8TdZ099DANi7dy8SEhIM/p8E5Pke6rovyOmz05iyGMMqw4mDgwOCg4MRHR2t2lZSUoLo6GiEhoZasGSlw8ymTp2K33//Hbt27dKqRpQSHx8PAPDx8QEAhIaG4tSpUxofJsoP1Pbt26v2Ub9+5T7K66+u31FOTg4uXrwIHx8fBAcHw97eXuM1ExISkJSUpHrNmnRtq1atgqenJ4YNG6Z3v5r8/jVr1gze3t4ar5OVlYVDhw5pvGcZGRmIi4tT7bNr1y6UlJSoglloaCj27NmDwsJCjetp06YN6tevb9Q1G1MWcyiDyYULF7Bz5040bNjQ4DHx8fGwsbFRNYXI+fqkXLt2Dbdv39b4m6zJ76HSihUrEBwcjMDAQIP7yuk9NHRfkNNnpzFlMYrRXWdrmTVr1ghHR0exevVqcebMGfHyyy8Ld3d3jZ7MljB58mTh5uYmYmJiNIa03bt3TwghRGJiopg3b544evSouHz5svjjjz9E8+bNRZ8+fVTnUA4ZGzx4sIiPjxdbt24VjRo1khwy9uabb4qzZ8+KJUuWSA4Zq+zf0fTp00VMTIy4fPmy2L9/vwgLCxMeHh4iPT1dCFE6BK1JkyZi165d4ujRoyI0NFSEhobWiGtTV1xcLJo0aSJmzJihsb0mvn/Z2dni+PHj4vjx4wKAWLRokTh+/LhqtMqCBQuEu7u7+OOPP8TJkyfFo48+KjmUOCgoSBw6dEjs27dPtGrVSmMYakZGhvDy8hLPPfecOH36tFizZo1wcXHRGqZpZ2cnFi5cKM6ePSvmzJkjOUzTUFlMub6CggIxYsQI4efnJ+Lj4zX+TypHOBw4cEB8+umnIj4+Xly8eFH8+OOPolGjRmLcuHGyuD5D15idnS3eeOMNERsbKy5fvix27twpunbtKlq1aiXy8vJq/HuolJmZKVxcXMTSpUu1jpf7e2joviCEvD47DZXFGFYbToQQ4osvvhBNmjQRDg4Oonv37uLgwYOWLpIAIPmzatUqIYQQSUlJok+fPqJBgwbC0dFRtGzZUrz55psa82QIIcSVK1dEeHi4cHZ2Fh4eHmL69OmisLBQY5/du3eLLl26CAcHB9G8eXPVa6ir7N/RqFGjhI+Pj3BwcBCNGzcWo0aNEomJiarn79+/L1599VVRv3594eLiIh5//HGRkpJSI65N3bZt2wQAkZCQoLG9Jr5/u3fvlvybHD9+vBCidHjkrFmzhJeXl3B0dBQDBw7Uuu7bt2+L0aNHi7p16wpXV1cxYcIEkZ2drbHPiRMnRK9evYSjo6No3LixWLBggVZZ1q1bJ1q3bi0cHBxEhw4dxKZNmzSeN6Ysplzf5cuXdf6fVM5bExcXJ0JCQoSbm5twcnIS7dq1E/Pnz9e4sVvy+gxd471798TgwYNFo0aNhL29vWjatKmYOHGiVoitqe+h0tdffy2cnZ1FRkaG1vFyfw8N3ReEkNdnpzFlMURRduFEREREsmCVfU6IiIhIvhhOiIiISFYYToiIiEhWGE6IiIhIVhhOiIiISFYYToiIiEhWGE6IiIhIVhhOiIiISFYYToiIiEhWGE6IiIhIVhhOiIiISFYYToiIiEhW/h+q9MIjPFWRuQAAAABJRU5ErkJggg==",
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plt.plot(lossi)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Seeing the loss in train and val loss. There is a slight modification to this as to how the splitting is done."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Here the decorator `@torch.no_grad()` basically tells PyTorch to not maintain the grad value, as it assumes/anticipated that the backpropagation will be calculated after this and we are saying No."
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train 2.12243390083313\n",
"val 2.1646578311920166\n"
]
}
],
"source": [
"@torch.no_grad() # this decorator disables gradient tracking\n",
"def split_loss(split):\n",
" x,y = {\n",
" 'train': (Xtr, Ytr),\n",
" 'val': (Xdev, Ydev),\n",
" 'test': (Xte, Yte),\n",
" }[split]\n",
" emb = C[x] # (N, block_size, n_embd)\n",
" embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)\n",
" h = torch.tanh(embcat @ W1 + b1) # (N, n_hidden)\n",
" logits = h @ W2 + b2 # (N, vocab_size)\n",
" loss = F.cross_entropy(logits, y)\n",
" print(split, loss.item())\n",
"\n",
"split_loss('train')\n",
"split_loss('val')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Sampling of the model: Forward pass -> Sampling from the distribution -> Continuing till we get the special token '.'"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mora.\n",
"mayah.\n",
"see.\n",
"mel.\n",
"rylee.\n",
"emmadiejd.\n",
"leg.\n",
"adelyn.\n",
"elin.\n",
"shi.\n",
"jen.\n",
"eden.\n",
"estanar.\n",
"kayziquetta.\n",
"noshir.\n",
"roshiriel.\n",
"kendreth.\n",
"konnie.\n",
"casube.\n",
"ged.\n"
]
}
],
"source": [
"# sample from the model\n",
"g = torch.Generator().manual_seed(2147483647 + 10)\n",
"\n",
"for _ in range(20):\n",
" \n",
" out = []\n",
" context = [0] * block_size # initialize with all ...\n",
" while True:\n",
" # forward pass the neural net\n",
" emb = C[torch.tensor([context])] # (1,block_size,d)\n",
" h = torch.tanh(emb.view(1, -1) @ W1 + b1)\n",
" logits = h @ W2 + b2\n",
" probs = F.softmax(logits, dim=1)\n",
" # sample from the distribution\n",
" ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n",
" context = context[1:] + [ix]\n",
" out.append(ix)\n",
" # if we sample the special '.' token, break\n",
" if ix == 0:\n",
" break\n",
" \n",
" print(''.join(itos[i] for i in out)) # decode and print the generated word"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"So yeah, this will be our starting point. Also use this as a revision for the previous lecture."
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
|