{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "id": "7lDFibCgbHwr" }, "outputs": [], "source": [ "import torch\n", "import torch.nn.functional as F\n", "import matplotlib.pyplot as plt # for making figures\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "htdQCnUPbhS9" }, "outputs": [], "source": [ "# download the names.txt file from github\n", "!wget https://raw.githubusercontent.com/karpathy/makemore/master/names.txt" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ylW9Ir3GbHws", "outputId": "0c874340-2a0f-4d12-fcb0-2035564cebf3" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "32033\n", "15\n", "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']\n" ] } ], "source": [ "# read in all the words\n", "words = open('names.txt', 'r').read().splitlines()\n", "print(len(words))\n", "print(max(len(w) for w in words))\n", "print(words[:8])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "GPwi-_rEbHwt", "outputId": "237fb410-4940-448b-a00b-0ab334f768e3" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n", "27\n" ] } ], "source": [ "# build the vocabulary of characters and mappings to/from integers\n", "chars = sorted(list(set(''.join(words))))\n", "stoi = {s:i+1 for i,s in enumerate(chars)}\n", "stoi['.'] = 0\n", "itos = {i:s for s,i in stoi.items()}\n", "vocab_size = len(itos)\n", "print(itos)\n", "print(vocab_size)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "j8JdduA3bHwu" }, "outputs": [], "source": [ "# shuffle up the words\n", "import random\n", "random.seed(42)\n", "random.shuffle(words)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kcrfsUG_bHwu", "outputId": "4d05c8d0-9d1f-4b1e-dfdb-db2becfe8afc" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch.Size([182625, 8]) torch.Size([182625])\n", "torch.Size([22655, 8]) torch.Size([22655])\n", "torch.Size([22866, 8]) torch.Size([22866])\n" ] } ], "source": [ "# build the dataset\n", "block_size = 8 # context length: how many characters do we take to predict the next one?\n", "\n", "def build_dataset(words):\n", " X, Y = [], []\n", "\n", " for w in words:\n", " context = [0] * block_size\n", " for ch in w + '.':\n", " ix = stoi[ch]\n", " X.append(context)\n", " Y.append(ix)\n", " context = context[1:] + [ix] # crop and append\n", "\n", " X = torch.tensor(X)\n", " Y = torch.tensor(Y)\n", " print(X.shape, Y.shape)\n", " return X, Y\n", "\n", "n1 = int(0.8*len(words))\n", "n2 = int(0.9*len(words))\n", "Xtr, Ytr = build_dataset(words[:n1]) # 80%\n", "Xdev, Ydev = build_dataset(words[n1:n2]) # 10%\n", "Xte, Yte = build_dataset(words[n2:]) # 10%" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9aFDMks6bHwv", "outputId": "2bb77886-1172-4b43-f446-b2e990e549aa" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "........ --> y\n", ".......y --> u\n", "......yu --> h\n", ".....yuh --> e\n", "....yuhe --> n\n", "...yuhen --> g\n", "..yuheng --> .\n", "........ --> d\n", ".......d --> i\n", "......di --> o\n", ".....dio --> n\n", "....dion --> d\n", "...diond --> r\n", "..diondr --> e\n", ".diondre --> .\n", "........ --> x\n", ".......x --> a\n", "......xa --> v\n", ".....xav --> i\n", "....xavi --> e\n" ] } ], "source": [ "for x,y in zip(Xtr[:20], Ytr[:20]):\n", " print(''.join(itos[ix.item()] for ix in x), '-->', itos[y.item()])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-goFZmrabHww" }, "outputs": [], "source": [ "# Near copy paste of the layers we have developed in Part 3\n", "\n", "# -----------------------------------------------------------------------------------------------\n", "class Linear:\n", "\n", " def __init__(self, fan_in, fan_out, bias=True):\n", " self.weight = torch.randn((fan_in, fan_out)) / fan_in**0.5 # note: kaiming init\n", " self.bias = torch.zeros(fan_out) if bias else None\n", "\n", " def __call__(self, x):\n", " self.out = x @ self.weight\n", " if self.bias is not None:\n", " self.out += self.bias\n", " return self.out\n", "\n", " def parameters(self):\n", " return [self.weight] + ([] if self.bias is None else [self.bias])\n", "\n", "# -----------------------------------------------------------------------------------------------\n", "class BatchNorm1d:\n", "\n", " def __init__(self, dim, eps=1e-5, momentum=0.1):\n", " self.eps = eps\n", " self.momentum = momentum\n", " self.training = True\n", " # parameters (trained with backprop)\n", " self.gamma = torch.ones(dim)\n", " self.beta = torch.zeros(dim)\n", " # buffers (trained with a running 'momentum update')\n", " self.running_mean = torch.zeros(dim)\n", " self.running_var = torch.ones(dim)\n", "\n", " def __call__(self, x):\n", " # calculate the forward pass\n", " if self.training:\n", " if x.ndim == 2:\n", " dim = 0\n", " elif x.ndim == 3:\n", " dim = (0,1)\n", " xmean = x.mean(dim, keepdim=True) # batch mean\n", " xvar = x.var(dim, keepdim=True) # batch variance\n", " else:\n", " xmean = self.running_mean\n", " xvar = self.running_var\n", " xhat = (x - xmean) / torch.sqrt(xvar + self.eps) # normalize to unit variance\n", " self.out = self.gamma * xhat + self.beta\n", " # update the buffers\n", " if self.training:\n", " with torch.no_grad():\n", " self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * xmean\n", " self.running_var = (1 - self.momentum) * self.running_var + self.momentum * xvar\n", " return self.out\n", "\n", " def parameters(self):\n", " return [self.gamma, self.beta]\n", "\n", "# -----------------------------------------------------------------------------------------------\n", "class Tanh:\n", " def __call__(self, x):\n", " self.out = torch.tanh(x)\n", " return self.out\n", " def parameters(self):\n", " return []\n", "\n", "# -----------------------------------------------------------------------------------------------\n", "class Embedding:\n", "\n", " def __init__(self, num_embeddings, embedding_dim):\n", " self.weight = torch.randn((num_embeddings, embedding_dim))\n", "\n", " def __call__(self, IX):\n", " self.out = self.weight[IX]\n", " return self.out\n", "\n", " def parameters(self):\n", " return [self.weight]\n", "\n", "# -----------------------------------------------------------------------------------------------\n", "class FlattenConsecutive:\n", "\n", " def __init__(self, n):\n", " self.n = n\n", "\n", " def __call__(self, x):\n", " B, T, C = x.shape\n", " x = x.view(B, T//self.n, C*self.n)\n", " if x.shape[1] == 1:\n", " x = x.squeeze(1)\n", " self.out = x\n", " return self.out\n", "\n", " def parameters(self):\n", " return []\n", "\n", "# -----------------------------------------------------------------------------------------------\n", "class Sequential:\n", "\n", " def __init__(self, layers):\n", " self.layers = layers\n", "\n", " def __call__(self, x):\n", " for layer in self.layers:\n", " x = layer(x)\n", " self.out = x\n", " return self.out\n", "\n", " def parameters(self):\n", " # get parameters of all layers and stretch them out into one list\n", " return [p for layer in self.layers for p in layer.parameters()]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "v01gpFOSbHwx" }, "outputs": [], "source": [ "torch.manual_seed(42); # seed rng for reproducibility" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "vHMDMtvpbHwx", "outputId": "dbb414a9-5753-4f41-c545-34957779b09e" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "76579\n" ] } ], "source": [ "# original network\n", "# n_embd = 10 # the dimensionality of the character embedding vectors\n", "# n_hidden = 300 # the number of neurons in the hidden layer of the MLP\n", "# model = Sequential([\n", "# Embedding(vocab_size, n_embd),\n", "# FlattenConsecutive(8), Linear(n_embd * 8, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n", "# Linear(n_hidden, vocab_size),\n", "# ])\n", "\n", "# hierarchical network\n", "n_embd = 24 # the dimensionality of the character embedding vectors\n", "n_hidden = 128 # the number of neurons in the hidden layer of the MLP\n", "model = Sequential([\n", " Embedding(vocab_size, n_embd),\n", " FlattenConsecutive(2), Linear(n_embd * 2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n", " FlattenConsecutive(2), Linear(n_hidden*2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n", " FlattenConsecutive(2), Linear(n_hidden*2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n", " Linear(n_hidden, vocab_size),\n", "])\n", "\n", "# parameter init\n", "with torch.no_grad():\n", " model.layers[-1].weight *= 0.1 # last layer make less confident\n", "\n", "parameters = model.parameters()\n", "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n", "for p in parameters:\n", " p.requires_grad = True" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Sh9SEATabHwy", "outputId": "4238b680-4e09-4da1-a52f-5153a4119ba5" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 200000: 3.3167\n", " 10000/ 200000: 2.0576\n", " 20000/ 200000: 2.0723\n", " 30000/ 200000: 2.5134\n", " 40000/ 200000: 2.1476\n", " 50000/ 200000: 1.7836\n", " 60000/ 200000: 2.2592\n", " 70000/ 200000: 1.9331\n", " 80000/ 200000: 1.6875\n", " 90000/ 200000: 2.0395\n", " 100000/ 200000: 1.7736\n", " 110000/ 200000: 1.9570\n", " 120000/ 200000: 1.7465\n", " 130000/ 200000: 1.8126\n", " 140000/ 200000: 1.7406\n", " 150000/ 200000: 1.7466\n", " 160000/ 200000: 1.8806\n", " 170000/ 200000: 1.6266\n", " 180000/ 200000: 1.6476\n", " 190000/ 200000: 1.8555\n" ] } ], "source": [ "# same optimization as last time\n", "max_steps = 200000\n", "batch_size = 32\n", "lossi = []\n", "\n", "for i in range(max_steps):\n", "\n", " # minibatch construct\n", " ix = torch.randint(0, Xtr.shape[0], (batch_size,))\n", " Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y\n", "\n", " # forward pass\n", " logits = model(Xb)\n", " loss = F.cross_entropy(logits, Yb) # loss function\n", "\n", " # backward pass\n", " for p in parameters:\n", " p.grad = None\n", " loss.backward()\n", "\n", " # update: simple SGD\n", " lr = 0.1 if i < 150000 else 0.01 # step learning rate decay\n", " for p in parameters:\n", " p.data += -lr * p.grad\n", "\n", " # track stats\n", " if i % 10000 == 0: # print every once in a while\n", " print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n", " lossi.append(loss.log10().item())\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XrOSTEFzbHwz", "outputId": "64b7e0c9-a0a0-4daa-cee0-0c12541fa959" }, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0n0lEQVR4nO3deXxU1f3/8ddnJnvInhBCFpIAASGEgGFVcMEF3IBqcau2amvtV1trF6u168/altbWblZqXWqrdce6L4gLIGuAsIYlLNnJvu+TOb8/ZggTSGDQJBMmn+fjwcOZM/fOfO7N+M7JufeeK8YYlFJKeS+LpwtQSinVvzTolVLKy2nQK6WUl9OgV0opL6dBr5RSXs7H0wX0JDo62iQnJ3u6DKWUOmNs3ry50hgT09NrgzLok5OTyc7O9nQZSil1xhCR/N5e06EbpZTychr0Sinl5TTolVLKy2nQK6WUl3Mr6EVkvojsFZE8Ebmvh9cXish2EckRkWwROdfltXtEZJeI7BSR50UkoC83QCml1MmdMuhFxAo8CiwAJgDXi8iE4xZbCUw2xmQCtwJPONeNB74DZBlj0gErcF2fVa+UUuqU3OnRTwfyjDEHjTHtwAvAQtcFjDGN5tg0mMGA65SYPkCgiPgAQUDJFy9bKaWUu9wJ+nig0OV5kbOtGxFZLCJ7gLdx9OoxxhQDDwMFQClQZ4z5oKcPEZHbncM+2RUVFae3FU5/WbmfT/d9vnWVUspbuRP00kPbCZPYG2NeM8aMBxYBDwKISASO3n8KMBIIFpGv9PQhxpjHjTFZxpismJgeL+46pX98eoBVGvRKKdWNO0FfBCS6PE/gJMMvxphVwGgRiQYuAg4ZYyqMMR3AcmD2F6j3pAL9fGhut/XX2yul1BnJnaDfBIwVkRQR8cNxMPUN1wVEZIyIiPPxVMAPqMIxZDNTRIKcr88DcvtyA1wF+Vlpbu/sr7dXSqkz0innujHG2ETkLuB9HGfNPGWM2SUidzhfXwZcDdwsIh1AC3Ct8+DsBhF5BdgC2ICtwOP9syka9Eop1RO3JjUzxrwDvHNc2zKXx0uBpb2s+3Pg51+gRrcF+llp0aBXSqluvOrKWEePXsfolVLKlVcFfaCvjw7dKKXUcbwq6IP8rLR0aNArpZQrrwt67dErpVR3XhX0ejBWKaVO5FVBH+y8YOrYtDtKKaW8KugD/azYDbTZ7J4uRSmlBg2vCvogPyuADt8opZQLrwz6Zj3zRimlunhV0Af6OS70bdGLppRSqotXBX2Qr7NHr0M3SinVxbuC3k+DXimljudVQR/YFfQ6dKOUUkd5VdAHOcfotUevlFLHeFnQ69CNUkodz6uCPlDPo1dKqRN4VdBrj14ppU7kVUEf4HO0R68HY5VS6iivCnqLRQj01amKlVLKlVcFPUCwv1WnQFBKKRdeF/Q6J71SSnXnVtCLyHwR2SsieSJyXw+vLxSR7SKSIyLZInKuy2vhIvKKiOwRkVwRmdWXG3C8IF8fvWBKKaVc+JxqARGxAo8CFwNFwCYRecMYs9tlsZXAG8YYIyIZwEvAeOdrfwbeM8ZcIyJ+QFCfbsFxAvV2gkop1Y07PfrpQJ4x5qAxph14AVjouoAxptEcu61TMGAARCQUmAs86Vyu3RhT20e19yhIh26UUqobd4I+Hih0eV7kbOtGRBaLyB7gbeBWZ3MqUAE8LSJbReQJEQnu6UNE5HbnsE92RUXFaW2EK71BuFJKdedO0EsPbSfclNUY85oxZjywCHjQ2ewDTAUeM8ZMAZqAE8b4nes/bozJMsZkxcTEuFN7jwL9fGjRs26UUqqLO0FfBCS6PE8ASnpb2BizChgtItHOdYuMMRucL7+CI/j7TZCvVQ/GKqWUC3eCfhMwVkRSnAdTrwPecF1ARMaIiDgfTwX8gCpjzBGgUETGORedB7gexO1zgX5Wmtu0R6+UUked8qwbY4xNRO4C3geswFPGmF0icofz9WXA1cDNItIBtADXuhyc/TbwnPOXxEHgln7Yji5Bfo4LpowxOH/3KKXUkHbKoAcwxrwDvHNc2zKXx0uBpb2smwNkff4ST0+Qn5VOu6G9046/c+4bpZQayrzuythgf8fvrsZWHadXSinwwqCPCwsEoKS21cOVKKXU4OB1QZ8U6bjwtrCm2cOVKKXU4OB1QZ8Y6ejRF1Rr0CulFHhh0IcE+BIR5KtBr5RSTl4X9OAYvinUoFdKKcBLgz5Bg14ppbp4ZdAnRQZRVNNCp/2EKXmUUmrI8dqgt9kNpXUtni5FKaU8ziuDPjHCeYpltQa9Ukp5ZdB3nUuv4/RKKeWdQR8XHoDVInqKpVJK4aVB72u1kBQZxL6yBk+XopRSHueVQQ+QkRDGtqJaT5ehlFIe57VBn5kYTll9G0fqdHIzpdTQ5rVBPzkxHICcwlqP1qGUUp7mtUE/IS4UX6to0CulhjyvDfoAXysT4kLZpkGvlBrivDbowTF8s72oVqdCUEoNaV4d9FnJkTS1d7Imr9LTpSillMd4ddBfOjGWkWEB/GXlfozRXr1Samjy6qD397HyrQvGsDm/hs/yqjxdjlJKeYRbQS8i80Vkr4jkich9Pby+UES2i0iOiGSLyLnHvW4Vka0i8lZfFe6uJVkJDA/x59/rDg/0Ryul1KBwyqAXESvwKLAAmABcLyITjltsJTDZGJMJ3Ao8cdzrdwO5X7jaz8Hfx8qcsTFk59fo8I1Sakhyp0c/Hcgzxhw0xrQDLwALXRcwxjSaYykaDHQlqogkAJdzYvgPmGnJEVQ3tXOwsslTJSillMe4E/TxQKHL8yJnWzcislhE9gBv4+jVH/Un4F7AfrIPEZHbncM+2RUVFW6U5b6s5AgANh+u6dP3VUqpM4E7QS89tJ0wBmKMec0YMx5YBDwIICJXAOXGmM2n+hBjzOPGmCxjTFZMTIwbZbkvNXoY4UG+ZOdX9+n7KqXUmcCdoC8CEl2eJwAlvS1sjFkFjBaRaOAc4CoROYxjyOdCEXn285f7+VgsQtaoCLK1R6+UGoLcCfpNwFgRSRERP+A64A3XBURkjIiI8/FUwA+oMsbcb4xJMMYkO9f7yBjzlT7dAjedPSqSg5VNVDW2eeLjlVLKY04Z9MYYG3AX8D6OM2deMsbsEpE7ROQO52JXAztFJAfHGTrXmkF2ikumczbLXSX1ni1EKaUGmI87Cxlj3gHeOa5tmcvjpcDSU7zHJ8Anp11hH0mLHQbAvrIG5qb17TEApZQazLz6ylhXUcP8iQr2Y39Zo6dLUUqpATVkgh5gbOww9pXrfWSVUkPLkAr6tNgQ8soa9QpZpdSQMqSCfmxsCA1tNkr1PrJKqSFkSAV92vBjB2SVUmqoGFpBHxsCoAdklVJDypAK+ohgP6KH+ZOdX41dby+olBoihlTQA8xPj+X9XWUs+cc6Gttsni5HKaX63ZAL+gcXpvPgwolk59fw0Z5yT5ejlFL9bsgFvYhw3fQk/Hws7Ciq9XQ5SinV74Zc0AP4Wi1MiAtle1Gdp0tRSql+NySDHiAjIYydxXV06kFZpZSXG7JBPyk+jKb2Tg5V6qmWSinvNmSDfrJz2mIdvlFKebshG/SjY4YR6GvVoFdKeb0hG/RWizA5MYzXc4pZsbvM0+UopVS/GbJBD/CrRemMCAvkG//OZtW+Ck+Xo5RS/WJIB/2Y4SH8787ZRAb78WJ2oafLUUqpfjGkgx7A38fKlRlxfLi7jPrWDk+Xo5RSfW7IBz3AoinxtNnsvLfziKdLUUqpPqdBD2QmhpMcFcQ/Pj3A5vwaT5ejlFJ9yq2gF5H5IrJXRPJE5L4eXl8oIttFJEdEskXkXGd7ooh8LCK5IrJLRO7u6w3oCyLCT6+YQF1LB1c/tpZ3dpR6uiSllOozpwx6EbECjwILgAnA9SIy4bjFVgKTjTGZwK3AE852G/B9Y8xZwEzgzh7WHRTmnRXLqnsvYMzwYfztozy9r6xSymu406OfDuQZYw4aY9qBF4CFrgsYYxrNsWQMBoyzvdQYs8X5uAHIBeL7qvi+FuTnw+1zUtldWs/aA1UANLR2UFjd7OHKlFLq83Mn6OMB13MPi+ghrEVksYjsAd7G0as//vVkYAqwoacPEZHbncM+2RUVnjunfeGUkUQP8+c37+by0qZCLnlkFZf9eTVttk6P1aSUUl+EO0EvPbSdMK5hjHnNGDMeWAQ82O0NRIYBrwLfNcbU9/QhxpjHjTFZxpismJgYN8rqH/4+Vn5y+VnkVzZz76vbaWy10dBm06kSlFJnLB83likCEl2eJwAlvS1sjFklIqNFJNoYUykivjhC/jljzPIvVu7AWDQlnvnpI9heVMfI8ADOXfoxGw5WkRIdzLbCWuadFevpEpVSym3uBP0mYKyIpADFwHXADa4LiMgY4IAxxojIVMAPqBIRAZ4Eco0xf+zb0vtXgK+V6SmRAIwfEcKGQ9XsLWvkre0l5PzsEsICfT1coVJKueeUQW+MsYnIXcD7gBV4yhizS0TucL6+DLgauFlEOoAW4Fpn6J8L3ATsEJEc51v+2BjzTj9sS7+ZkRLJC5sKsdkNxkBeeQNnj4r0dFlKKeUWd3r0OIP5nePalrk8Xgos7WG9NfQ8xn9GmZ4SxTPr8rue7ytr1KBXSp0x9MpYN8xIjUQEFmaOJNDXyr6yBk+XpJRSbnOrRz/URQ/z55lbpjMpPoxDlU3sL9PbDyqlzhzao3fT3LQYIoL9GDs8RHv0Sqkzigb9aUqLHUZ5Qxub82v444p9dHTaPV2SUkqdlA7dnKa02BAAbnl6I/WtNgC+d3GaJ0tSSqmT0h79aRobOwyA+lYbGQlhPPpxHpvzq2nt6OSXb+7i9ZxiD1eolFLdaY/+NMWHBxIb6s95aTE8cPkELvvzaq57fD3JUcHsL29k4shQFmYO2nnblFJDkPboT5OIsPL75/PbL2UQFujLG3edw4L0OAprmpmREkluaT0NrR3c9d8tXP6X1azNq/R0yUqpIU6D/nMY5u+DxeK4DixqmD9/uX4KO35xKd++cCx2Ax/mlvHOjlL2lTVwwxMbWH+wysMVK6WGMg36PuJrtTAlKRyrRXj4/X3YDbxw+0wignz512eHPV2eUmoI06DvQ8H+PkwcGUpxbQtJkUFMTYpgSVYiK3LLOFLX6unylFJDlAZ9H8tyzoGzIH0EIsINM5LotBte2FTg4cqUUkOVBn0fm5MWjQhckTESgFFRwZwzJoo3t/U6hb9SSvUrDfo+dsG44ay7bx6TEsK62s4ZE82BiiaqGtt4dn0+dz63hfyqJg9WqZQaSjTo+8GIsIBuz2c4b2Cy6XA1j36cx9s7SrnkkVXkFNZ6oDql1FCjQT8AJsWH4+9j4ck1hyita+WHl47DIsJrW4o8XZpSagjQK2MHgJ+PhczEcDYcqsZqEW6ckcTm/Bo+3VcBgN1uus7LV0qpvqY9+gFydPhmZmok4UF+zB0bzeGqZtYeqGT2bz/ipU2FHq5QKeWtNOgHyPSUKAAunTgCgPPGDQfgW89u4Uh9K795N5fc0noW//0zXtvqGNKxddoxxnimYKWU19ChmwEye3QUf7o2kwWTHEGfHBVEUmQQBdXNXJERx9s7Srnqb2vo6DTsL2skPMiPH768jW/MSeWb5432cPVKqTOZ9ugHiMUiLJoSj7+PFXBMjnZ5Rhyp0cH8/prJXDctCV+rhT9dm0lHp51bnt5EZWM77+064uHKlVJnOreCXkTmi8heEckTkft6eH2hiGwXkRwRyRaRc91ddyi799JxrPjeeQT6WXloUTrr7p/HoinxPHD5WaREB3N5Rhw7iupobred8r1K61oGoGKl1JnolEEvIlbgUWABMAG4XkQmHLfYSmCyMSYTuBV44jTWHbJEBKvzbBuLRQgL9AXg5lnJfPyD81mSlYjNbth4qJor/7qGpe/t6bb+0fH71fsrmPWbj1izX6dEVkqdyJ0e/XQgzxhz0BjTDrwALHRdwBjTaI4dNQwGjLvrqt6dPSoCq0V46O1cdhTX8eSaQ5TVOyZHW72/gvN+/wl7jzTw7Pp8AJ7X+XSUUj1wJ+jjAddz/4qcbd2IyGIR2QO8jaNX7/a6zvVvdw77ZFdUVLhTu9cb5u9DenwY+8sbSYoMotNu+MenB2lut3HfqzsoqG7m/uXbWZlbTqCvlRW7yqhr7vB02UqpQcadoO/pSp4TzvkzxrxmjBkPLAIePJ11nes/bozJMsZkxcTEuFHW0HD0/PsfXjqOxVPieXZ9Pl/6+1qKa1tYmDmSLQW12OyG33xpEu2ddt7Y3n3ytM35NXR02j1RulJqkHAn6IuARJfnCUCvUzEaY1YBo0Uk+nTXVSe6aeYovndxGpdNiuNH88dzVeZI2mx2vjk3ld9fM5mU6GCmJ0eyMHMk40eE8N8NBV1j97ml9Vz92FqW61QLSg1p7pxHvwkYKyIpQDFwHXCD6wIiMgY4YIwxIjIV8AOqgNpTratOLjEyiO/MGwtATIg/D395crfXl39rNhaLICJ8Y04q3395Gx/sLuPSiSO6pljYnF/DtdOSBrx2pdTgcMoevTHGBtwFvA/kAi8ZY3aJyB0icodzsauBnSKSg+Msm2uNQ4/r9sN2DFkRwX5dZ+sszBxJSnQwj6zYh91uus7C2VZY17X8q5uLuPVfm/SKW6WGELeujDXGvAO8c1zbMpfHS4Gl7q6r+oeP1cJ35o3hnhe38cKmQjYeribA18K+8gYa22wE+1l59JM8DlY0Ud7QRmzosemU61o62HiomosnxHpwC5RS/UGvjPUyV02OZ0pSOD99fSftNjvXTUvCGNhRVEdOYS0HKxw3PNlRVNdtvec25PONf2dzqFJviKKUt9Gg9zJWi7D06gwsAn5WC7fPTQVgW1Etr24pwt/HggjsLOke9LtL6gHYdKh6wGtWSvUvndTMC6XFhvDLq9Ipb2hlZHggo6KCeD2nhKLqZhakj2BHcR07i+u7rbP3SAMAGw9Xs2RaYk9vq5Q6Q2mP3kvdMCOJ716UBsDUpAhyS+sZHurPnReMIT0+jJ3Fx3r0bbZODjqHbDb20qPfdLiaLQU1/V+4UqrPaY9+CLj/svEsyUpkRkokFoswKT6M13NKqGhoIybEn7zyRjrthszEcHIKaymrbyV6mH/XPDwA9y/fQV1LB6vvvYAAX6sHt0Ypdbq0Rz8EDA8JYNboqK7bFU4cGQbA+7uOUFbfyp5Sx7DNTTNHAfDVpzYy+ZcfdM2I2drRycGKRioa2nh+YwE7i+vYnF+DMYblW4r07lhKDXLaox+CJsaHYrUIP/nfTv7fW7uZPToKPx8Ll2fE8fM3drHf2cP/cHcZN81KJq+8EbuBID8rD7+/l+aOToyB8SNC2HOkAT+rhYsnxBIR7Nfj5328p5x/rzvM4zdn4WvVvoVSA03/rxuCQgN8+e/XZ/C3G6YwzN+HT/ZWkBY7jABfK/+6ZRrv3j2H5KggVu4pBxxTKQD8/MoJdNgNX5kxirvnjeVIfSvXT090zLGzrfeZLZ5Zd5iP91awap9OVqeUJ2iPfoiakeq4h63dwHee38q42FAAspIdk6hdMH44/91QQEt7J3uONBDga+GasxO5emoCPs5e+XcvGouIsK2wjpc3FyICDa027rxgTNfnNLbZWJtXBcDyLcXMO0svyFJqoGnQD3FXZsRRXt/KTGfwH3Xh+OE8/dlh1h6oZM+ResbFhjgPzh47QCviePzlrAR++eZudhbvwmoRbj0nhUA/xwHbT/dW0N5pZ1J8GCtyy6hsbMNuDMNDAlBKDQwduhniRISvz0klPT6sW/v0lEiC/ay8trWY3NIGxo8I7fU9Fk+JZ1pyBFdkxNFpN+wsqcPWaaektoX3dx0hMtiP/7dwIu02OzN/vZJzfvtR13CQUqr/aY9e9cjfx8ot56Twt4/zABgfF9LrsuFBfrx8x2wqGtp4a3spOQW1rMwtZ9mnBwC45uwEMhPDuXFGEp12w4rdZTzw2g5euWN215lASqn+o0GvenXPxWlsK6pl9f7Kk/boj4oJ8Sc+PJCthTVsLahlckIY546N5pqzExERHlo8CYBpyUV8/+VtvJhdyLVZidzyr02kxQ7jgcv1dsJK9QcNetUrq0X42/VT+V9OMdOSI9xaJzMpnBW7ymjvtHPv/HEsnpJwwjJfmhrPS9mF/PbdPbR2dPLpvgrW5FVy08xk6ls7CPC1Mmb4sL7eHKWGLB2jVycVFuTLV2cnd51pcypTEsNp77Tj52Phol7OsHH07tNpbrfxyzd3M2b4MKwW4e4Xt7L4759x05MbaGnv7Fq+sLqZJcvW8dWnNna15Vc18cBrO8grb/xiG6jUEKBBr/rU5MRwAOaOjSEkwLfX5cYMD+Gbc0cD8MurJnLD9CS2FtSSEBFEaV0rT312CIDyhlYu/8tqNh6uZtX+ChpaO3hv5xEueWQVz20o4Nn1+f2+TUqd6XToRvWpSfFhnD0qgq/NTj7lst+/JI0vZyUwKiqY9JFhxIYGcMOMJH7w8jYe++QA109PYt2BKupbbdxzURqPfLiPrQW1PL7qAPERgYQH+rL2QOVJP+P1nGJGxww74awipYYS7dGrPhXga+XVb83m3LHRp1xWRBgVFQw4hoi+df5owgJ9uXveWBrbbHyYW0ZOYS0Bvha+NjsZi8BKZ9sVGSO5ZOII9pU55uABOFjRyDNrD9Npd9wmsaqxje+/tI0/fbiv/zZYqTOA9ujVoDNxZChRwX6sP1BFfnUzk+LDCAvyZfyIUJ7fVIjdwPnjYvBxnpq57mAVc8ZEc/NTGymqaWFzfg1/XDKZN7eVYLMbcgprMcZ0XeDV2Gbj7e0lRAT5cfGE2K52pbyVBr0adESEmaOjWJNXSV1LB19xzqqZlRzB7tJ6IoJ8mZwQDkBIgA9vbSvh2fX5lNe3ceOMJJ7bUIBF6LotYmVjO0U1LSRGBrH+YBXfeCabhjYbABeMi+HP108h9CTHE5Q60+nQjRqUZqVGUd7QRpvN3nWA9+xRjlM856bFYLUIVoswMzWKD3Y7hnN+d00GDy2exA8uSeN/OSVsK6pjYeZIAHIKa6loaOPbz28lJsSfV781m59eMYFV+yu5/9UdGGO6PttuNzy55hBPf3ao2w1alDpTaY9eDUqzRh+beyfT2XuflRpFkJ+VKzJGdr12+9xURoQGcPvcVBIjgwC484IxVDS0sXxrMffOH897O4+wtaCWl7ILaWjt4D+3TWf8iFDOHhVBm62T3723l3M2RnPDjCQA1h+s4sG3dgOOvxg2/+Ri/Hz6pk/U0NrBfct38N15Yxkb2/vVxkr1Jbe+vSIyX0T2ikieiNzXw+s3ish257+1IjLZ5bV7RGSXiOwUkedFRGezUqeUGh3M8BB/IoJ8SYwMBGB4aAA5P7uEiyccOz9/WnIkDy5K7wp5cAz9/HJhOpseuIj48EAmxYfx/MYCVu+v5IHLJ3S7yveOuaOZMzaaB9/aTX6VY6jnvV1HCPC18Mclk2lotbH+YFW32l7YWEDGL95n6Xt72HiomrL6Vre3K/twDW9vL+X/ntvS7VoBoNtfFUr1pVMGvYhYgUeBBcAE4HoROf5a9UPAecaYDOBB4HHnuvHAd4AsY0w6YAWu67vylbcSEW6fm8rXZqd0O1h6Oj3ro7c8zEwMp6Wjk2nJEdw4PanbMhaL8LtrMvCxCPe+sh1bp533dh7h/LThXDYpjiA/Kx/sPtJtnWfW5WOAZZ8eYMk/1jFn6ccU17bQ0NrB6znFJw3sPc6bsOdVNPLQO7u72utaOpj20Ie8s6PU7e1Tyl3u/F8zHcgzxhw0xrQDLwALXRcwxqw1xhy9c/R6wPW6dx8gUER8gCCg9ztUKOXi63NSufuisV/4fS4cP5zoYf785kuTepxELS4skJ9eMYENh6q55V+bKG9oY376CAJ8rZyXFsOK3WXYnads7jlST25pPT+4ZByr772AZV85mw67nec3FPCHD/Zx9ws5ZOcfu4n6ytyybjdc2VfWQFxYAIunxPP61pKu9113oJLKxvZerwtYmVvGuUs/otF5EFmp0+FO0McDrjcFLXK29eY24F0AY0wx8DBQAJQCdcaYD3paSURuF5FsEcmuqNA7Eam+M3tMNJsemMeY4b2PiX85K4E7zhvN6v2V+FqFC88aDsAlE2Mpq29ju/Og7P+2lmC1CJdnxJEQEcT89BHMGz+c5zbk89+NBQB8mFsGOO61+72XtnHPizm0djiGafYeaSAtNoSZKVE0tNk46DwzaPX+yq7Xe7JidxlFNS1sLajp8fVNh6vJKaw9zT2jhgp3gr6nk4x7/NtURC7AEfQ/cj6PwNH7TwFGAsEi8pWe1jXGPG6MyTLGZMXExLhTu1JuO9W58iLCfQvG89DidH40f3zX6ZYXjovFz2ph+ZYibJ123sgpZu7YaKKH+Xet+5WZo6hp7sAYw/gRIazMddyCcWVuOXUtHVQ1tfPa1mJsnXbyKhoZPyKEjETHlbrbi2oBWJPnCPo9Rxq6hn52FNXx0Nu7McZ0hXj24ROD3hjDd1/I4f7lO054raPT3vVLxh25pfV876UcbJ12t9dRg587QV8EJLo8T6CH4RcRyQCeABYaY44evboIOGSMqTDGdADLgdlfrGSl+s+NM0bx9TmpXc/Dgny5IiOOVzcX8Z/1+ZTUtXL9ceP8c8fGkB4fyi3npLAkK5G88kbyq5p4eXMhcWEBTBwZypNrDnGosol2m5202BDGxAwj0NfK9qI6Cqqaya9qZuzwYTS02iitcxzcXbbqAP9cfYj1B6vZV+bo6W92Dgt12g1ffWojS9/bw4GKJoprW9hzpJ665o5utf3g5W0s+cc6t7d/+ZYilm8pJr+6+XPtPzU4uRP0m4CxIpIiIn44Dqa+4bqAiCThCPGbjDGu15sXADNFJEgcXap5QG7flK7UwLh5djJN7Z08+NZuMhLCup31A44Dum99ew4/vuysrhk7f/vuHlbtq+BLU+P5xpxU8sob+d37ewEYNyIEH6uF9PhQx3z/eY6hyq/PSQEcwzdttk4+cd6c/Q8f7MVuYFRUEFsLaui0G57+7BCf7qvgmbWHeW+n4wCuMbDxcHVXXcW1Lby5rYTtRXUcqWvl5exC7n1l20m3dVuhY4iqqKbli+42NYicMuiNMTbgLuB9HCH9kjFml4jcISJ3OBf7GRAF/F1EckQk27nuBuAVYAuww/l5j/f9ZijVfzITw5mcEIbdwPcvGXfSYaCkqCAmxYfx7s4jhAb6cm1WEldOHknWqAhW7C7DInTNtZ+REM6uknqeWH2I5Kgg5qfHAY7hm7UHqmhq7yQ0wKfr4O4tzl84b24r4eEP9pIWO4zm9k7+9nEeiZGB+PlY2OByKuiz6/NxHutlTV4lj31ygJeyiyiqOdZbL69v7ZobyNZpZ0fx0aDXHr03cetcNWPMO8aYNGPMaGPMQ862ZcaYZc7HXzfGRBhjMp3/slzW/bkxZrwxJt0Yc5Mxpq1/NkWp/vOTKybwnXljmevGZG0v3zGLnJ9dzNafXkxSVBBWi/CHJZMJ8rOSHBXcddpnRkIY7TY7h6ua+O3VGYQF+hIXFsDeI/Ws2F1GsJ+Vb1/oOOsoOSqIec6/Fr77Yg7D/H155tbpjI4JprXDzrzxsUxJDGfdwSoefn8v1z2+jmfX53PxhFiigv14as2hrgO/R48hbDhYxezffsTvnX9p7C9vpMU5nq89eu+iUyAo5YZpyZF87+I0tyZAC/C1Eh7k123ZUVHB/PPmLH5+1cSutrNHRWAR+L/zRzMz1XEl8LgRIazJq+Lt7aWcNy6GKyY7evmZieEkRAQyOiaY8SNC+N+ds4kLC+TaaY7DZ+elxTAjNYpdJfX87eM8qpvaCfC18q3zRzNrdBS7S+uxWoT48EA+zC2juLaF/3tuCza74bkN+TS12boODAf4WnoN+m2Ftewu0Ru7n2l0CgSlBsg5Y7r/NZAQEcTqH13IyLBjF4tPS47kk70VpMeHcucFY4gLC+TXiyeRmRiOiPDGXecS4GvF6rwe4OZZyYQH+jE3LYbwIF/++tF+/u/80fzAZYhp35gG3tpeyjljojkrLoSn1hzixn+up91m53fXZHDvK9t5bWsxu0rqCQ3wIT0+jKKaZupbO1i+uYhrshIZ5u9Dc7uNrz29kRFhgbx795yB23HqC5PBeNl1VlaWyc7O9nQZSg04u93Q1G476d25TqamqZ2IYL9ubSW1LZz/8Cf84cuTiQsL4Jpl6xjm78Mzt05nalI4V/5tDdWN7XQaQ1psCCPDAvlobzm3nZvCb9/dw5jhw1j2lbP5ZG85v3rbcS7Fhh/PIzY0gKY2G797bw93XjiG4SE6u4knichm12FzVzp0o9QgYrHI5w554ISQBxgZHsjmn1zElZNHMiUpgu9eNJb/fmMGZ4+KQES4f8FZ+PpYqGnuYN744SRGBlLR0MZHe8qJDfWnuqmdK/66mr9+lEeSc06ho1f7vp5TwjPr8nllcxEA9a0dXdcBbCusPWE+H+UZGvRKDQFHf3lYLcJ3L0ojwzkjKDiGlD794QXsfXA+XzsnhYQIR5hvPFTNgvQ43r17DjNTo6hv7WDp1RnEhPjzqTPoX9nsuGj+kz0VHKpsIutXH/LW9lIKqppZ9PfPeHFTwcBuqOqRjtErpYBjVw8nRAR2tc0aHUVsaABPf20aFQ1tDA8N6Jr/Z39ZA1sKaokJ8WdzQQ2PfZJHu83O6zklVDS0YQwcrtLTNAcD7dErpbo52qMXgZkpUc7HwvBQxxj8eWkx1LV0cP0/N2C1CA8unEin3fBSdhEisHp/BW9td1w8X1yrp2kOBhr0Sqluhof442sVJo4MJSzoxOMFl02K44eXjmNUVBA3zxrFxRNGEBboWO6uC8bQZrOzpaAWcBwIVp6nQzdKqW4sFuGqyfFMHRXe4+tWi3DnBWO484IxXW0L0keQU1jLd+aN5d/r8qlr6WD8iBDt0Q8SGvRKqRP8YcnkUy/k4leL0rHZDb5WCwvSR/Dx3nKuyIjj4Q/20dRmI9hfo8aTdOhGKfWF+VgtXVM7/PzKibz17Tldt3csrdNevadp0Cul+lSgn5WYEH/iwx1n7xTXun9PXdU/NOiVUv1i5NGg1wnSPE6DXinVL4aH+GO1iJ55Mwho0Cul+oWP1cKI0AAN+kFAg14p1W/iwwMpqmlh0+FqGlo7Tr2C6hca9EqpfhMfEcjGw9V8edk6nlxzyNPlDFka9EqpfpOVHMGI0ABCA3y6bnCuBp4GvVKq39w4YxTrfzyPrORIDlY0ebqcIUuDXinV71Kigzlc1YTdPvhudDQUaNArpfpdqvMm5kfq9eIpT3Ar6EVkvojsFZE8Ebmvh9dvFJHtzn9rRWSyy2vhIvKKiOwRkVwRmdWXG6CUGvxSooMBdPjGQ04Z9CJiBR4FFgATgOtFZMJxix0CzjPGZAAPAo+7vPZn4D1jzHhgMpDbF4Urpc4cqdHDADhU2ejhSoYmd3r004E8Y8xBY0w78AKw0HUBY8xaY0yN8+l6IAFAREKBucCTzuXajTG1fVS7UuoMERvqT5CflYOV2qP3BHeCPh4odHle5GzrzW3Au87HqUAF8LSIbBWRJ0QkuKeVROR2EckWkeyKigo3ylJKnSlEhJTo4G5DN5/3xuHGGJZvKaKxzdZX5Xk9d4Jeemjr8dC5iFyAI+h/5GzyAaYCjxljpgBNwAlj/ADGmMeNMVnGmKyYmBg3ylJKnUlSooM55OzR/+PTA2T9agV1Lb1fLWu3G3YW12FM97jZVVLP917axn835Pdrvd7EnaAvAhJdnicAJccvJCIZwBPAQmNMlcu6RcaYDc7nr+AIfqXUEDN2eAiFNc08s/Ywf1ixj6b2TnaV1PW4bF1LB7c9s4kr/rqGf6/rHuhbChyjxBsPVfd7zd7CnaDfBIwVkRQR8QOuA95wXUBEkoDlwE3GmH1H240xR4BCERnnbJoH7O6TypVSZ5SbZo1i4shQfv7GLvysjujZXVJ/wnKF1c0s/vtnrN5fSWpMMA+/v5dyl9MytzrvR7vpcI2el++mUwa9McYG3AW8j+OMmZeMMbtE5A4RucO52M+AKODvIpIjItkub/Ft4DkR2Q5kAr/uyw1QSp0ZIoP9eP4bM/ny2Qn8cclkhof4s7u0nq0FNcz/0yo2HqpmS0ENX3psLZUNbTz39Rk8cXMWbTY7l/91DRf/8VN2lziW9/OxUNfSwf5y98/i2VVSx41PrOdARfd1jDHsK2s4YYhooK3ZX8kLGwvo6LT3+XuLpzeuJ1lZWSY7O/vUCyqlzlhfe3ojR+payUgI46XsIvx8LNjthhFhATz1tWmkxYYA8Pb2Ut7aXsKa/ZWkx4ex7mAV109P4vmNBTy4KJ2bZo7q9TMqG9tYvb+CRZnx3PNiDv/LKSEuLICXvjmr61aHj686wK/f2cOzt83g3LHRA7LtPVnyj3WU1Lbw6Q8vwGrp6dDoyYnIZmNMVk+v6ZWxSimPmBAXSl55Ix/mlnNeWgwzUiKZnz6Ct78zpyvkAS7PiOOxr5zNTbNGse6g4/DfwsyRxIb6s+kU4/T3vrKde17cxkvZhby36whzxkbT1Gbjl286RpDXH6xi6Xt7Afh0X/lp1W+Mod3WN73v3NJ6Nh6q5uZZoz5XyJ+KBr1SyiMmjAzFZjdUN7Xzpanx/Oe2GfzthqmEBfr2uPzXZifjZ7VgtQgZCWFMT4li7YFKWjs6WbG7jFv/tYl/fHqAz/IqKa5t4cPdZXy0pxx/HwsPvLaT1g4791ycxuIp8azJq6C1o5Of/m8nSZFBTE4MZ+2BKkpqW1jw59XsPdJ9ps3Kxjbm/2kVK3PLutr+sz6f6b/+kKKa5q62Z9Ye5t5XtvHnD/fT2uH+6aP/XncYfx8LS7IST73w5+DTL++qlFKnMHFkGAA+FuH8ccNPufzw0ABuPTeF/Komgvx8uGF6Em9uK+GxTw7w3IZ8mts7+WhP9155akwwP15wFl//dzap0cFMSQynrqWDZ9bl88zaw+wvb+RXi9KpaWrnjx/u45EV+8gtreejPeWMG3Hsr4pfv5PLniMNvLipkHlnxQLwek4Jtc0dPPDaTv51yzTabHYeejsXX6vQ1N5Jbmk9j9449ZQ99LqWDl7bWsyizHjCg/xOdze6RYNeKeURoyKDCPazkpkU3msv/nj3LRjf9XjW6CjOHRPNn1fuRwTeuPNcYsP8yStr5FBVEwXVzVyZMZKJI0P55nmpZCaEIyLMSo3C38fCIx/uw8ciXDYpjkOVTfxhxT5e3lwEwI7i2q7PWXegiuVbigkL9GX1fsdfEE1tNrYU1JAaE8yn+yp4Y1sJI0IDaO+08+iNWRRUN/PgW7v5y8r93HNxWo/bYoxBRHhjWwmtHXZunJn0+XfmKejQjVLKIywW4ZFrM/nJ5cdPneW+H1zqOHP7ummJTEoIY3hIALPHRHPjjFHcv+As0uPDEBHuX3AWCybFARDga2XW6ChaO+zMGRtNZLAfGQlhBPtZAUiOCmJ7keP8fmMMv3k3l/jwQH53TQYtHZ18llfJx3srMAYeWZLJ6JhgnltfQHa+4/z+rFER3HZuCueMieL9XUe6ajXGkH24mnabnbV5lUx7aCXv7zrCK9mFjB8RwqT4sM+9H05Fe/RKKY+5ZOKIL7R+ZmI4H9wzt2t2THednxbDJ3srWJjpmM3F12rhwrNiOVLXwryzYvntu3uoaWpnS0EN24vq+N3VGZw/LoZh/j58mFtGbXMHsaH+ZCSEcdXkeP60ch+NbTbGDh9GRLBj+GVaciR/XrmfxjYbw/x9WLG7jNv/s5n0+FDyq5ppaLXx/Ze20dhm4yeXn4VI3x+EPUp79EqpM1pabAi+1tOLsqvPTuBH88ezYNKxXzSPLJnMc1+fSYazZ729uI4/fbifpMggFk+Nx9/HynlpMTy/sZB3dx7hwvGxiAiXZ4zAGNhdWs+0lMiu95uSFIExsL2oFoA3t5cSEuBDUU0LAb5W/nPbdGx2Oz4WYfGUk00f9sVpj14pNeSEBPjyrfNHd2vzcf6ymOgM+gff2k1eeSMPf3ly1y+S712SRnJ0EAE+Vq7JSgBgzPAQxo8IYc+RBqYnHwv6zIRwwHEl75TECFbmlrFoSjw/unQ8ncYQGezHn67NpKy+jahh/v26vRr0SinlIizQl5ToYPLKG7norFiunnqstz06Zhg/vHT8CetcOXkk+8r2duvRhwX5khoTzNaCWj6JLqe5vZPLJ8URFnTswPP89Lj+3RgnDXqllDrOzNRI2m12Hv5yhltj59+Yk8p5aTHEhwd2a5+SGMEne8upb+kgKtiPGS6/CAaSBr1SSh3n/y1Mp6PTTpCfexHp52MhvYezZjKTwnl1SxF1LTX84qqJXcNDA02DXimljuNrtZz2Ad6eXJkRx6GKJq6bnthtWoeBpkGvlFL9JDzIj59d+fmvE+grenqlUkp5OQ16pZTychr0Sinl5TTolVLKy2nQK6WUl9OgV0opL6dBr5RSXk6DXimlvJwYYzxdwwlEpALI/5yrRwOVfVhOX9G6Tt9grU3rOj1a1+n7PLWNMsbE9PTCoAz6L0JEso0xWZ6u43ha1+kbrLVpXadH6zp9fV2bDt0opZSX06BXSikv541B/7inC+iF1nX6BmttWtfp0bpOX5/W5nVj9Eoppbrzxh69UkopFxr0Sinl5bwm6EVkvojsFZE8EbnPg3UkisjHIpIrIrtE5G5n+y9EpFhEcpz/LvNQfYdFZIezhmxnW6SIrBCR/c7/RgxwTeNc9kuOiNSLyHc9sc9E5CkRKReRnS5tve4fEbnf+Z3bKyKXeqC234vIHhHZLiKviUi4sz1ZRFpc9t2yAa6r15/dQO2zXup60aWmwyKS42wfyP3VW0b03/fMGHPG/wOswAEgFfADtgETPFRLHDDV+TgE2AdMAH4B/GAQ7KvDQPRxbb8D7nM+vg9Y6uGf5RFglCf2GTAXmArsPNX+cf5ctwH+QIrzO2gd4NouAXycj5e61JbsupwH9lmPP7uB3Gc91XXc638AfuaB/dVbRvTb98xbevTTgTxjzEFjTDvwArDQE4UYY0qNMVucjxuAXCDeE7WchoXAM87HzwCLPFcK84ADxpjPe2X0F2KMWQVUH9fc2/5ZCLxgjGkzxhwC8nB8FwesNmPMB8YYm/PpeiChvz7/dOo6iQHbZyerS0QEWAI83x+ffTInyYh++555S9DHA4Uuz4sYBOEqIsnAFGCDs+ku55/YTw308IgLA3wgIptF5HZnW6wxphQcX0JguIdqA7iO7v/zDYZ91tv+GWzfu1uBd12ep4jIVhH5VETmeKCenn52g2WfzQHKjDH7XdoGfH8dlxH99j3zlqCXHto8et6oiAwDXgW+a4ypBx4DRgOZQCmOPxs94RxjzFRgAXCniMz1UB0nEBE/4CrgZWfTYNlnvRk03zsReQCwAc85m0qBJGPMFOB7wH9FJHQAS+rtZzdY9tn1dO9QDPj+6iEjel20h7bT2mfeEvRFQKLL8wSgxEO1ICK+OH6AzxljlgMYY8qMMZ3GGDvwT/rxT/yTMcaUOP9bDrzmrKNMROKctccB5Z6oDccvny3GmDJnjYNin9H7/hkU3zsR+SpwBXCjcQ7qOv/Mr3I+3oxjXDdtoGo6yc/O4/tMRHyALwEvHm0b6P3VU0bQj98zbwn6TcBYEUlx9gqvA97wRCHOsb8ngVxjzB9d2uNcFlsM7Dx+3QGoLVhEQo4+xnEgbyeOffVV52JfBV4f6NqcuvWyBsM+c+pt/7wBXCci/iKSAowFNg5kYSIyH/gRcJUxptmlPUZErM7Hqc7aDg5gXb397Dy+z4CLgD3GmKKjDQO5v3rLCPrzezYQR5kH6Ej2ZTiOXh8AHvBgHefi+LNqO5Dj/HcZ8B9gh7P9DSDOA7Wl4jh6vw3YdXQ/AVHASmC/87+RHqgtCKgCwlzaBnyf4fhFUwp04OhJ3Xay/QM84PzO7QUWeKC2PBzjt0e/a8ucy17t/BlvA7YAVw5wXb3+7AZqn/VUl7P9X8Adxy07kPurt4zot++ZToGglFJezluGbpRSSvVCg14ppbycBr1SSnk5DXqllPJyGvRKKeXlNOiVUsrLadArpZSX+/8hNOvuG3L/RgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.plot(torch.tensor(lossi).view(-1, 1000).mean(1))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ehknU7xtbHwz" }, "outputs": [], "source": [ "# put layers into eval mode (needed for batchnorm especially)\n", "for layer in model.layers:\n", " layer.training = False" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5W1wFpKubHwz", "outputId": "a965238b-34f8-42de-e6d6-3777a6837cd1" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train 1.7690284252166748\n", "val 1.9936515092849731\n" ] } ], "source": [ "# evaluate the loss\n", "@torch.no_grad() # this decorator disables gradient tracking inside pytorch\n", "def split_loss(split):\n", " x,y = {\n", " 'train': (Xtr, Ytr),\n", " 'val': (Xdev, Ydev),\n", " 'test': (Xte, Yte),\n", " }[split]\n", " logits = model(x)\n", " loss = F.cross_entropy(logits, y)\n", " print(split, loss.item())\n", "\n", "split_loss('train')\n", "split_loss('val')" ] }, { "cell_type": "markdown", "metadata": { "id": "yQdB3cZYbHw0" }, "source": [ "### performance log\n", "\n", "- original (3 character context + 200 hidden neurons, 12K params): train 2.058, val 2.105\n", "- context: 3 -> 8 (22K params): train 1.918, val 2.027\n", "- flat -> hierarchical (22K params): train 1.941, val 2.029\n", "- fix bug in batchnorm: train 1.912, val 2.022\n", "- scale up the network: n_embd 24, n_hidden 128 (76K params): train 1.769, val 1.993\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "CSE5ZEgebHw0", "outputId": "b8f5baf1-143e-4fd6-8e14-cc1d1bd3a0b3" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "arlij.\n", "chetta.\n", "heago.\n", "rocklei.\n", "hendrix.\n", "jamylie.\n", "broxin.\n", "denish.\n", "anslibt.\n", "marianah.\n", "astavia.\n", "annayve.\n", "aniah.\n", "jayce.\n", "nodiel.\n", "remita.\n", "niyelle.\n", "jaylene.\n", "aiyan.\n", "aubreana.\n" ] } ], "source": [ "# sample from the model\n", "for _ in range(20):\n", "\n", " out = []\n", " context = [0] * block_size # initialize with all ...\n", " while True:\n", " # forward pass the neural net\n", " logits = model(torch.tensor([context]))\n", " probs = F.softmax(logits, dim=1)\n", " # sample from the distribution\n", " ix = torch.multinomial(probs, num_samples=1).item()\n", " # shift the context window and track the samples\n", " context = context[1:] + [ix]\n", " out.append(ix)\n", " # if we sample the special '.' token, break\n", " if ix == 0:\n", " break\n", "\n", " print(''.join(itos[i] for i in out)) # decode and print the generated word" ] }, { "cell_type": "markdown", "metadata": { "id": "5w054JOpbHw1" }, "source": [ "### Next time:\n", "Why convolutions? Brief preview/hint" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zBP4pbSnbHw1", "outputId": "5dc847b5-d40b-4e6a-f25b-55eedc770b39" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "........ --> d\n", ".......d --> i\n", "......di --> o\n", ".....dio --> n\n", "....dion --> d\n", "...diond --> r\n", "..diondr --> e\n", ".diondre --> .\n" ] } ], "source": [ "for x,y in zip(Xtr[7:15], Ytr[7:15]):\n", " print(''.join(itos[ix.item()] for ix in x), '-->', itos[y.item()])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "HhUwWXX8bHw2", "outputId": "9d5d20ce-180c-4178-fd22-3e7a7dcc38ff" }, "outputs": [ { "data": { "text/plain": [ "torch.Size([1, 27])" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# forward a single example:\n", "logits = model(Xtr[[7]])\n", "logits.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gBDyYGUWbHw2", "outputId": "f288231e-6a20-4940-845f-596944950912" }, "outputs": [ { "data": { "text/plain": [ "torch.Size([8, 27])" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# forward all of them\n", "logits = torch.zeros(8, 27)\n", "for i in range(8):\n", " logits[i] = model(Xtr[[7+i]])\n", "logits.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PCoo_TvZbHw3" }, "outputs": [], "source": [ "# convolution is a \"for loop\"\n", "# allows us to forward Linear layers efficiently over space" ] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 0 }