{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# This exercise wasn't exactly smooth sailing for me, but I did try to understand most of it. Will try to come back to this whenever I can" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# there no change change in the first several cells from last lecture\n", "\n", "import torch\n", "import torch.nn.functional as F\n", "import matplotlib.pyplot as plt # for making figures\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# download the names.txt file from github\n", "!wget https://raw.githubusercontent.com/karpathy/makemore/master/names.txt" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# read in all the words\n", "words = open('names.txt', 'r').read().splitlines()\n", "# print(len(words))\n", "# print(max(len(w) for w in words))\n", "# print(words[:8])\n", "\n", "# build the vocabulary of characters and mappings to/from integers\n", "chars = sorted(list(set(''.join(words))))\n", "stoi = {s:i+1 for i,s in enumerate(chars)}\n", "stoi['.'] = 0\n", "itos = {i:s for s,i in stoi.items()}\n", "vocab_size = len(itos)\n", "# print(itos)\n", "# print(vocab_size)\n", "\n", "# build the dataset\n", "block_size = 3 # context length: how many characters do we take to predict the next one?\n", "\n", "def build_dataset(words):\n", " X, Y = [], []\n", "\n", " for w in words:\n", " context = [0] * block_size\n", " for ch in w + '.':\n", " ix = stoi[ch]\n", " X.append(context)\n", " Y.append(ix)\n", " context = context[1:] + [ix] # crop and append\n", "\n", " X = torch.tensor(X)\n", " Y = torch.tensor(Y)\n", " # print(X.shape, Y.shape)\n", " return X, Y\n", "\n", "import random\n", "random.seed(42)\n", "random.shuffle(words)\n", "n1 = int(0.8*len(words))\n", "n2 = int(0.9*len(words))\n", "\n", "Xtr, Ytr = build_dataset(words[:n1]) # 80%\n", "Xdev, Ydev = build_dataset(words[n1:n2]) # 10%\n", "Xte, Yte = build_dataset(words[n2:]) # 10%" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# utility function we will use later when comparing manual gradients to PyTorch gradients\n", "def cmp(s, dt, t):\n", " ex = torch.all(dt == t.grad).item()\n", " app = torch.allclose(dt, t.grad)\n", " maxdiff = (dt - t.grad).abs().max().item()\n", " print(f'{s:15s} | exact: {str(ex):5s} | approximate: {str(app):5s} | maxdiff: {maxdiff}')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "4137\n" ] } ], "source": [ "n_embd = 10 # the dimensionality of the character embedding vectors\n", "n_hidden = 64 # the number of neurons in the hidden layer of the MLP\n", "\n", "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n", "C = torch.randn((vocab_size, n_embd), generator=g)\n", "# Layer 1\n", "W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5)\n", "b1 = torch.randn(n_hidden, generator=g) * 0.1 # using b1 just for fun, it's useless because of BN\n", "# Layer 2\n", "W2 = torch.randn((n_hidden, vocab_size), generator=g) * 0.1\n", "b2 = torch.randn(vocab_size, generator=g) * 0.1\n", "# BatchNorm parameters\n", "bngain = torch.randn((1, n_hidden))*0.1 + 1.0\n", "bnbias = torch.randn((1, n_hidden))*0.1\n", "\n", "# Note: I am initializating many of these parameters in non-standard ways\n", "# because sometimes initializating with e.g. all zeros could mask an incorrect\n", "# implementation of the backward pass.\n", "\n", "parameters = [C, W1, b1, W2, b2, bngain, bnbias]\n", "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n", "for p in parameters:\n", " p.requires_grad = True" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "batch_size = 32\n", "n = batch_size # a shorter variable also, for convenience\n", "# construct a minibatch\n", "ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)\n", "Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(3.3596, grad_fn=)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# forward pass, \"chunkated\" into smaller steps that are possible to backward one at a time\n", "\n", "emb = C[Xb] # embed the characters into vectors\n", "embcat = emb.view(emb.shape[0], -1) # concatenate the vectors\n", "# Linear layer 1\n", "hprebn = embcat @ W1 + b1 # hidden layer pre-activation\n", "# BatchNorm layer\n", "bnmeani = 1/n*hprebn.sum(0, keepdim=True)\n", "bndiff = hprebn - bnmeani\n", "bndiff2 = bndiff**2\n", "bnvar = 1/(n-1)*(bndiff2).sum(0, keepdim=True) # note: Bessel's correction (dividing by n-1, not n)\n", "bnvar_inv = (bnvar + 1e-5)**-0.5\n", "bnraw = bndiff * bnvar_inv\n", "hpreact = bngain * bnraw + bnbias\n", "# Non-linearity\n", "h = torch.tanh(hpreact) # hidden layer\n", "# Linear layer 2\n", "logits = h @ W2 + b2 # output layer\n", "# cross entropy loss (same as F.cross_entropy(logits, Yb))\n", "logit_maxes = logits.max(1, keepdim=True).values\n", "norm_logits = logits - logit_maxes # subtract max for numerical stability\n", "counts = norm_logits.exp()\n", "counts_sum = counts.sum(1, keepdims=True)\n", "counts_sum_inv = counts_sum**-1 # if I use (1.0 / counts_sum) instead then I can't get backprop to be bit exact...\n", "probs = counts * counts_sum_inv\n", "logprobs = probs.log()\n", "loss = -logprobs[range(n), Yb].mean()\n", "\n", "# PyTorch backward pass\n", "for p in parameters:\n", " p.grad = None\n", "for t in [logprobs, probs, counts, counts_sum, counts_sum_inv, # afaik there is no cleaner way\n", " norm_logits, logit_maxes, logits, h, hpreact, bnraw,\n", " bnvar_inv, bnvar, bndiff2, bndiff, hprebn, bnmeani,\n", " embcat, emb]:\n", " t.retain_grad()\n", "loss.backward()\n", "loss" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Similar boiler plate codes as done in the prev exercise and provided in the starter code^\n", "\n", "------------" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Exercise 2: backprop through cross_entropy but all in one go\n", "# to complete this challenge look at the mathematical expression of the loss,\n", "# take the derivative, simplify the expression, and just write it out\n", "\n", "# forward pass\n", "\n", "# before:\n", "# logit_maxes = logits.max(1, keepdim=True).values\n", "# norm_logits = logits - logit_maxes # subtract max for numerical stability\n", "# counts = norm_logits.exp()\n", "# counts_sum = counts.sum(1, keepdims=True)\n", "# counts_sum_inv = counts_sum**-1 # if I use (1.0 / counts_sum) instead then I can't get backprop to be bit exact...\n", "# probs = counts * counts_sum_inv\n", "# logprobs = probs.log()\n", "# loss = -logprobs[range(n), Yb].mean()\n", "\n", "# now:\n", "# loss_fast = F.cross_entropy(logits, Yb)\n", "# print(loss_fast.item(), 'diff:', (loss_fast - loss).item())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In the above example we are seeing how the forward pass is broken down if we do the manual breakdown of calculation vs just directly using PyTorch" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[1:28:34](https://youtu.be/q8SA3rM6ckI?si=O-RCp2YO7QbSbUIW&t=5314) to 1:32:48 - Andrej sensei gives us an hint followed with an explaination of solving the equation and convert that to code" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "logits | exact: False | approximate: True | maxdiff: 8.381903171539307e-09\n" ] } ], "source": [ "# backward pass\n", "\n", "dlogits = F.softmax(logits, 1)\n", "dlogits[range(n), Yb] -= 1\n", "dlogits /= n\n", "\n", "cmp('logits', dlogits, logits)\n", "\n", "#This wasnt exactly very clear to me, but i will come back to this\n", "#Also my output came slightly bigger than sensei's though" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "[1:32:49](https://youtu.be/q8SA3rM6ckI?si=-204uFZWpJPaT9oU&t=5569) to 1:36:36 - Breakdown of what `dlogits` actually is by taking one row and representing it dynamically" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([32, 27]), torch.Size([32]))" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "logits.shape, Yb.shape" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([0.0727, 0.0823, 0.0164, 0.0532, 0.0213, 0.0895, 0.0218, 0.0357, 0.0174,\n", " 0.0327, 0.0371, 0.0337, 0.0347, 0.0311, 0.0346, 0.0131, 0.0086, 0.0178,\n", " 0.0161, 0.0499, 0.0532, 0.0226, 0.0259, 0.0712, 0.0607, 0.0274, 0.0192],\n", " grad_fn=)" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "F.softmax(logits, 1)[0]" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([ 0.0727, 0.0823, 0.0164, 0.0532, 0.0213, 0.0895, 0.0218, 0.0357,\n", " -0.9826, 0.0327, 0.0371, 0.0337, 0.0347, 0.0311, 0.0346, 0.0131,\n", " 0.0086, 0.0178, 0.0161, 0.0499, 0.0532, 0.0226, 0.0259, 0.0712,\n", " 0.0607, 0.0274, 0.0192], grad_fn=)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dlogits[0] * n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(2.0955e-09, grad_fn=)" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dlogits[0].sum()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAKTCAYAAADlpSlWAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAMVBJREFUeJzt3X2MXXWdP/DPnTuPpdPB8tCHpS0FhKrQmiDURmVRupS6ISI1wYdkwRCMbiELjavpRkVck+5iov7cIP6zC2ti1WUjGEgWo1VKzBZcakhFobSlLmWhZYW00047T/ee3x9NZx1pgWk/5Q7fvl7JTTozt+/53HPP98x7zsycW6uqqgoAgEK0tXoAAIBMyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKK0t3qAP9VsNuP555+P3t7eqNVqrR4HAJgEqqqKvXv3xuzZs6Ot7dXPzUy6cvP888/HnDlzWj0GADAJ7dixI84444xXvc+kKze9vb0REfHrX/967N/H4rXa3UT09/enZUVEdHV1pWUNDw+nZfX19aVlRUTs2bMnLater6dlXXDBBWlZv/nNb9KyIiL1rOVkvQh59lyZ22xkZCQtK9NkPpudeTzLlHlsjMh9DqZMmZKW1Ww207IGBwfTsiLy1vq+ffvive997+vqBpOu3BzacXp7eydduck+GE/WcpOx3f9Y5qLLLDeZB6nsbabcTNxkLTeZjzPzeJZNuZm4yVpuOjo60rIiWrPWJ+9KAQA4CsoNAFAU5QYAKMpxKzd33HFHnHnmmdHd3R2LFy+OX/3qV8frUwEAjDku5eaHP/xhrFq1Km699db49a9/HYsWLYply5bFiy++eDw+HQDAmONSbr7+9a/HDTfcEJ/85Cfj7W9/e3znO9+JKVOmxL/8y78cj08HADAmvdwMDw/Hxo0bY+nSpf/3SdraYunSpbFhw4ZX3H9oaCj6+/vH3QAAjlZ6ufnDH/4QjUYjZsyYMe79M2bMiJ07d77i/mvWrIm+vr6xm6sTAwDHouV/LbV69erYs2fP2G3Hjh2tHgkAeBNLv0LxqaeeGvV6PXbt2jXu/bt27YqZM2e+4v5dXV2T9sqWAMCbT/qZm87Ozrjwwgtj3bp1Y+9rNpuxbt26WLJkSfanAwAY57i8ttSqVavi2muvjXe9611x8cUXxze/+c0YGBiIT37yk8fj0wEAjDku5eaaa66J//3f/40vfelLsXPnznjnO98ZDz744Ct+yRgAINtxe1XwG2+8MW688cbjFQ8AcFgt/2spAIBMyg0AUJTj9mOpYzUyMhIjIyOtHmOc6dOnp+YNDg6mZbW35z2Ve/fuTcvKVq/X07K2bNmSltVsNtOyIiI6OjrSsqqqSsuq1WppWY1GIy0rIuLcc89Ny8rcNzIfZ/Z+Nlmfz8yszMcYkfscZGZlfj1pa8s975H1OCfyXDpzAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIrS3uoBjmRoaCg6OzuPOaetLa+/vfzyy2lZERFVVaVlZT7Ojo6OtKyISHkeD8l8nJlZw8PDaVnZeZmPs16vT8qsiIgnn3wyLeuss85Ky3rqqafSsrLXZuYxqK+vLy1rcHAwLSt7bWbutyMjI2lZmXONjo6mZUVE1Gq11LzXw5kbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoSnurBziSer0e9Xr9mHOazWbCNAd1dHSkZUVEtLfnbf6MbXXIgQMH0rKyNRqNVo9wWJn7WUTuvjE6OpqWlSlzn42I6OzsTMv6n//5n7Ss/fv3p2Vl7/9VVaVl7du3Ly1reHg4LatWq6VlRUQsWLAgLevJJ59My2pryztXkbmWIvL2s4kcF525AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVpb/UAR7Jw4cKUnK1bt6bkREQ0Go20rOy8ZrOZltXR0ZGWFZE72+joaFpWV1dXWlZ7e+5SqqoqLStz+2c+zsznMiJ3thkzZqRlPfPMM2lZnZ2daVnZ2tryvlfOfJxDQ0NpWRERTz75ZFrWZD1uj4yMpGVF5B8fXw9nbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBR2ls9wJFs2rQpent7Wz3GOO3tuZsrM6+tLa+nDgwMpGVl6+npScsaGhpKy2o2m2lZERGdnZ1pWbVaLS2r0WikZXV0dKRlRUTU6/W0rOeeey4tq6qqtKzMfTYid7YFCxakZW3bti0tK/u4nXmsHR4enpRZ06ZNS8uKiBgcHEzNez2cuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAo6eXmy1/+ctRqtXG3zN+iBwB4NcflT8Hf8Y53xM9+9rP/+yTJf4oHAHAkx6V1tLe3x8yZM49HNADAqzouv3OzZcuWmD17dpx11lnxiU98Ip599tkj3ndoaCj6+/vH3QAAjlZ6uVm8eHHcfffd8eCDD8add94Z27dvj/e9732xd+/ew95/zZo10dfXN3abM2dO9kgAwAmkVmVef/swdu/eHfPmzYuvf/3rcf3117/i40NDQ+MuKd7f3x9z5szx8gsTdKK8/EJXV1da1ony8guZl2XP3M8m88svZL7MRPZLJmQ6EV5+IdtkffmFTJP15Rf27t0b73jHO2LPnj2vOeNx/03fk08+Oc4999zYunXrYT/e1dWV+gULADixHffr3Ozbty+2bdsWs2bNOt6fCgAgv9x89rOfjfXr18fvf//7+M///M/48Ic/HPV6PT72sY9lfyoAgFdI/7HUc889Fx/72MfipZdeitNOOy3e+973xiOPPBKnnXZa9qcCAHiF9HLzgx/8IDsSAOB189pSAEBRlBsAoCiT9kWf2tvbU64Dk/X39RG511iJyL2eTOY1PrIvfdTT05OalyXzOitnn312WlZExO9+97u0rMm6b0zm679kXmOru7s7LWv//v1pWRG511nJvDZN5n42mV/bsFarpWVlPs59+/alZUXkPc6JXH/KmRsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGAChKe6sHOJJmsxnNZvOYc9rb8x7i4OBgWlZExKxZs9KyXnzxxbSszs7OtKyI3O3W29ubljU8PJyW9cQTT6RlRUS0teV93zE6OpqWlTnXlClT0rIiImbPnp2WtXXr1rSsqqrSsrLVarW0rKlTp6ZlDQwMpGVlGxkZScuq1+tpWY1GIy2rq6srLSsib5tNZH915gYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAUpb3VAxxJW1tbtLUde/dqNBoJ0xzUbDbTsiIiXnrppbSs0dHRtKwFCxakZUVEbN26NS2rVqulZWU+n/V6PS0rIvdxtrfnLfOMNXnI4OBgWlZExJYtW9KyMrd/Zlb2fjaZj49Zenp6UvOqqkrNy5K5Ng8cOJCWFZG3305k2ztzAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIrS3uoBjmRkZCRGRkaOOeess85KmOag7du3p2VFRMrjO6SjoyMta/PmzWlZERGNRiMta9++fWlZvb29aVmZz2VExMDAQFpW5r6Rqb099/BTVVVaVltb3vd9XV1daVmjo6NpWRERtVotLStzbU6ZMiUtq7+/Py0rIqKnpyctK3Od1+v1tKzsY0bW8XEiX0ucuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFaW/1AEfSbDaj2Wwec87TTz+dMM1BbW25XbBer6dlNRqNtKxarZaWFRExOjqalpWxTxwPk3nfyNz+PT09aVnDw8NpWRER7e15h7NZs2alZe3atSstK3ttdnd3p2Xt378/LWvevHlpWU888URaVkTEvn370rIyjxuZWZlfTyLyZptIjjM3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKMuFy8/DDD8eVV14Zs2fPjlqtFvfdd9+4j1dVFV/60pdi1qxZ0dPTE0uXLo0tW7ZkzQsA8KomXG4GBgZi0aJFcccddxz247fffnt861vfiu985zvx6KOPxkknnRTLli2LwcHBYx4WAOC1TPiqV8uXL4/ly5cf9mNVVcU3v/nN+MIXvhAf+tCHIiLiu9/9bsyYMSPuu++++OhHP/qK/zM0NBRDQ0Njb/f39090JACAMam/c7N9+/bYuXNnLF26dOx9fX19sXjx4tiwYcNh/8+aNWuir69v7DZnzpzMkQCAE0xqudm5c2dERMyYMWPc+2fMmDH2sT+1evXq2LNnz9htx44dmSMBACeYlr+2VFdXV3R1dbV6DACgEKlnbmbOnBkRr3yhuF27do19DADgeEotN/Pnz4+ZM2fGunXrxt7X398fjz76aCxZsiTzUwEAHNaEfyy1b9++2Lp169jb27dvj8cffzymT58ec+fOjZtvvjm++tWvxlvf+taYP39+fPGLX4zZs2fHVVddlTk3AMBhTbjcPPbYY/H+979/7O1Vq1ZFRMS1114bd999d3zuc5+LgYGB+NSnPhW7d++O9773vfHggw9Gd3d33tQAAEcw4XJz6aWXRlVVR/x4rVaLr3zlK/GVr3zlmAYDADgaXlsKACiKcgMAFKXl17k5klqtFrVa7ZhzOjo6EqY5aHR0NC0rIuIv//Iv07Luv//+tKzs34/q7OxMy2o0GmlZr/bj1YnKnCsiotlspuZlyXyNuIz1/cf++GVcjtXvf//7tKx6vT4psyJyX+4m83plzzzzTFpW9tocGRlJy8p8PjPXU3t7bjUYHh5OyZnIcdGZGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFCU9lYPcCRVVUVVVcecMzo6mjDNQd3d3WlZEREPPPBAWlZbW15PHRoaSsuKiOjt7U3Lynw+FyxYkJa1efPmtKyIiEajkZbV3p63zDP3s8zHGBFRq9XSsrq6utKyOjs707KGh4fTsiIiOjo60rIyZ8vcZtlOPvnktKyXX345LStzbWaupYiIer3+huc4cwMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCK0t7qAY6kVqtFrVY75py2tsnb3zIe3yHNZjMtq7e3Ny0rImJgYCAtK/NxPvHEE2lZ2er1elpWVVVpWZ2dnWlZQ0NDaVkREeeff35a1tatW9OyDhw4kJaVbcqUKWlZ/f39aVkdHR1pWfv27UvLiogYHh5Oy8p8nJkyvzZF5B2DJjLX5P3KDwBwFJQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAo7a0e4Eg6Ojqio6PjmHNGR0cTpsnPiojo6elJy9q/f39a1oEDB9KyIiJqtVpa1pQpU9KyGo1GWtZk1taW9z3MmWeemZb19NNPp2VFRDz55JNpWSMjI2lZVVWlZXV2dqZlReQeN7q7u9OyMtdmV1dXWlZE7r6R6UQ4nk3kMTpzAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARWlv9QBH8s53vjNqtdox52zfvj1hmoOGh4fTsiIiDhw4kJaVsa0OmTJlSlpWRMTAwEBa1uDgYFpWpo6OjtS8zOczM+uZZ55Jy8rc/yNyH2ez2UzLytw3hoaG0rIiIrq6utKyMmfL3GaNRiMtKyKirS3vnEDm9q+qKi0rez/LXE+vlzM3AEBRlBsAoCjKDQBQFOUGACiKcgMAFGXC5ebhhx+OK6+8MmbPnh21Wi3uu+++cR+/7rrrolarjbtdccUVWfMCALyqCZebgYGBWLRoUdxxxx1HvM8VV1wRL7zwwtjt+9///jENCQDwek34OjfLly+P5cuXv+p9urq6YubMmUc9FADA0Touv3Pz0EMPxemnnx7nnXdefOYzn4mXXnrpiPcdGhqK/v7+cTcAgKOVXm6uuOKK+O53vxvr1q2Lf/zHf4z169fH8uXLj3iVyDVr1kRfX9/Ybc6cOdkjAQAnkPSXX/joRz869u8LLrggFi5cGGeffXY89NBDcdlll73i/qtXr45Vq1aNvd3f36/gAABH7bj/KfhZZ50Vp556amzduvWwH+/q6opp06aNuwEAHK3jXm6ee+65eOmll2LWrFnH+1MBAEz8x1L79u0bdxZm+/bt8fjjj8f06dNj+vTpcdttt8WKFSti5syZsW3btvjc5z4X55xzTixbtix1cACAw5lwuXnsscfi/e9//9jbh35f5tprr40777wzNm3aFP/6r/8au3fvjtmzZ8fll18ef//3f5/60u4AAEcy4XJz6aWXRlVVR/z4T37yk2MaCADgWHhtKQCgKMoNAFCU9OvcZHnssceit7f3mHMGBwcTpjkoY54/ljlbZ2dnWtbQ0FBaVkQc8QKOR6Ner6dlZc41PDyclhWR+3zOnTs3Lev3v/99WlZPT09aVrZarZaWNTAwkJaVLXO/7ejoSMvKXJvNZjMtKyJ3tra2vPMLo6OjaVnt7bnVICtvIsdFZ24AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUdpbPcCRXHTRRVGr1Y4557//+78TpjloaGgoLSsiol6vp2UNDw+nZTUajbSsiEh5Hg+ZMmVKWtbAwEBaVrPZTMuKiOjs7EzLevrpp9OyMveN0dHRtKyIiPb2vMNZ5vNZVVVaVuYxIyL3cba15X2vnLlvdHV1pWVFRIyMjKRlZR63M7d/tqz9diI5k3drAAAcBeUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGAChKe6sHOJJHH300ent7jzln7969CdMc1N3dnZYVEXHgwIG0rLa2vJ7abDbTsiIi5Xk8ZGhoKC2rq6srLSt7m2Xutx0dHWlZmftZo9FIy4qIGB4eTsvK3Dd6enrSsjIfY0REVVVpWZmzdXZ2pmVlPsaIiL6+vrSsl19+OS0r83GOjo6mZUVEzJ8/PyVnIo/RmRsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGAChKe6sHOJJarRa1Wi0lJ0uj0UjLylav19Oy2tpyO+/o6GhaVubzOTw8nJZ17rnnpmVFRGzZsiUtK/P5bG/PO2RkZkXk7meTNavZbKZlReQeN6ZOnZqWlbk2s7fZwMBAWlZ3d3daVuZ+VlVVWlZE3vFs7969sWjRotd1X2duAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFHaWz3AkXR2dkZnZ+cx5wwODiZMc1BVVWlZEREdHR1pWZmz1Wq1tKyIiKGhobSs9va8XTYz66mnnkrLiojo6upKy8rc/pn7WebajIiU48Uh3d3daVl79+5Ny8pem5l5w8PDkzKrXq+nZUVEjI6OpuZlyXycb3vb29KyIiKefPLJlJyJPEZnbgCAoig3AEBRlBsAoCjKDQBQFOUGACjKhMrNmjVr4qKLLore3t44/fTT46qrrorNmzePu8/g4GCsXLkyTjnllJg6dWqsWLEidu3alTo0AMCRTKjcrF+/PlauXBmPPPJI/PSnP42RkZG4/PLLY2BgYOw+t9xyS9x///1xzz33xPr16+P555+Pq6++On1wAIDDmdCFPh588MFxb999991x+umnx8aNG+OSSy6JPXv2xD//8z/H2rVr4wMf+EBERNx1113xtre9LR555JF497vfnTc5AMBhHNPv3OzZsyciIqZPnx4RERs3boyRkZFYunTp2H0WLFgQc+fOjQ0bNhw2Y2hoKPr7+8fdAACO1lGXm2azGTfffHO85z3vifPPPz8iInbu3BmdnZ1x8sknj7vvjBkzYufOnYfNWbNmTfT19Y3d5syZc7QjAQAcfblZuXJlPPHEE/GDH/zgmAZYvXp17NmzZ+y2Y8eOY8oDAE5sR/XiOjfeeGM88MAD8fDDD8cZZ5wx9v6ZM2fG8PBw7N69e9zZm127dsXMmTMPm9XV1ZX6OjoAwIltQmduqqqKG2+8Me699974+c9/HvPnzx/38QsvvDA6Ojpi3bp1Y+/bvHlzPPvss7FkyZKciQEAXsWEztysXLky1q5dGz/+8Y+jt7d37Pdo+vr6oqenJ/r6+uL666+PVatWxfTp02PatGlx0003xZIlS/ylFADwhphQubnzzjsjIuLSSy8d9/677rorrrvuuoiI+MY3vhFtbW2xYsWKGBoaimXLlsW3v/3tlGEBAF7LhMpNVVWveZ/u7u6444474o477jjqoQAAjpbXlgIAiqLcAABFOao/BX8jvPOd74xarXbMOdu3b0+Y5qBGo5GWlW1kZCQtK/tP85vNZlpWxj5xyPDwcFrW6/mR7URkbrPMrKGhobSser2elpUtc9/IlL3NMo9pJ510UlrW4OBgWlb2NstcT5N1Dfz2t79Nzcs6Pk4kx5kbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUJT2Vg9wJI8++mj09vYec87s2bMTpjlox44daVkREUNDQ2lZbW15PfXAgQNpWRGR8jwekrnNuru707KazWZaVkTuc9DenrfMM/ezRqORlhURMTIykpbV1dWVljV16tS0rOHh4bSsiIharZaW1d/fn5aVuTarqkrLioh4y1vekpb18ssvp2Vlrs3M/SLTRJ5LZ24AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAo7a0e4Eg6Ozujs7PzmHNqtVrCNAeNjIykZUVEVFWVltXV1ZWWNTQ0lJYVEdFsNidl1ujoaFpWvV5Py4qIaG+fnEszc/tnrs2IiI6OjrSszNkm8zEoc7/NPJ4NDw+nZWXvZ5lrM3O27u7utKzM7R8R0Wg03vAcZ24AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUdpbPcCRNBqNaDQax5zzwgsvJExz0MDAQFpWRER3d3da1vDwcFpWT09PWlZExP79+9OyzjvvvLSsp59+Oi2r2WymZUVEnHzyyWlZL730UlpWvV5PyxodHU3Liojo6OhIy8pcT0NDQ2lZVVWlZUXk7reZ+0bGsf+QzLkiIl588cW0rHnz5qVl7dq1Ky0rez/r6upKyZnIunTmBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABSlvdUDHElXV1d0dXUdc87+/fsTpjmoqqq0rIiI4eHhtKx6vT4psyIi2tvzdrMtW7akZWU+n7VaLS0rImLPnj1pWRnr6JC2trzvh7K32ejoaGpelsz11Gw207IiIs4///y0rE2bNqVlZR4zsrfZ1KlT07J27dqVltXR0ZGWlf21bnBw8A3PceYGACiKcgMAFEW5AQCKotwAAEVRbgCAokyo3KxZsyYuuuii6O3tjdNPPz2uuuqq2Lx587j7XHrppVGr1cbdPv3pT6cODQBwJBMqN+vXr4+VK1fGI488Ej/96U9jZGQkLr/88hgYGBh3vxtuuCFeeOGFsdvtt9+eOjQAwJFM6GICDz744Li377777jj99NNj48aNcckll4y9f8qUKTFz5sycCQEAJuCYfufm0IXGpk+fPu793/ve9+LUU0+N888/P1avXv2qF9IbGhqK/v7+cTcAgKN11JeBbDabcfPNN8d73vOecVe5/PjHPx7z5s2L2bNnx6ZNm+Lzn/98bN68OX70ox8dNmfNmjVx2223He0YAADjHHW5WblyZTzxxBPxy1/+ctz7P/WpT439+4ILLohZs2bFZZddFtu2bYuzzz77FTmrV6+OVatWjb3d398fc+bMOdqxAIAT3FGVmxtvvDEeeOCBePjhh+OMM8541fsuXrw4IiK2bt162HKT9RpSAAAREyw3VVXFTTfdFPfee2889NBDMX/+/Nf8P48//nhERMyaNeuoBgQAmIgJlZuVK1fG2rVr48c//nH09vbGzp07IyKir68venp6Ytu2bbF27dr44Ac/GKecckps2rQpbrnllrjkkkti4cKFx+UBAAD8sQmVmzvvvDMiDl6o74/dddddcd1110VnZ2f87Gc/i29+85sxMDAQc+bMiRUrVsQXvvCFtIEBAF7NhH8s9WrmzJkT69evP6aBAACOhdeWAgCKotwAAEU56uvcHG8jIyMxMjLS6jHGqdVqqXnNZjMtq7OzMy1r7969aVkREdOmTUvLOnDgQFpWo9FIy1qwYEFaVkTEb3/727Sstra872Ey18Br/Zh7orLXZ5bMtTk0NJSWFRHxu9/9LjUvS+a+Ua/X07IiIqZOnZqWtWvXrrSszK8nmcfGVnHmBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAitLe6gGOpNFoRKPRaPUY43R0dKTmnXnmmWlZzzzzTFpWtoGBgbSsqqrSstra8rr9li1b0rIiIoaHh9OyRkdH07Im6/bPzuvs7EzLytz+9Xo9LSsid5sNDQ2lZZ166qlpWX/4wx/SsiIidu/enZaVuZ5GRkbSstrbc6tB1tfOiTxGZ24AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUdpbPcCRdHd3R3d39zHnjIyMJExz0PDwcFpWRMSWLVtS87IsXLgwNe93v/tdWlatVkvLynw+6/V6WlZEREdHR1rW6OhoWlaj0UjLmsyGhobSsnp6etKyBgYG0rIicveztra875V3796dltXenvtlrqqqtKwpU6akZXV2dqZlZW7/iLxj0ETWpTM3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjtrR7gSA4cOBDt7cc+XlVVCdMcVK/X07IiImq1WlpW5mybNm1Ky4qI6OjoSMsaHh5Oy5o6dWpa1pw5c9KyIiK2bNmSlpW5n2Vqa8v93qrZbKZldXd3p2Xt378/LStb5nrKlHk8Gx0dTcuKyJ0tc9/IfC4z9/+IvLU5ka8lztwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAorS3eoAjufDCC6NWqx1zzjPPPJMwzUGjo6NpWRER3d3daVmZs3V2dqZlRUQMDQ2l5mU5cOBAWtbmzZvTsiIi2tryvu/I3DeqqkrLypb5OCfrPpstcz+brFnZBgcH07J6e3vTstrb876c79mzJy0rIlK+lkdENBqN133fybsHAQAcBeUGACiKcgMAFEW5AQCKotwAAEVRbgCAokyo3Nx5552xcOHCmDZtWkybNi2WLFkS//Ef/zH28cHBwVi5cmWccsopMXXq1FixYkXs2rUrfWgAgCOZULk544wz4h/+4R9i48aN8dhjj8UHPvCB+NCHPhS//e1vIyLilltuifvvvz/uueeeWL9+fTz//PNx9dVXH5fBAQAOp1Yd41W5pk+fHl/72tfiIx/5SJx22mmxdu3a+MhHPhIREU899VS87W1viw0bNsS73/3uw/7/oaGhcRfM6u/vjzlz5kS9Xi/+In49PT1pWdmzZZqss2VekC774naZF+SarBfxq9fraVkRk/dCls1mMy1rMu9nmTL3jeHh4bSsiNzn86STTkrLOhEu4rd3795YtGhR7NmzJ6ZNm/aq9z3q37lpNBrxgx/8IAYGBmLJkiWxcePGGBkZiaVLl47dZ8GCBTF37tzYsGHDEXPWrFkTfX19Y7c5c+Yc7UgAABMvN7/5zW9i6tSp0dXVFZ/+9Kfj3nvvjbe//e2xc+fO6OzsjJNPPnnc/WfMmBE7d+48Yt7q1atjz549Y7cdO3ZM+EEAABwy4fNY5513Xjz++OOxZ8+e+Pd///e49tprY/369Uc9QFdXV3R1dR31/wcA+GMTLjednZ1xzjnnRMTBF7f8r//6r/h//+//xTXXXBPDw8Oxe/fucWdvdu3aFTNnzkwbGADg1RzzdW6azWYMDQ3FhRdeGB0dHbFu3bqxj23evDmeffbZWLJkybF+GgCA12VCZ25Wr14dy5cvj7lz58bevXtj7dq18dBDD8VPfvKT6Ovri+uvvz5WrVoV06dPj2nTpsVNN90US5YsOeJfSgEAZJtQuXnxxRfjr/7qr+KFF16Ivr6+WLhwYfzkJz+Jv/iLv4iIiG984xvR1tYWK1asiKGhoVi2bFl8+9vfPi6DAwAczjFf5yZbf39/9PX1uc7NBE3Wa8lETN7ZXOdm4lznZuJc52biXOdm4lznZjyvLQUAFEW5AQCKMjnPSUbEpk2bore395hzRkZGEqY5aMqUKWlZERH79+9Py8rYVofs27cvLSsi9zRuW1teH280GmlZ3d3daVkRuT9iyTolHJH744LMU/IRuespc5tl/rggcy1FxNhlPTIceo3BDJn7RvaPxTNnGxgYSMvKlP3jyqyvwxPZ/525AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCK0t7qAf5UVVUREbFv376UvJGRkZSciIhGo5GWFRGxf//+1LwsWdv+kGazmZbV1pbXxzOfz8z9LCJidHQ0LevQmsqQuf0z94uI3PVUq9UmZVb2NsvcN/bu3ZuWlfk4BwYG0rIicmc7cOBAWlam9vbcapB1fDz0ten17Le1KnPvTvDcc8/FnDlzWj0GADAJ7dixI84444xXvc+kKzfNZjOef/756O3tfdXvePr7+2POnDmxY8eOmDZt2hs4IRG2f6vZ/q3nOWgt27+1WrH9q6qKvXv3xuzZs1/zLPKk+7FUW1vbazayPzZt2jQ7dgvZ/q1l+7ee56C1bP/WeqO3f19f3+u6n18oBgCKotwAAEV505abrq6uuPXWW6Orq6vVo5yQbP/Wsv1bz3PQWrZ/a0327T/pfqEYAOBYvGnP3AAAHI5yAwAURbkBAIqi3AAARVFuAICivCnLzR133BFnnnlmdHd3x+LFi+NXv/pVq0c6YXz5y1+OWq027rZgwYJWj1Wshx9+OK688sqYPXt21Gq1uO+++8Z9vKqq+NKXvhSzZs2Knp6eWLp0aWzZsqU1wxbotbb/dddd94r1cMUVV7Rm2AKtWbMmLrrooujt7Y3TTz89rrrqqti8efO4+wwODsbKlSvjlFNOialTp8aKFSti165dLZq4LK9n+1966aWvWAOf/vSnWzTx/3nTlZsf/vCHsWrVqrj11lvj17/+dSxatCiWLVsWL774YqtHO2G84x3viBdeeGHs9stf/rLVIxVrYGAgFi1aFHfcccdhP3777bfHt771rfjOd74Tjz76aJx00kmxbNmyGBwcfIMnLdNrbf+IiCuuuGLcevj+97//Bk5YtvXr18fKlSvjkUceiZ/+9KcxMjISl19++bhX+r7lllvi/vvvj3vuuSfWr18fzz//fFx99dUtnLocr2f7R0TccMMN49bA7bff3qKJ/0j1JnPxxRdXK1euHHu70WhUs2fPrtasWdPCqU4ct956a7Vo0aJWj3FCiojq3nvvHXu72WxWM2fOrL72ta+NvW/37t1VV1dX9f3vf78FE5btT7d/VVXVtddeW33oQx9qyTwnohdffLGKiGr9+vVVVR3c3zs6Oqp77rln7D5PPvlkFRHVhg0bWjVmsf50+1dVVf35n/959Td/8zetG+oI3lRnboaHh2Pjxo2xdOnSsfe1tbXF0qVLY8OGDS2c7MSyZcuWmD17dpx11lnxiU98Ip599tlWj3RC2r59e+zcuXPceujr64vFixdbD2+ghx56KE4//fQ477zz4jOf+Uy89NJLrR6pWHv27ImIiOnTp0dExMaNG2NkZGTcGliwYEHMnTvXGjgO/nT7H/K9730vTj311Dj//PNj9erVsX///laMN86ke1XwV/OHP/whGo1GzJgxY9z7Z8yYEU899VSLpjqxLF68OO6+++4477zz4oUXXojbbrst3ve+98UTTzwRvb29rR7vhLJz586IiMOuh0Mf4/i64oor4uqrr4758+fHtm3b4u/+7u9i+fLlsWHDhqjX660eryjNZjNuvvnmeM973hPnn39+RBxcA52dnXHyySePu681kO9w2z8i4uMf/3jMmzcvZs+eHZs2bYrPf/7zsXnz5vjRj37UwmnfZOWG1lu+fPnYvxcuXBiLFy+OefPmxb/927/F9ddf38LJ4I330Y9+dOzfF1xwQSxcuDDOPvvseOihh+Kyyy5r4WTlWblyZTzxxBN+x69FjrT9P/WpT439+4ILLohZs2bFZZddFtu2bYuzzz77jR5zzJvqx1Knnnpq1Ov1V/wm/K5du2LmzJktmurEdvLJJ8e5554bW7dubfUoJ5xD+7z1MHmcddZZceqpp1oPyW688cZ44IEH4he/+EWcccYZY++fOXNmDA8Px+7du8fd3xrIdaTtfziLFy+OiGj5GnhTlZvOzs648MILY926dWPvazabsW7duliyZEkLJztx7du3L7Zt2xazZs1q9SgnnPnz58fMmTPHrYf+/v549NFHrYcWee655+Kll16yHpJUVRU33nhj3HvvvfHzn/885s+fP+7jF154YXR0dIxbA5s3b45nn33WGkjwWtv/cB5//PGIiJavgTfdj6VWrVoV1157bbzrXe+Kiy++OL75zW/GwMBAfPKTn2z1aCeEz372s3HllVfGvHnz4vnnn49bb7016vV6fOxjH2v1aEXat2/fuO+Atm/fHo8//nhMnz495s6dGzfffHN89atfjbe+9a0xf/78+OIXvxizZ8+Oq666qnVDF+TVtv/06dPjtttuixUrVsTMmTNj27Zt8bnPfS7OOeecWLZsWQunLsfKlStj7dq18eMf/zh6e3vHfo+mr68venp6oq+vL66//vpYtWpVTJ8+PaZNmxY33XRTLFmyJN797ne3ePo3v9fa/tu2bYu1a9fGBz/4wTjllFNi06ZNccstt8Qll1wSCxcubO3wrf5zraPxT//0T9XcuXOrzs7O6uKLL64eeeSRVo90wrjmmmuqWbNmVZ2dndWf/dmfVddcc021devWVo9VrF/84hdVRLzidu2111ZVdfDPwb/4xS9WM2bMqLq6uqrLLrus2rx5c2uHLsirbf/9+/dXl19+eXXaaadVHR0d1bx586obbrih2rlzZ6vHLsbhtn1EVHfdddfYfQ4cOFD99V//dfWWt7ylmjJlSvXhD3+4euGFF1o3dEFea/s/++yz1SWXXFJNnz696urqqs4555zqb//2b6s9e/a0dvCqqmpVVVVvZJkCADie3lS/cwMA8FqUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFCU/w//ZE9Dt1OyrAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.figure(figsize=(8,8))\n", "plt.imshow(dlogits.detach(), cmap='gray')" ] } ], "metadata": { "kernelspec": { "display_name": "venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.0" } }, "nbformat": 4, "nbformat_minor": 2 }