{ "cells": [ { "cell_type": "markdown", "id": "0a62f1aa", "metadata": {}, "source": [ "# 7. Initialisations et normalisations" ] }, { "cell_type": "markdown", "id": "9df07533", "metadata": {}, "source": [ "Suite de l'implémentation d'un réseau de neurones de type _Feed-Forward Network_ inspiré du papier de Bengio et al. de 2003 [_A Neural Probabilistic Language Model_](https://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf).\n", "\n", "Étude des initialisations, des fonctions d'activation et de la normalisation." ] }, { "cell_type": "markdown", "id": "633e6dbb", "metadata": {}, "source": [ "\n", "Bibliographie indicative:\n", "\n", "- \"Kaiming init\": \n", "- BatchNorm: \n", "- Illustration de certains problèmes liés à BatchNorm: " ] }, { "cell_type": "markdown", "id": "0f112f5d", "metadata": {}, "source": [ "## Restructuration du code précédent" ] }, { "cell_type": "code", "execution_count": 4, "id": "6d923dcc-b56d-4f77-9217-29ebcb477730", "metadata": { "editable": true, "slideshow": { "slide_type": "" }, "tags": [] }, "outputs": [], "source": [ "import random\n", "\n", "import torch\n", "import torch.nn.functional as F\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "id": "5e6e8395", "metadata": {}, "source": [ "### Words" ] }, { "cell_type": "code", "execution_count": null, "id": "9c5ef2c7-c01c-4d6f-aa60-e78407a83e64", "metadata": {}, "outputs": [], "source": [ "class Words(object):\n", " \"\"\"Représente une liste de mots, ainsi que la liste ordonnée des caractères les composants.\"\"\"\n", "\n", " EOS = '.'\n", "\n", " def __init__(self, filename):\n", " self.filename = filename\n", " self.words = open(self.filename, 'r').read().splitlines()\n", " self.nb_words = len(self.words)\n", " self.chars = sorted(list(set(''.join(self.words))))\n", " self.nb_chars = len(self.chars) + 1 # On ajoute 1 pour EOS\n", " self.ctoi = {c:i+1 for i,c in enumerate(self.chars)}\n", " self.ctoi[self.EOS] = 0\n", " self.itoc = {i:s for s,i in self.ctoi.items()}\n", "\n", " def __repr__(self):\n", " l = []\n", " l.append(\"')\n", " return '\\n'.join(l)" ] }, { "cell_type": "code", "execution_count": 6, "id": "e415605c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "words = Words('civil_mots.txt')\n", "print(words)" ] }, { "cell_type": "markdown", "id": "a3f53580", "metadata": {}, "source": [ "### Datasets" ] }, { "cell_type": "code", "execution_count": 7, "id": "284e8c00", "metadata": {}, "outputs": [], "source": [ "\n", "class Datasets:\n", " \"\"\"Construits les jeu de données d'entraînement, de test et de validation.\n", " \n", " Prend en paramètres une liste de mots et la taille du contexte pour la prédiction.\n", " \"\"\"\n", "\n", " def _build_dataset(self, lwords:list, context_size:int):\n", " X, Y = [], []\n", " for w in lwords:\n", " context = [0] * context_size\n", " for ch in w + self.words.EOS:\n", " ix = self.words.ctoi[ch]\n", " X.append(context)\n", " Y.append(ix)\n", " context = context[1:] + [ix] # crop and append\n", " X = torch.tensor(X)\n", " Y = torch.tensor(Y)\n", " return X, Y\n", " \n", " def __init__(self, words:Words, context_size:int, seed:int=42):\n", " # 80%, 10%, 10%\n", " self.shuffled_words = words.words.copy()\n", " random.shuffle(self.shuffled_words)\n", " self.n1 = int(0.8*len(self.shuffled_words))\n", " self.n2 = int(0.9*len(self.shuffled_words))\n", " self.words = words\n", " self.Xtr, self.Ytr = self._build_dataset(self.shuffled_words[:self.n1], context_size)\n", " self.Xdev, self.Ydev = self._build_dataset(self.shuffled_words[self.n1:self.n2], context_size)\n", " self.Xte, self.Yte = self._build_dataset(self.shuffled_words[self.n2:], context_size)\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "ddee5bdf", "metadata": {}, "outputs": [], "source": [ "context_size = 3\n", "datasets = Datasets(words, context_size)" ] }, { "cell_type": "markdown", "id": "422eb7b0", "metadata": {}, "source": [ "### Réseau de neurones à propagation avant (Feed-Forward Network)" ] }, { "cell_type": "code", "execution_count": null, "id": "e51241e7", "metadata": {}, "outputs": [], "source": [ "class BengioFFN:\n", " \n", " def __init__(self, e_dims, n_hidden, context_size, nb_chars, g):\n", " self.g = g\n", " self.nb_chars = nb_chars\n", " self.e_dims = e_dims\n", " self.n_hidden = n_hidden\n", " self.context_size = context_size\n", " self.create_network()\n", "\n", " def layers(self):\n", " self.C = torch.randn((self.nb_chars, self.e_dims), generator=self.g)\n", " self.W1 = torch.randn((self.context_size * self.e_dims, self.n_hidden), generator=self.g)\n", " self.b1 = torch.randn(self.n_hidden, generator=self.g)\n", " self.W2 = torch.randn((self.n_hidden, self.nb_chars), generator=self.g)\n", " self.b2 = torch.randn(self.nb_chars, generator=self.g)\n", "\n", " def create_network(self):\n", " self.layers()\n", " self.loss = None\n", " self.steps = 0\n", " self.parameters = [self.C, self.W1, self.b1, self.W2, self.b2]\n", " self.nb_parameters = sum(p.nelement() for p in self.parameters) # number of parameters in total\n", " for p in self.parameters:\n", " p.requires_grad = True\n", "\n", " def forward(self, X, Y):\n", " self.emb = self.C[X] # Embed characters into vectors\n", " self.embcat = self.emb.view(self.emb.shape[0], -1) # Concatenate the vectors\n", " self.hpreact = self.embcat @ self.W1 + self.b1 # hidden layer pre-activation\n", " self.h = torch.tanh(self.hpreact) # hidden layer\n", " self.logits = self.h @ self.W2 + self.b2 # output layer\n", " self.loss = F.cross_entropy(self.logits, Y) # loss function\n", "\n", " def backward(self):\n", " for p in self.parameters:\n", " p.grad = None\n", " self.loss.backward()\n", "\n", " def train(self, datasets: Datasets, max_steps, mini_batch_size):\n", " lossi = []\n", " for i in range(max_steps):\n", " # minibatch construct\n", " ix = torch.randint(0, datasets.Xtr.shape[0], (mini_batch_size,), generator=self.g)\n", " Xb, Yb = datasets.Xtr[ix], datasets.Ytr[ix]\n", " \n", " # forward pass\n", " self.forward(Xb, Yb)\n", " \n", " # backward pass\n", " self.backward()\n", " \n", " # update\n", " lr = 0.2 if i < 100000 else 0.02 # step learning rate decay\n", " self.update_grad(lr)\n", " \n", " # track stats\n", " if i % 10000 == 0:\n", " print(f\"{i:7d}/{max_steps:7d}: {self.loss.item():.4f}\")\n", " lossi.append(self.loss.log10().item())\n", " self.steps += max_steps\n", " return lossi\n", "\n", " def update_grad(self, lr):\n", " for p in self.parameters:\n", " p.data += -lr * p.grad\n", "\n", " @torch.no_grad() # this decorator disables gradient tracking\n", " def compute_loss(self, X, Y):\n", " emb = self.C[X] # Embed characters into vectors\n", " embcat = emb.view(emb.shape[0], -1) # Concatenate the vectors\n", " hpreact = embcat @ self.W1 + self.b1 # hidden layer pre-activation\n", " h = torch.tanh(hpreact) # hidden layer\n", " logits = h @ self.W2 + self.b2 # output layer\n", " loss = F.cross_entropy(logits, Y) # loss function\n", " return loss\n", "\n", " @torch.no_grad() # this decorator disables gradient tracking\n", " def training_loss(self, datasets:Datasets):\n", " loss = self.compute_loss(datasets.Xtr, datasets.Ytr)\n", " return loss.item()\n", "\n", " @torch.no_grad() # this decorator disables gradient tracking\n", " def test_loss(self, datasets:Datasets):\n", " loss = self.compute_loss(datasets.Xte, datasets.Yte)\n", " return loss.item()\n", "\n", " @torch.no_grad() # this decorator disables gradient tracking\n", " def dev_loss(self, datasets:Datasets):\n", " loss = self.compute_loss(datasets.Xdev, datasets.Xdev)\n", " return loss.item()\n", "\n", " @torch.no_grad()\n", " def generate_word(self, itoc, g):\n", " out = []\n", " context = [0] * self.context_size\n", " while True:\n", " emb = self.C[torch.tensor([context])]\n", " h = torch.tanh(emb.view(1, -1) @ self.W1 + self.b1)\n", " logits = h @ self.W2 + self.b2\n", " probs = F.softmax(logits, dim=1)\n", " # Sample from the probability distribution\n", " ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n", " # Shift the context window\n", " context = context[1:] + [ix]\n", " # Store the generated character\n", " if ix != 0:\n", " out.append(ix)\n", " else:\n", " # Stop when encounting '.'\n", " break\n", " return ''.join(itoc[i] for i in out)\n", "\n", " def __repr__(self):\n", " l = []\n", " l.append(\"')\n", " return '\\n'.join(l)" ] }, { "cell_type": "code", "execution_count": 10, "id": "5907dcf9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "e_dims = 10 # Dimensions des embeddings\n", "n_hidden = 200\n", "seed = 2147483647\n", "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "print(nn)" ] }, { "cell_type": "markdown", "id": "da84827a", "metadata": {}, "source": [ "### Apprentissage" ] }, { "cell_type": "code", "execution_count": 11, "id": "128f9611", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 200000: 30.4901\n", " 10000/ 200000: 2.1505\n", " 20000/ 200000: 1.5062\n", " 30000/ 200000: 1.9053\n", " 40000/ 200000: 1.8812\n", " 50000/ 200000: 2.8604\n", " 60000/ 200000: 2.1297\n", " 70000/ 200000: 1.6733\n", " 80000/ 200000: 2.1570\n", " 90000/ 200000: 1.7283\n", " 100000/ 200000: 1.9989\n", " 110000/ 200000: 1.5019\n", " 120000/ 200000: 1.2979\n", " 130000/ 200000: 1.5895\n", " 140000/ 200000: 1.6045\n", " 150000/ 200000: 1.8367\n", " 160000/ 200000: 1.7854\n", " 170000/ 200000: 1.1291\n", " 180000/ 200000: 1.7075\n", " 190000/ 200000: 1.4321\n" ] } ], "source": [ "max_steps = 200000\n", "mini_batch_size = 32\n", "lossi = nn.train(datasets, max_steps, mini_batch_size) " ] }, { "cell_type": "code", "execution_count": 12, "id": "bc6516fd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "print(nn)" ] }, { "cell_type": "code", "execution_count": 13, "id": "408cb951", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGdCAYAAAAIbpn/AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAATqxJREFUeJzt3Qd8VFX2wPFDTUBIkN6CNGmCVEUUUAQBYXFtK4sFllWwsRasWMC2wlqw7B/EhtgQxEUsIEiVDtKkCCi9hipJCBBK5v85F2eYmcxM3iQzmfeS3/fzeZ9MeTPz3rzJvDP3nntuIZfL5RIAAAAHKhzrDQAAAMgpAhkAAOBYBDIAAMCxCGQAAIBjEcgAAADHIpABAACORSADAAAci0AGAAA4VlHJZzIzM2XPnj1SunRpKVSoUKw3BwAAWKD1edPS0qRq1apSuHDhghvIaBCTlJQU680AAAA5sHPnTqlevXrBDWS0Jcb9RiQkJMR6cwAAgAWpqammIcJ9Hi+wgYy7O0mDGAIZAACcJdy0EJJ9AQCAYxHIAAAAxyKQAQAAjkUgAwAAHItABgAAOBaBDAAAcCwCGQAA4FgEMgAAwLEIZAAAgGMRyAAAAMcikAEAAI5FIAMAAByLQMaizEyXfLRgq6zZlRLdIwIAAAru7NfRMnHlbnn+u1/N5W3Dusd6cwAAAC0y1m3Ym8oHBgAAm6FryaJGVROieyQAAEDYCGQsKl8qzvxtWIWABgAAuyCQAQAAjkUgAwAAHItABgAAOBaBDAAAcCwCGQAA4FgEMgAAwLEIZAAAgGMRyITJ5XJF50gAAICwEchYVKhQ+G8uAACILgIZAADgWFENZObOnSs9evSQqlWrSqFChWTSpEmWH7tgwQIpWrSoNGvWLJqbCAAAHCyqgUx6ero0bdpURowYEdbjjhw5Ir1795aOHTtGbdsAAIDzFY3mk1977bVmCdc999wjt956qxQpUiSsVhwAAFCw2C5H5qOPPpItW7bIkCFDYr0pAACgILfIhOv333+XJ598UubNm2fyY6zIyMgwi1tqamoUtxAAANiJbVpkzpw5Y7qTnn/+ealXr57lxw0dOlQSExM9S1JSUlS3EwAA2IdtApm0tDRZtmyZDBgwwLTG6PLCCy/IL7/8Yi7PmjUr4OMGDRokKSkpnmXnzp15vu0AAKCAdy0lJCTImjVrfG4bOXKkCWC++uorqVWrVsDHxcXFmQUAABQ8UQ1kjh49Kps2bfJc37p1q6xatUrKli0rNWrUMK0pu3fvlk8++UQKFy4sjRs39nl8xYoVJT4+PsvtseCemWBDclqsNwUAAORFIKNdRR06dPBcHzhwoPnbp08fGTNmjOzdu1d27NghTvDNqj2x3gQAAOCnkCufzYKoo5Y06VfzZbS7KlJuHLlAVuw4Yi5vG9Y9Ys8LAAAkx+dv2yT7AgAAhItAxiKdKwoAANgLgYxFhDEAANgPgQwAAHAsAhmL6FkCAMB+CGQsKkTnEgAAtkMgAwAAHItAxiqyfQEAsB0CGYuIYwAAsB8CGYtI9gUAwH4IZCwi2RcAAPshkLGIFhkAAOyHQMbqG0UkAwCA7RDIWEQcAwCA/RDIAAAAxyKQsYjZrwEAsB8CGYuoIwMAgP0QyAAAAMcikLH6RtEkAwCA7RDIWESODAAA9kMgYxENMgAA2A+BjEXUkQEAwH4IZCyjTQYAALshkLGIFhkAAOyHQMYi2mMAALAfAhmLaJEBAMB+CGQsKkSbDAAAtkMgYxEtMgAA2A+BjEUEMgAA2A+BjEV0LQEAYD8EMlYxbAkAgIIVyMydO1d69OghVatWNXMVTZo0KeT6EydOlGuuuUYqVKggCQkJ0qZNG5k2bZrYAXEMAAAFLJBJT0+Xpk2byogRIywHPhrITJkyRZYvXy4dOnQwgdDKlSsl1pg0EgAA+ykazSe/9tprzWLVm2++6XP95Zdflm+++Ua+++47ad68ucQSLTIAABSwQCa3MjMzJS0tTcqWLRt0nYyMDLO4paamRmVbGLUEAID92DrZ97XXXpOjR4/KLbfcEnSdoUOHSmJiomdJSkqKyrYUJpIBAMB2bBvIjB07Vp5//nn58ssvpWLFikHXGzRokKSkpHiWnTt35ul2AgCA2LFl19K4cePkrrvukgkTJkinTp1CrhsXF2eWaKNBBgAA+7Fdi8wXX3whffv2NX+7d+8udkFBPAAACliLjOa3bNq0yXN969atsmrVKpO8W6NGDdMttHv3bvnkk0883Ul9+vSRt956S1q3bi3Jycnm9hIlSpj8FwAAgDxrkVm2bJkZNu0eOj1w4EBzefDgweb63r17ZceOHZ7133vvPTl9+rTcf//9UqVKFc/y4IMPSqwVZvw1AAAFq0XmqquuEpfLFfT+MWPG+FyfM2eO2BU5MgAA2I/tcmQAAACsIpCx+kbRJAMAgO0QyFhEHAMAgP0QyFhGti8AAHZDIAMAAByLQMbqG0WDDAAAtkMgYxE5MgAA2A+BjEUH005G90gAAICwEchY9Pv+tPDfXQAAEFUEMhYVom8JAADbIZCxiFxfAADsh0DGIhpkAACwHwIZiwrRJgMAgO0QyFhEiwwAAPZDIAMAAByLQAYAADgWgYxFDL8GAMB+CGQsYvg1AAD2QyCTg2TfX/ekRulwAACAcBDI5CCQOZpxOqw3GQAARAeBjEXUkQEAwH4IZHLQIkNNGQAA7IFAxiKSfQEAsB8CGau8mmEIagAAsAcCGatvFNELAAC2QyBjkXccQ44MAAD2QCCTIzTPAABgBwQyFjFFAQAA9kMgYxFdSwAA2A+BjEXkxQAAUMACmblz50qPHj2katWqpmtm0qRJ2T5mzpw50qJFC4mLi5O6devKmDFjxG6VfcmQAQCgAAQy6enp0rRpUxkxYoSl9bdu3Srdu3eXDh06yKpVq+Shhx6Su+66S6ZNmxbNzQQAAA5VNJpPfu2115rFqlGjRkmtWrXk9ddfN9cbNmwo8+fPlzfeeEO6dOki9pmigDYZAADswFY5MosWLZJOnTr53KYBjN4eTEZGhqSmpvos0UDsAgCA/dgqkElOTpZKlSr53KbXNTg5fvx4wMcMHTpUEhMTPUtSUlJUti2uaBHPZdpjAACwB1sFMjkxaNAgSUlJ8Sw7d+6MyutcXqdcVJ4XAADYNEcmXJUrV5Z9+/b53KbXExISpESJEgEfo6ObdIm2wt6TRtIkAwCALdiqRaZNmzYyc+ZMn9umT59ubo+1knHnupYyTmfGdFsAAEAeBDJHjx41w6h1cQ+v1ss7duzwdAv17t3bs/4999wjW7Zskccff1w2bNggI0eOlC+//FIefvhhibW2dct7Ln84b2tMtwUAAORBILNs2TJp3ry5WdTAgQPN5cGDB5vre/fu9QQ1SodeT5482bTCaP0ZHYb9wQcfxHzotX/X0tR1yTHdFgAAkAc5MldddZW4XK6g9weq2quPWblypdhNiN0AAAAxYqscGTtzCZEMAAB2QyADAAAci0DGIrqWAACwHwIZAADgWAQyFpEhAwCA/RDIAAAAxyKQsSjUMHIAABAbBDIAAMCxCGQsoj0GAAD7IZCxiJ4lAADsh0AGAAA4FoEMAABwLAIZAADgWAQylpHuCwCA3RDIAAAAxyKQsYhRSwAA2A+BjEV0LAEAYD8EMgAAwLEIZCyiawkAAPshkLHIRecSAAC2QyADAAAci0DGIrqWAACwHwIZiwhkAACwHwIZAADgWAQyFpHsCwCA/RDIWETXEgAA9kMgAwAAHItABgAAOBaBDAAAcCwCGQAA4FhRD2RGjBghNWvWlPj4eGndurUsXbo05Ppvvvmm1K9fX0qUKCFJSUny8MMPy4kTJyTWSPYFAKCABTLjx4+XgQMHypAhQ2TFihXStGlT6dKli+zfvz/g+mPHjpUnn3zSrL9+/Xr58MMPzXM89dRT0dxMAADgUFENZIYPHy79+vWTvn37SqNGjWTUqFFSsmRJGT16dMD1Fy5cKFdccYXceuutphWnc+fO0qtXr2xbcfICdWQAAChAgczJkydl+fLl0qlTp3MvVriwub5o0aKAj7n88svNY9yBy5YtW2TKlCnSrVs3iTW6lgAAsJ+i0XrigwcPypkzZ6RSpUo+t+v1DRs2BHyMtsTo49q2bSsul0tOnz4t99xzT8iupYyMDLO4paamSl5YsytFmlRPzJPXAgAADhi1NGfOHHn55Zdl5MiRJqdm4sSJMnnyZHnxxReDPmbo0KGSmJjoWTRBOBpcftef/25dVF4HAADYoEWmfPnyUqRIEdm3b5/P7Xq9cuXKAR/z7LPPyh133CF33XWXud6kSRNJT0+X/v37y9NPP226pvwNGjTIJBR7t8hEI5jRFiJvy7b/EfHXAAAANmmRKV68uLRs2VJmzpzpuS0zM9Ncb9OmTcDHHDt2LEuwosFQoEDCLS4uThISEnwWAABQMEStRUZpS0mfPn2kVatWcumll5oaMdrCoqOYVO/evaVatWqme0j16NHDjHRq3ry5qTmzadMm00qjt7sDmlgJHEYBAIB8G8j07NlTDhw4IIMHD5bk5GRp1qyZTJ061ZMAvGPHDp8WmGeeeUYKFSpk/u7evVsqVKhggph///vfEmuMWgIAwH4KuYL12TiU5sho0m9KSkpEu5mWb/9Dbnpnoc9t24Z1j9jzAwBQkKXm8Pxtq1FLdlazXMlYbwIAAPBDIGNRuVJxWW47ceqM1YcDAIAoIJDJhTW7UyJ3JAAAQNgIZAAAgGMRyAAAAMcikAEAAI5FIAMAAByLQAYAADgWgUwu5K9SggAAOA+BTC78sHZv5I4EAAAIG4FMLny0YJtkZtIsAwBArBDI5FIm/UsAAMQMgUwuLd16ODJHAgAAhI1AJpf2ppzI7VMAAIAcIpDJpWnrknP7FAAAIIcIZHKJVF8AAGKHQAYAADgWgUwuFRKRtbtT5LPF28XFCCYAAPJU0bx9ufznx1/3mUWVji8qf21WLdabBABAgUGLTASt35sWyacDAADZIJABAACORSATQS7GMAEAkKcIZAAAgGMRyETQ5v3pkXw6AACQDQKZCJqx/uzoJQAAkDcIZAAAgGMRyAAAAMcikImwTxdvj/RTAgCAIAhkIuzZSWsj/ZQAACAIAhkAAOBYBDIAAMCxoh7IjBgxQmrWrCnx8fHSunVrWbp0acj1jxw5Ivfff79UqVJF4uLipF69ejJlypRobyYAAHCgqAYy48ePl4EDB8qQIUNkxYoV0rRpU+nSpYvs378/4PonT56Ua665RrZt2yZfffWVbNy4Ud5//32pVs1ZM0q7XC55b+5mmff7gVhvCgAA+VrRaD758OHDpV+/ftK3b19zfdSoUTJ58mQZPXq0PPnkk1nW19sPHz4sCxculGLFipnbtDXHaeb8dkBenrLBXN42rHusNwcAgHwrai0y2rqyfPly6dSp07kXK1zYXF+0aFHAx3z77bfSpk0b07VUqVIlady4sbz88sty5syZoK+TkZEhqampPkus7Tp8LNabAABAgRC1QObgwYMmANGAxJteT05ODviYLVu2mC4lfZzmxTz77LPy+uuvy0svvRT0dYYOHSqJiYmeJSkpKeL7AgAA7MlWo5YyMzOlYsWK8t5770nLli2lZ8+e8vTTT5suqWAGDRokKSkpnmXnzp0Sa65YbwAAAAVE1HJkypcvL0WKFJF9+3wnUtTrlStXDvgYHamkuTH6OLeGDRuaFhztqipevHiWx+jIJl3sZPA36zyX9xw5LlXLlBA7Onk6U37blyYXVU2QQoUKxXpzAACwT4uMBh3aqjJz5kyfFhe9rnkwgVxxxRWyadMms57bb7/9ZgKcQEGME9h5RuwHvlgpf/nvfHlv7pZYbwoAAPbrWtKh1zp8+uOPP5b169fLvffeK+np6Z5RTL179zZdQ256v45aevDBB00AoyOcNNlXk38ReVPXnc1VIpABADhVVIdfa47LgQMHZPDgwaZ7qFmzZjJ16lRPAvCOHTvMSCY3TdSdNm2aPPzww3LxxReb+jEa1DzxxBPR3EwAAOBQUQ1k1IABA8wSyJw5c7Lcpt1Oixcvlvzi931HZeWOP6R5jfNjvSkAAOQ7thq1lB99uni73DByoUn63ZtyXG59f7H8+GeXTrhST5ySjxduk/1pJyK+nQAAOBGBTB65fNgseXbSWlm4+ZD0/3R5jp7j8QmrZci36+SOD0LPVwUAQEFBIJOHZqwPPMeUVdP/HAG1cV+aRBJ1bwAATkUgA8D2MjNdMvDLVfLBPEoFAPBFIBPDYnRuB49mxGozAEf46bcDMnHFbnlp8vpYbwoAmyGQiZHxP++QtbtTpNVLM8wycs6mWG2KuFx0LsHe0jJOx3oTABTU4dcITH9ZZni1yrwydaPcd1XdXL1dx06elpLFOaQAgIKDFpkY8Q5i3H5Ys1fGLd0hszbskyuGzZIlWw5Zfr5Fmw9Jo8HT5KXvf43wlgIAYF/8fLeRez9f4XP97+8vlq1Du8vwHzfK27NCdz0N++Fs7sAH87fKo13qS3yxIvLe3M1SsXS8XN+8WsjH0rEEu6P7E0AwtMjYmDt1Jbsgxt/IOZvl931p8vKUDfLQ+FUSS5sPHJXjJ8+Ik2ky9pFjJ2O9GQCAAAhkHG7dnhQZ/M1aOZR+7kSrUyL8ceyUT0XgWFi4+aB0fP0n6f72PHHKr/6dh4/53KZBmCZjN3thOq0CAGBDBDI2d/pM1lwaNXTKepm0crd0f3u+fLJou+z643jQ57j4uR9l8uq9Zv1TAZ7vyLFTUQl2vl21x/zdcjBdnODF79dLu1dm+9Qq2ZNy7n3NpA8OAGyHQMbmGj83LeDt787dErLbyL9w2P1jV5j1h0//LeD6YxZsk4Ju9IKt5u+wHzbEelMAABYRyNjciVOBW2Sy8+OvZ6cz8PfOnM3m7y87jwQt0BfMmUyXPPLlLzJ2yQ7JS1pvp+9HS2VDcmqevi4AwP4IZAqov45YEPZjfli7V/63Ypc89fUaE1Qs3HRQJq7YlSWvJJI0ePrLf+fL7I0HpOe7i8N6rI7a0sArUiNevl+9x2yP24G0DMk4nbNEZt0mK8FjfvdH+klTbiBYFyoAZIfh1/nQvN8P5uhx+1NPSEKJYmbodiCpx89VV+36pm8C77Zh3SUabh610HM55fgp2X4oXS4od56lx+qoLXVji2pyRd3yIde1EpA8OG6VHDt5RnpdWsMEb5pPU61MCVnw5NUSrn99sVKmrNkriwZ1lEoJ8RJtGigcO3VGEuKLid0C6h2Hj8nT3RpKv/a1g65H8WkAwdAiUwBpS4q/jxduk0tfnilt/zM7y33pFsvD68R+M9fvM4nFetkqDSJ2/RG4VWflDt8usKtf/0nCpcFHICdOnZHpv+4z+9fqxRmWCw+qWRvOzmS++0jwJGvv1peNyWk+rTnf63vkEvny552SF7q9Pc8kfWuwahcaXGkQo6as3RvrzQHgULTIFEC3frAk6Fw2/hNYaoAz5Nt18spNF8v0X5NDPu+bM37z1Lx59eaLQ66rycjaNZNYspiZnkFNvO9yaVHj/JCP8w4GzHafOCV7jpyQ+pVL+9xupbtryDfrZPyynVKzXEmfuXwKFZKIemP62ffl9stqyEvXNwn78TqcvliRwtK4WmKOt+G3fUfN39kb90vPS2qE/XgNxgpF+I35YW3ozxMAWEGLDILam3LcBDHq8f+tNnkqwWgBPu/CfQv8Wn3emvG7yYfwnmtKR165gxj1cg5mNu7w2k/S5c25snTrYZ/b35zxu+dyv0+WyWGv13bTIEZtOxQ86An31B1oeLv7fflscdYk6exiAx0Wf8PIhSZPyD+I8w/c3p75e1QK9+mx1da6Txdt80nA1tao5dv/MLfnJA/JaksfAIRCiwyC8s+DCeWaN+b6XPc/rb0x4zdZsztFPujTKuhzLNv+hzw24RczD9Vbf28WsgVAW3/W7k71tCD9uC5ZLq1V1lz+cP5Wk5Ts7amJa2TUHS0lt0IFHsu3H5ab3lkkD3eqJ10aV5KjJ05Lq5pntymYiSt2S7lScSbvJhDv4O/D+VukZrnzpPNFlT23aQBx96fLPaPUNMB4r3fw99g9uWiJYkUstbBsO5guPd9bbALBZ79ZJ3e0qWmCJg2svFUtU0I6NqwkOZVdHORiIg0AQdAigyw27U/zJNfm1NQA3QaLLUyCOWH5Lvn2lz2ybk9q0F/5eru2uMxYn3WIuebavBhg4sx1e1Pkia9Wyz/H/BxW64H/mukZZ/NtDvl1wanB36zzBG0aBN48apHsyyYnRYsFDpq4xrR+WUle7v/pcpmz8Wx+jlq85bDPUPslfi1TgYr+6eSiWlfIXbl41c4jAd8TzXO66rU5Pq1ZXyzdIev3Zh0Gv+VA7ose6jYEOzaverXcAYA3Apkw3NSiuhQEnYbPDdmNYYW2qozzS2TVk5S2GPR8d1G2jx/45Spp/2rWxGP15P/WZLlNt/bat+YFTFb27krSJN0BX6yU+SFGdp064zKzj4+ck3WOKw2eNEjzn//qtWkbTfDlL1TFZW9ths4yU018tni7NHx2qtk+TYYNFPz946OfPTOj+3fPZBd8Hv1z/SlrzgaaPd9bJNePWCATlvm2YKnMAEGFBl2hWkx0m3/blya93ltsAtJQvJ9dgylt+bntgyVZghntrtuTci4g/MSriwsA6FoKQ+0K1ob95gftXwkeEORGj/+bb2korTs5NRB3bos3PYEGailw23n4XECho6p0CUXzPzR/p4tXN47b8Om+rQOal/J/s8Ob2DMQnWrC7fYPl8iADnWDPu/yHX9I69rlAt6n21OmZHEzwmrs0h0ypEejoK+5eleK+fvV8l1yyyVJlrYzUJeUtha5h7u7LdpySK5rWtV3u7f/IT9t3C8Drr4wy+fAneekw/w1CVw9/tUvppXOv+Wrd5ualrYVQP5HIIOArAwrDld6FGfB/iJKw5h1iLa/fam+3UrhFOrTiTS3HbRWQPDjEC0P7iAgUJpL5zfmytKnO0mv989ul5Wh8MmpJ2TubwekTZ1yZoSUjm76IkgFZx1in1M3vXO2LpAOu57051xc/g6mny00WDEhXr4M0FIEAN4IZJAvRKtKrk7K6c+/+2bjvrM5RaFO3G63vp916HswaSeCj+rRLjodln3nx8uy3Lc/zTfQmrwm+xotGlj0Hr3UtAI92qW+9P3o56Dr+ncZ5kSwIEbpjOlq/Qtdc/06API/cmQAB9IaLDosO5yWJCu0225PFFrjcmJ/mn2K9wGwLwIZIB/yrs8TTKCEbi1SePmwWRHdFs2/+c/UDWHXmsltwjmAgoGupTBEuuIrEC3T1mVfNbfOU1OifgC0a00TfFXT6uFVJg40jB4A/NEiA+RD0UjWzgl3EKPu+exs7RqrQlWSBpxKWxoHjF0h7/60Odabkm8QyABwHK3o3O2teWYuMMBJZm/YbyaNHfqDb7kC2DyQGTFihNSsWVPi4+OldevWsnTpUkuPGzdunKlZcf3114sdlI6jJw6wA52i4te9qZ65wPLydf/1xcqwZncHvB3LYSI+YhjIjB8/XgYOHChDhgyRFStWSNOmTaVLly6yf/+5MuuBbNu2TR599FFp166d2MXfWlkrGAYgutxTReQ1nRrju1/2yNJtoaeCiBWt+Ky1ioCCJOqBzPDhw6Vfv37St29fadSokYwaNUpKliwpo0ePDvqYM2fOyG233SbPP/+81K5dW+wivliRWG8CABH56bfY5s/oFBx2o0UEdZoHrVWUdsLaPGk6kuz9uVtMMUSEnhB2utecZihAgczJkydl+fLl0qlTp3MvWLiwub5oUfD5dl544QWpWLGi3Hnnndm+RkZGhqSmpvosAPI3nezSDkbM3iSv/7jRdkUhQxVT9LZg0yH595T1phgigtNZ7ft9skx2HLJWlRv5KJA5ePCgaV2pVKmSz+16PTk58PDQ+fPny4cffijvv/++pdcYOnSoJCYmepakJLp/gPwu0ISWeU2LDr46baP8d9ambGc5tyudLR7W7bEwSz0K+KiltLQ0ueOOO0wQU758eUuPGTRokKSkpHiWnTujM+cOAPuIfRhzbr4rlXEq05Hvjx3ex4KGcmSRF9VhOBqMFClSRPbt8+1b1OuVK2edVXjz5s0mybdHjx6e2zIzz35BFC1aVDZu3Ch16tTxeUxcXJxZABQc3lWCdSh2+VLZfwecOpNpJsTMrwLNSh5Jh45myPUjF8gNzarJwM71xcm0G+7bX/bIFXXLSZXEErHeHORSVP+rixcvLi1btpSZM2f6BCZ6vU2bNlnWb9CggaxZs0ZWrVrlWa677jrp0KGDuUy3EQB16sy5QKbVSzNMMmZ2lY4vfPoHGbc08IzeueUqAG0b787dIjsPH5e3Z20Sp9DuP/9JXt25TY9O+MXMFB+omOS2g+l5tIWIhKj/PNGh19pV9PHHH8v69evl3nvvlfT0dDOKSfXu3dt0DymtM9O4cWOfpUyZMlK6dGlzWQMjAPD3yaLtId+Uuz9dbv4+OXFNlhNdTvNb8sOUJYFSjYLNieU991WoebB09NSsDfskPcNawnFuPP/dOnlo3Mqg29z65ZnS9PkfJeWYbzAz589RWv5J0Vof6Iphs+Sq1+bI0TzY/ux8MG+LPPDFyqjMO7brj2M+CeJOFvVApmfPnvLaa6/J4MGDpVmzZqZlZerUqZ4E4B07dsjevXujvRkA4KEnvt/2pUn7V2abk93WMH+BT/81WT5asM3S62htl1SLw6EjJdwJOt2+WbVbLvn3DJ+pJQJp8eJ0+SP9ZMD7/j15vfxzzDK59/PwpqSwQrsR7/p4mVw/YoFsSE41x2DSqj2yJcjxc7fGrNmdEnYS+f4AAW6ot1W3TWsMhRMcaOAU6li9NHm96QKbuT6yQ78Xbzkkbf8zW/72bvDRw06SJ6VqBwwYYJZA5syZE/KxY8aMidJWAcgv/vD7xe0t0Ill+PTfzGgjtzs+XCKl4orK272aS1zRwnJBufNCvt5ni327qIKdi/63YrfpwqhZrqTMeayDz30a3JQqXlQKF45M004knuXBcavM3/6fLJPlz14TdD0NEP63Ypfc1S5rna+xS86+N5GuTfPe3M3y8pRzZf27vjnPc9kOlZY1uNr1x3F54Oq6IXOIvFvytOWnfuXS8n7vViGf+3iEqwF/uezsoJhfdh4xgVS086uiLf9mvgEoMPSkqV/O/jkwCzcdlHrP/OBzmzbVewcxSk9AG5LTTM7Ela/Okf1p4Xc3uX9Z60n1vs+Xy1t/VgFW2/zqj2w5cFQufu5HafHSdLNuuC1C4dCunnfmbJZl2w7LsZOnLeX2nLHYoqMJ1HeO+VnqPjUl6hWFvYOY3HTVdH97nqQcC9ya5M3KO7B61xH564gFsnTrYfMZUtPWWW892XH4WMQK7S3f/odc/foc89lSXyzdIbd/sCRLF9nP2w57PpfqkQm/eLpZn/zfapNPFojeP2XN3oA5R7FGIAMgX3j8q9UmB0bzIQ6nnzQVa2/9YEmW9bSpPju/JR81wdHTX68xX+DZ+fHXZGn2wnRzgli4+ZBMWZMsb8z4Lej67/60xfw9cuyUWbfnu4tMvseKHb5dOpobcdHgqdL1zbnZdhd556ToqntTjpvHfDBvq/xn6ga5edQiaTR4mnnO2Rv3m26kp79e63nMze8s9FzW7dKAb+fhswGYPs+H87dmec3vV++RmRv2y+lMl6korPSyFfpe6XbpPg/+Zq088uUvprsoJ7JrUHDfr1016/ak+gSW1wz/yVMJ2btlYn9qRrZ5Mj3fXWxaNW7x66LRz5i+v4GMmB3+rNcHj2YfeN30zkLZciBdrn79J3N90MQ1Mn/TQfN/4O1voxb5JMtPXHF2Oz9dtF3G/bzTk0/mbjV8ePwqmbNxv+kyvO/zFSZwtRtmQQSQr9w/doX5As8td7XbiqXj5e4ra2cJMgK1FuiEkgnx575WA53SNTgY/2fTvtv+tAyT76HLtmHdPbeP/3mnpJ88Y1qLdJLMi6omeu47kJZhEpUbV0vMMm3DhGU7zeii21rXkD/8Wh90GHXfj7KejJb55cXoyfj3/UflhwfbyaIth7Ksr8HSUYsVhM+u7zK/9htVSZQa5Uqa90ppa5Hb1yt3yZah3X0qOD84bqVc08i3qGpWoSOZX/ekyt6UwK1suo+fL9kh91xZx6clrtf7i81f7+PhbfOBowG7fI6dOm2CQNWhQUVJTjlhuuHubn/2+dfvDR6saeuM5vM83OlCn4Dwxe9/lTvb1pKcSLN4jAK1Qg7/8Tf5euVuswT7nNgBgUyYpj3UXrq8mXXIHgB7iEQQ43/CavDsVMvrp3qdOA6mZXgua9Jv69rl5EeLXQk1n5zsc/20169opUm5asoD7aRR1QSfoMk9RFpP0N2a+NbsOhbG9A560tXuhjZ1ymW578jxk1K5WLzfc58OWLtHW4HemvGbZ7uCBQf+jTkfLdxq3i+r71kwOg1DKO5RQZrnEop3V9zLkwM/pw5Rdztx8oxniLe2wo3t1zrk8+s0CEqD4SK5yJ1yebXebT+ULnuOHJc7P14mfa+oGdbzOGUCUrqWwlSxNMX3gPzOu6siN1/m2oriNmvjfssjirKri+Pd3bUgwoGbP+1ucCeH+neRPPvNupCJ1VqvRb0ydYNP/ZlQybnamuRuHfAfNh1qtJEGflpTyL0d/oGgFftSzwWegejzaldLsG4jf4f9WsOs7o92gY1ekLUrT1sFR87Z5DMc+9jJ0/LxQt8RdLUGTfFc1q6/TsN/MkGpdr+G47d9Z/Nt7I5ABgBymZ9gleZiWKlbk11CpfdJLrvWhkJ+3S46UiZcOrlkTrjzkbSYnjfNBwrmsa9Wy83vnM05sToW6bPF2z1DoO/+dFmWBO9I0RFZ2s3iHt2VHe+RVSrYkH3NWdLWq2AtO+4cqBtHLpRXpm70vH9auO/pr9fKkG+Dv59WW+EiPTIqL9G1FKbYD/IDEG3+J95ImL1hvyfJNxRtSRh+S9Mstx9KzzCjZCav3ptl+wLVPHELlN8SLdp94U2TTwP5OJsChjqaJ5x6OFoTyC2cUUNuP208IL3bXJDteoe8audoS0e4lm4L3NKmOUuNqiSEfOzj/1vtUwDy0lplZcDYs7k4keBfUmDR5rz73OQWgQwA+Il0DZRwm+kHfnl2SKw3LTIXzPer9wZNItURXHklUCE9HZqcE2MWbJX352XtXglk8ZacvYZ3gBHoPVdXvTo714UGw+2GDEQDWG+RDGKO+HWB/bguWfp7jV7ytzE5zdS/sYtCrmgemRhITU2VxMREMxN2QkLoCDcn9EtBq1oCALJXq/x5Ua2Tk5fqVSrlmLyRaOrfvrY81a2hbc7f5MiEydn1DwEgb+WXIEYRxNgTgQwAALDMbh05BDIAAMAyzV3SKS/sgkAGAACERae8sAsCGQAA4FgEMgAAwLEIZCI8yyoAAMg7BDIAAMCxCGQAAIBjEcgAAADHIpABAACORSADAAAci0AmTIWYbQkAANsgkAEAAI5FIAMAAByLQAYAADgWgQwAAHAsApkwFS/KWwYAgF1wVg5TieJFonMkAABA2AhkcmDe4x2kTe1yOXkoAACIIAKZHEgqW1J6NK0ayeMAAADsGsiMGDFCatasKfHx8dK6dWtZunRp0HXff/99adeunZx//vlm6dSpU8j1AQBAwRX1QGb8+PEycOBAGTJkiKxYsUKaNm0qXbp0kf379wdcf86cOdKrVy+ZPXu2LFq0SJKSkqRz586ye/fuaG8qAABwmKgHMsOHD5d+/fpJ3759pVGjRjJq1CgpWbKkjB49OuD6n3/+udx3333SrFkzadCggXzwwQeSmZkpM2fOFDspVCjWWwAAAKIayJw8eVKWL19uuofcChcubK5ra4sVx44dk1OnTknZsmWjuKUAAMCJikbzyQ8ePChnzpyRSpUq+dyu1zds2GDpOZ544gmpWrWqTzDkLSMjwyxuqampkhdcrjx5GQAA4NRRS8OGDZNx48bJ119/bRKFAxk6dKgkJiZ6Fs2pAQAABUNUA5ny5ctLkSJFZN++fT636/XKlSuHfOxrr71mApkff/xRLr744qDrDRo0SFJSUjzLzp07I7b9AACgAAcyxYsXl5YtW/ok6roTd9u0aRP0ca+88oq8+OKLMnXqVGnVqlXI14iLi5OEhASfBQAAFAxRzZFROvS6T58+JiC59NJL5c0335T09HQzikn17t1bqlWrZrqI1H/+8x8ZPHiwjB071tSeSU5ONreXKlXKLAAAAHkWyPTs2VMOHDhgghMNSnRYtba0uBOAd+zYYUYyub3zzjtmtNPNN9/s8zxah+a5556L9uYCAAAHiXogowYMGGCWYAXwvG3bti0vNgkAAOQDth61ZGcuYfw1AACxRiADAAAci0AGAAA4FoFMBNzZtlYkngYAAISJQCYCujYOXdwPAABEB4EMAABwLAKZHCpauFBkjwQAAAgbgUwOXde0mjSqkiD92pEfAwBAvi6Ilx+VKF5EpjzYzlxetu1wrDcHAIACiRYZAADgWAQyUVayeJFovwQAAAUWgUyUzXn0qmi/BAAABRaBTBRcXqec5/J5caQhAQAQLZxlI6BpUhmpVf482Xow3VyvnBgv79zWQooVKUwgAwBAFNEiEwEasMwceKXPbdc2qSKdGlWKxNMDAIAgCGQipLADCuQN6FA31psAAEBEEchEweV1yovdzHrkSrm1dY1YbwYAABFFIBNBC5+8Wkbd3kJubF4t189VvlTxXD3+9svOBS0J8UWldoVSud4mAADshkAmgqqWKSFdG1eJSDfTzS2TcvX4CyuW9lyuV+nc5ZzaNqy7RMqzf2kkdvCfm5rEehMAALlEIGNT1crE5+rxN7Y41yqUUKKY2Em9SqXki36XxXozpOcldLUBgNMx/Nqm/n5pDdlyMF3aX1hB+o75OazHFitSSErHnwteqp9fwvwtZP98ZAAAwkKLTB5od2F5KerV3fTuHS3lLxdXyXZI95AeF0mHBhXl3zc09gQj4XjvjpZybePK8sg19cN63EVVEySS3Uj/uLymz22NqyaKS1xhPU+nhhXFaR7tXC/WmwAA+R6BTB745J+XyvoXu3quxxX1fduzaym5rfUFZqqDtnXDGw3V+aLK8s7tLSWxZHhdS091ayhrnusst/mNciodH34DXqWEOGleo4zn+vJnOsn55xUX7zimhdf9wTzWpYH87942EinXWKjxs+Xlbrl6jULZHNjHu4YXYDpdmZLF5JchnaXlBefHelMA5CMEMnlAT2jawhLM4kEdZfQ/WoV8jqJFCstnd7U2rTvZvp7krg9JW4+0a6qw34lY81pahXkS0m3xDtzKlYrLsk77ehWCPv67AW1Ny1L9yrlPWPb2317NY14byL+lKr/TdzOxRDEZ1z/2+VEA8g8CGRuolBAvVzeoFPUCd7kNcBpXS5Sv7r087Md1bFjJzD9131V1PLd5dyzd2Lx60Mc2qZ5oWpZUqGAwHBeUKynxxWI/K3nJ4kVl69Busvq5zlIQuFuoInUcAUDxjZKHzv+zi6dZUvZdKVbc4DUyKRoilRysJ66x/S6Tx7s2CHh/pcSsrTSBNKmWaGrihKN1rbLyWBffLpyKpa29XqTNe7yD53LHBhU9J/eE+GKmpcLuNAD86B+X+IyIC0fJ4rEPHgHkPwQyeWjxUx1NjkCZksUDprp+2KeVyUP5oHfwbqYeTauavzXLlfS5/dWbLz53JZcBSM3y50m0ubzegLiiRcz7kl19GT3pr36uiyx9uqPl1ykVV1Tu92u5Gn5LM8kLGgjWKHv2ON1x2QWS9OflQPzzpuzovOJFTfJ5Trd11O0tg95npcsUAAKx/7dnPqIn7FC/vLUL5pfBnUNONnlzi+oyvv9l8s2Atj6339giePdMTrq6VNL5wU+86hXv4MnLZ3e2zvY1/Ect6ftSKi5vfrGHCigi7Zv7rzCB6ZAe0S8CeHf72nJZ7bLm8jPdG8pNEfxM+Ao/Uv7+X21N12Q4Stig+w+A/VFHxmaySzDV+1vXLmcuH0jLCOu544udi1t/eLCdKZT347pkef67X81tOlTbezRPn8tryt6UE3JV/SDJuEFGUFfLwVDxvBJusnKVxHjzHuSUjtCyMgt6pLrxNCH7xKlMKVG8iHy1fJf8b8UucQL/EV4ahOmoLu12G/zNWpmw3Bn7ASDv0SLjaOciCe/TgLs7w592aWn3zfPXXSQNqyRItTK+AYcO1fZu2SletLAM7tEo5Kgib+XOKy6PXFMv7K6lULdFOmn5Sr990aHx+l4E8/ldreXmltWlQ7BgLkzuY9M9mzpC/soHGO0VLCDQIEZpgnU0dL4o/MR0fxo0e/M+orMfvUpev6WpaRnUfXn1b01z/XoA8i9aZPIJ/UE76f4rZMTsTaYOTDB3tq0VwRf1vbrsmU7mRHrqTKbntiJRGMIcbjE9pcPbp63dJ/3a1/a5XYM0XWo+OTng43Syzdf+PJGeOHVG3pzxu4z6aXOOA67vH2grG5PTsrQMZRectbygjExbty/kOo0iWMgwlKvqVZCv77tcbhi5MMfPESporZUHOVoA8o88aZEZMWKE1KxZU+Lj46V169aydOnSkOtPmDBBGjRoYNZv0qSJTJkyRfKbkhHo//c/GehoqPd7twrrRGC1FSSnw2uDdZmU1aJ4EZBdhWQ3Hd7+n5svDjrs2kqyqT42t11A2lVySc2y2RbLy0krlI7qirRAnyXd9uY1IlvUTlv/ACAnov7tMX78eBk4cKAMGTJEVqxYIU2bNpUuXbrI/v37A66/cOFC6dWrl9x5552ycuVKuf76682ydu1ayU8e61rfnHiG3hiZGZjDPTFGSp82F5i/L/z1ooD3B9sqTfzUYdFv/T13I4heur6xRMKYvpfKz093Civwm/pQuyw1fZ4O0RoWSrDDpwGWBqexoNNifHl3m6hMXRFuIDn94faeStEAYq9JFH442TaQGT58uPTr10/69u0rjRo1klGjRknJkiVl9OjRAdd/6623pGvXrvLYY49Jw4YN5cUXX5QWLVrI//3f/0l+UrF0vHz3r7bS61Jnz8D83HUXyZKnOkrvNuFXqdVh0X9tdq4mSU4bh6Y80M7UN9E6JzmlXWAVSsdlyaEJpUHlcyf2S2uVlW3DumfpusqJTg3P5aB8emdrk4Dt3Z0Wbn5NMFUTQ8+wrqPP9D3RxPB7rqwjzwQZHm9lHrDsjs3Vf9bV0dcL5MJKpWXhk1fLrEeuyva1AESfe3Rrvg9kTp48KcuXL5dOnc790i1cuLC5vmjRooCP0du911faghNs/YyMDElNTfVZCorz4mKf4qQtQbH8QGsLieaGaH2TQC6pdXY4slV/bXa2Tk8khNNIVsivlSlUS0ywYfHhdtfd3CpJFg26WupVKhWynpAmQz95bYOgpQP0vux4z8au3AnJbtXPL2laxLyLBvqrWqaELSoyA1ARzEuwcyBz8OBBOXPmjFSq5DvKQa8nJycHfIzeHs76Q4cOlcTERM+SlJQkBYV+sWu9kFduClzPxYq6FQOfxPKLcKvJasFBHVGTF3VfvOnIKNW0eqJUToz3tMRYyRHSGi06oaaOSgv32FdJLCGX/TmcP6c0f+f1MEcWPRGgyrO2xuS3QKV8qeJm5Btgd6Vs8MM4pxyfYTdo0CBJSUnxLDt37pSC5K52teWWS3IevOkv/2E3NjEjnpz4z+Hd6jHQ4tDvUDRRWYeh970igqO7LPhXxwtN99inQU56wZKy/+/WFibfqOUF1lue9FgP/ksj+UuTKkGDinDd9GcgZpUGazkRm0ywnNPALL//WIiESM5sj7OKhzmn2axHrhSnimogU758eSlSpIjs2+c7bFSvV67sW0fCTW8PZ/24uDhJSEjwWRBe19DfL60Rsfmf3J7r0UhubV1D2oRRy6TLn5ND5rQEvubbjLq9hfRrF70g5I4/k5uvj2AXlDuA0u4xHdUUKTrdRSB6rP/Ztpan+KJ2Uc5/ooMJaHPCv2uqf/va5vkaV4v8/2JuZyS3Wo8nJwIFhPqZdvIv3bwSyc89zpr3RPBuWn8rn71GKtoo58VWgUzx4sWlZcuWMnPmTM9tmZmZ5nqbNoEjcL3de301ffr0oOsjdoqESAL5xxW15OUbmoQ1mkpzPDa82DVXTfFdG1eRp7tHr1tIiwhufKmrvNGzWcxaD6zW0fHPS9FujmCjgzRHRQPacMwYeKU5VpqI6584rc+X1zSg0jnHlj4VfC6uaE68XaZkMfP51b9aEfrNns3MyDwNFPWvFowMlQPkNN8NaOuZEiMUK6UNcpttwVxdWYWTu6gVyNV5YXTFx9moGzjqXUs69Pr999+Xjz/+WNavXy/33nuvpKenm1FMqnfv3qZ7yO3BBx+UqVOnyuuvvy4bNmyQ5557TpYtWyYDBgyI9qbCIm3x0KF3kRo9E3atFlfs58zK6+HuV9TN/aSKiwd1NJWMI0W7TCKxXeEqGqRVRmvR/K1VUshfltnV48ntDN36+dWk5QVPXC3XN6/myfnREXrLn73GZ56vyjb7Bdy8xrlW2ToVsq9F1aR6oozrn/0PzD5taoYcuv+3MLslg/nqnuy35bo/J911qmCJ+W7aCu79Xozrf5mlY5kTF9qoyzTqgUzPnj3ltddek8GDB0uzZs1k1apVJlBxJ/Tu2LFD9u7d61n/8ssvl7Fjx8p7771nas589dVXMmnSJGncODL1QpB72uKhQ8fzW2Kmnd3WuoapuaNdNjlVtEjhmNUbiqS5UWzV+PWFrjl+rDuPSbsJw+0Cy0mXVySKSj5w9bmZ4XV02opnrzE1ob7of5kpq3DvVXV8cs+s5Pto64j39Cf6thQO8bl7MUK1oFrVzL516O1ezaPyA0xZqfeU227GHx++UmqHCEz+fX1jWft8F897oYn8MwtAyYI8SfbV1pTt27ebodJLliwx1X3d5syZI2PGjPFZ/29/+5ts3LjRrK+F8Lp165YXmwmncMi5uFXNyFW/1SBEc4Bi0WVjt8Ojo/UCdSkES4i+pdW5X/xW4jgdBaZ1c6I9dYaekBY8ebXMfORKM+VDuLSuTqg8JH3e7HT+My/NOzjSmlBa50q7JjTvR4OZrhdVNnlv7jmyvOdz0+TxT++81KdmkO6XVZH4QdQzjAEPw29p6inkmVMNKvt2pyodZahdi/r5CaaQhefNTQHMQoUKFcicLMePWgLsRk8w2qQbzkiiaHHP6ZRU1r4zkufGfVeda1HwVsLr5Ni5UeCBAsHoKDD/2jjaGjYom3o5Vqf7+HbAFXJji2oy/JZmpuWiToVSPl1O/rQLSk+QmuPkHwBUCNGSo8+7+eXgPwLb1i1v9jW7k6y2MI26o6XJe/vX1RfK/93a3Cfw0uTxdhdWCBmwBatBFCl/ubhqWF3DbUNsb27oMfFvffLJ38kmktHPhHfZhUA1mnLb/WlVNCt5RxqBDGwnkvM/XVw9sqOxgvH+haotBrmtzRLKecWt/+IaeXsL+dfVdeWLfpdJfjLhnjYmmdx7VJz35ybYZKXet7bwygnRekyrBl8T9PW0NSwhQidj/UzqCcvqEPRB3RqYE+TldcLPRwo1aauVisyB8pA0aChnoStsxsD2Mvau1iag0qlY9P32Tw4Od8jvQ50uFDu4sn7wQKh4Ud/33Eo9qHDmOmtb1/e1S+WyBSZYgK41ptxG3d5Stg7tZupc2RGBDJzHQqAz97EO5ks00sPK/Wkz8pi+l3iq4OYFHTrdulZZef66wPNbedMugkc613dEl1Q4dOJNd2Kj5nToSKFXbg6vMKR3l4bWYwqnoKBWRPZvAbI6gakVVqolR1pu06fubFvLnFTv/bOVrG7F0nL5n8ng2uI08b4rTBeV/+zy4XioUz2f+b9yKpxdfaCjb/DUsUFFebhT8JpVGrjltJZLoOfSz9oNzavJN3/W+nqgo28r5Nh+Z7/nNLgPlyZa3x2kG7VTw3PV0rUkhnZbvX5L7ubGi5aC15mGfCXYKJQa5UqaJdq8m+bzip4sxkfgyzzaJ8RhN14svd5fHPJLPxI0p+P21hcETbD1jnv/e2sL6TN6qTzVraF8v3pPjl/T+9eqHo+Vg6/xmfU9P40IserZvzQy72uoVqBgv0HO9woiK5aOk/1pGUGfQ+c107onzV+cLnnlx4fby7AfNpgWpX7taodMmtf7NNh68ftfTZC9dnfKufssJnBPe6i97E874Umu9i73UNKvRfbi6mXCLmiq+VOTV++VvlcEnyPvllZJsmZ3iqzbkypt/+we80721h8TdkEgAyBqQd4vgzvnuoidFVZfo+UF58vqIWe3KdxAxp3rcH5J3y4mfeVIBjHBaNfMt7/ssZyHEy4dtZRboYKYUHR6ig/7tDJzcOlcYv9bsUvenPF7yLonOp3IdxbeD02IXrvbdw6+cn75Rm6BnlODynqVSsvof1wS9DX8k7U12NKRnco7kNHaTqknTgd8jpG3tfB0N9avXNoskfJM94by0uT1Pi09/i1N/vR/5N83ZC2SqSMnNx9Ij0nphWDoWoLtlApSkTYSI0UQmhZuU/+4PPzZzAPJiyAmp9vkHoHjPVQ4FO2e02J7iwYFL7gXDe5E2tsvOzfSpvuf00sEosOLreSGuRsVtHtUT3R/vyS8goiR1rFhJZMHpN1Q2oWUXeVk7eJRCdl8X3i3nrk1r3G+PNq5njTyC97+26t52Mm1vdtcYJ4vKK/WG80VCkSD424hjmlu3dWutnRqWMnynG/uQQLB/g+urBedZOmcokUGttOgcoJJUA1a2Mx+58Z8476r6pgTfKAkw+xoQLD7yHHPVBN2d2vrC8xJs2kYCeHen8kO9SvI7I0HPNNWRKKmy9uzNmVJvnR3N+gQfB29tGjzoYgkk7uTo7XlLBZdpOFqUKV0lpnqtWXFuyVJKye3e2W2pecbcPWFZqn55GSJJu+vK+/RaT882E4OHT0pQ39Yb7ph7eCp7g1NXlM4U8vYAYEMbEkTVIOiESZqtH8/3ARM7ykLtF//gnJ5l/ic226Qq+qfS2j0V79Sadm4Ly3onFU6uejKHUfkkgjVC3r4mnrSq3UN04Kw9WB60ARlnZMrHFrl9eZRi8TpNHfG/7PqP+xbAwWt3uvdBRfJUZA5+V3lnU7jva527zSsUlgmX9hO8obL0tB0p/wQ8UbXEhwtu5LzyDua32DHIMaVwzPZB31ayd8vSQparE6DCv3lqi0lkaAnZnc3iLaI6ag7Tfq0EvD/02u2du9RM0qrvH7U9xJPcOZEL13fWC6qaq3VSLtvtKssUqIdCCH3aJEBgAD01/2wm2LX5O8euhyKdgnpJKb6S7r7xZXlx1/3me5Bfx3qV5TpD7c3+9Tg2aniBFq75J05m+StvzcPq7yBTtL5t5ZJPsmt4dBcnP9M3SCRwo+t6KNFBkC+k9NWGCfSIEZpJelB1zYMWvJfZyn3vs/ub1HXxpXlmwFtc1SjyXvAQIXSoesDDehwti7LXW3PtmrplAw6X5GdPdXtbB7V3e1rW36Mu77Pjc2rSX5Diwwcx3uoa5EidC05lRZHm7ou2ZTrByKd/7T+ha5mVOPQKaFbVx7pXE+ub15Vapc/1yUXyfmKOmphua/P1mCpdn4JiS9W2NSCKZaL767+7euYUU5WR9x5Sg8811lK58O5mPLfHiHf0xoSD3a80HxZFcQJ0vILLfJ186aDnmJb0WLzhgdEMWfLam6SViHOqexmlNfJN38Z0lnOK17E5FOt0tpKhQrleib6nFTrToiP7pxXscJZAI6kIzzg/BNNp1zMQwPn8e7OeqNn07x5TRuEst6TZkZitm/4IkcGAAqgWJ/gb2h+tvia3cU+DEJ2CGQAAIBjEcgAyNfyY3Ij7DP8+bw4uopijf9wAPmaFq27rXWNiE7CB/z7hsZmgsm7r8xatwd5i0AGQL6mo0MCzeJb0GnhuLymQ+3fmPFbyKkhnOK21heYBbFHIAMABcgrN10sX63YZUoY5LUyJYvLymevidi0DoDi0wQg37mu2dkie02TrM9sXVDcckmSfHl3GxNUxAJBDCKNFhkA+c6QHo1Mbkz7KBfbg/31ubymfLp4u3RrkrNZnaufb716LmKjkCufTUqSmpoqiYmJkpKSIgkJCbHeHABAjKVnnJaSxYuEVU13+fY/ZM+R49KjadWobhtyf/6mRQYAkK/lJLFZ5ybSBfZHjgwAAHAsAhkAAOBYBDIAAMCxCGQAAIBjEcgAAADHIpABAACOFbVA5vDhw3LbbbeZseBlypSRO++8U44ePRpy/X/9619Sv359KVGihNSoUUMeeOABM54cAAAgTwMZDWLWrVsn06dPl++//17mzp0r/fv3D7r+nj17zPLaa6/J2rVrZcyYMTJ16lQTAAEAAORZZd/169dLo0aN5Oeff5ZWrVqZ2zQo6datm+zatUuqVrVWKXHChAly++23S3p6uhQtaq2gEZV9AQBwnpyev6PSIrNo0SLTneQOYlSnTp2kcOHCsmTJEsvP496ZUEFMRkaG2XnvBQAAFAxRCWSSk5OlYsWKPrdpMFK2bFlznxUHDx6UF198MWR3lBo6dKiJ4NxLUlJSrrYdAADk00DmySefNJNuhVo2bNiQ643SVpXu3bub7qnnnnsu5LqDBg0yLTfuZefOnbl+fQAA4AxhzaT1yCOPyD/+8Y+Q69SuXVsqV64s+/fv97n99OnTZmSS3hdKWlqadO3aVUqXLi1ff/21FCtWLOT6cXFxZgEAAAVPWIFMhQoVzJKdNm3ayJEjR2T58uXSsmVLc9usWbMkMzNTWrduHbIlpkuXLiYw+fbbbyU+Pl7C5c5dJlcGAADncJ+3wx6D5IqSrl27upo3b+5asmSJa/78+a4LL7zQ1atXL8/9u3btctWvX9/cr1JSUlytW7d2NWnSxLVp0ybX3r17Pcvp06ctv+7OnTv1HWDhPeAzwGeAzwCfAT4D4rz3QM/j4QirRSYcn3/+uQwYMEA6duxoRivddNNN8vbbb3vuP3XqlGzcuFGOHTtmrq9YscIzoqlu3bo+z7V161apWbOmpdfVod2aJ6NdU5qzE+loUZOJ9fnDGRrmFOyf83EMnY9j6Gz5/fhFcx+1JUbTS6yWaHGLWiCjI5TGjh0b9H4NTLybj6666qrwm5MC0KCpevXqEk164PLrB1Sxf87HMXQ+jqGz5ffjF6191NHH4WKuJQAA4FgEMgAAwLEIZMKgo6mGDBmSb4d7s3/OxzF0Po6hs+X342fHfYzKXEsAAAB5gRYZAADgWAQyAADAsQhkAACAYxHIAAAAxyKQsWjEiBGmiJ/O/6TzRS1dulRibejQoXLJJZeYKsYVK1aU66+/3lRL9qaFBv1nKL/nnnt81tmxY4eZbbxkyZLmeR577DEzyae3OXPmSIsWLUyWulZeHjNmTJ68Rzr7uf/2N2jQwHP/iRMn5P7775dy5cpJqVKlTAXpffv2OWb/9PkCzSKv++TE4zd37lzp0aOHqcyp2zpp0iSf+3VsweDBg6VKlSpSokQJ6dSpk/z+++8+6+jksrfddpsptFWmTBm588475ejRoz7rrF69Wtq1a2e2VSuMvvLKK1m2ZcKECeazous0adJEpkyZEva2hLuPWrH8iSeeMK933nnnmXV69+4te/bsyfa4Dxs2zBb7mN0x1ImD/bddJ/rNL8dQBfqf1OXVV1+1/TEcauG8YKfvTSvbkq2wJjQooMaNG+cqXry4a/To0a5169a5+vXr5ypTpoxr3759Md2uLl26uD766CPX2rVrXatWrXJ169bNVaNGDdfRo0c961x55ZVme73nrtJ5rdx0HqvGjRu7OnXq5Fq5cqVrypQprvLly7sGDRrkWWfLli2ukiVLugYOHOj69ddfXf/9739dRYoUcU2dOjXq79GQIUNcF110kc/2HzhwwHP/Pffc40pKSnLNnDnTtWzZMtdll13muvzyyx2zf/v37/fZt+nTp5u5RmbPnu3I46ev//TTT7smTpxo9uPrr7/2uX/YsGGuxMRE16RJk1y//PKL67rrrnPVqlXLdfz4cZ952po2bepavHixa968ea66dev6zNOm+1+pUiXXbbfdZj77X3zxhatEiRKud99917POggULzD6+8sorZp+feeYZV7FixVxr1qwJa1vC3ccjR46YYzF+/HjXhg0bXIsWLXJdeumlrpYtW/o8xwUXXOB64YUXfI6r9/9tLPcxu2PYp08fc4y8t/3w4cM+6zj5GCrvfdNF/y8KFSrk2rx5s+2PYRcL5wU7fW9mty1WEMhYoF9E999/v+f6mTNnXFWrVnUNHTrUZSd6UtR/yp9++slzm54IH3zwwaCP0Q9o4cKFXcnJyZ7b3nnnHVdCQoIrIyPDXH/88cdNMOGtZ8+e5h8m2u+RBjL6hRiInjT0n37ChAme29avX2/eAz2BOGH//OmxqlOnjiszM9Pxx8//BKH7VLlyZderr77qcwzj4uLMl7zSL0R93M8//+xZ54cffjAnkd27d5vrI0eOdJ1//vme/VNPPPGEmYTW7ZZbbnF1797dZ3t0Utq7777b8rbkZB8DWbp0qVlv+/btPifBN954I+hj7LKPwQKZv/71r0Efkx+Poe7v1Vdf7XObU47hfr/zgp2+N61sixV0LWXj5MmTsnz5ctOc5z2fk15ftGiR2ElKSopnniv/CTzLly8vjRs3lkGDBnkm6lS6D9qcWalSJc9tXbp0MZOCrVu3zrOO9/6713Hvf7TfI21G1Sbg2rVrm+ZqbfJU+pralO/9utpEW6NGDc/rOmH/3PR1PvvsM/nnP//pM+Gp04+f9+SvycnJPq+j86poc7P38dKuiFatWnnW0fV1e9yTyuo67du3l+LFi/vsjzaf//HHH5b22cq2RPL/Uo+n7pc37YbQ5vTmzZubLgvvZnu776N2KWh3Q/369eXee++VQ4cO+Wx7fjqG2s0xefJk0z3mzwnHMMXvvGCn700r2xLTSSPzi4MHD8qZM2d8DqjS6xs2bBC7yMzMlIceekiuuOIKc8Jzu/XWW+WCCy4wgYD212r/vf4jTZw40dyv/ySB9s19X6h19EN9/Phx808ZrfdI/2G131W/MPfu3SvPP/+86XNeu3at2S79kvA/QejrZrftdtk/b9pPf+TIEZODkF+Onzf39gR6He9t1ROkt6JFi5ovYe91atWqleU53Pedf/75QffZ+zmy25ZI0P5/PWa9evXymVzvgQceMLkFul8LFy40Aap+vocPH277fdR8mBtvvNFs3+bNm+Wpp56Sa6+91px4ihQpku+O4ccff2zyTXSfvTnhGGYGOC/Y6XvTyrZYQSCTT2iylJ7c58+f73N7//79PZc1wtaEsY4dO5ovoDp16ojd6Rek28UXX2wCGz2xf/nllybxLT/58MMPzf56T2Hv9ONXkOkvzVtuucUka77zzjs+9w0cONDnc61f5nfffbdJ1LRL2fdg/v73v/t8JnX79bOorTT62cxvRo8ebVqCNVnVacfw/iDnhfyGrqVsaJO+/srwz6LW65UrVxY7GDBggHz//fcye/ZsqV69esh1NRBQmzZtMn91HwLtm/u+UOvoL0wNJvLyPdLIvV69emb79bm1+VJbMYK9rlP2b/v27TJjxgy566678u3xcz9XqNfRv/v37/e5X5vrdRRMJI6p9/3ZbUskghg9rtOnT/dpjQl2XHU/t23b5ph9dNMuX/0MeX8m88MxVPPmzTMtoNn9X9rxGA4Icl6w0/emlW2xgkAmGxplt2zZUmbOnOnTXKfX27RpI7Gkv/T0w/r111/LrFmzsjRjBrJq1SrzV3/ZK92HNWvW+HzxuL94GzVq5FnHe//d67j3Py/fIx3Cqa0Ruv36msWKFfN5Xf3S0Rwa9+s6Zf8++ugj0xyvwx3z6/HTz6d+OXm/jjZDa96E9/HSLzXtO3fTz7ZujzuI03V0+KwGC977o92P2lxvZZ+tbEtugxjN7dLgVHMosqPHVfMH3F0ydt9Hb7t27TI5Mt6fSacfQ+9WUv3faNq0qWOOoSub84KdvjetbIslltOCCzAdQqZZ4mPGjDEZ+f379zdDyLwzumPh3nvvNcPy5syZ4zME8NixY+b+TZs2meGBOqRt69atrm+++cZVu3ZtV/v27bMMs+vcubMZqqdD5ypUqBBwmN1jjz1mMspHjBgRcJhdNN6jRx55xOyfbr8OVdThgDoMUDPx3UP3dGjhrFmzzH62adPGLE7ZP3cmv+6Djmjw5sTjl5aWZoZr6qJfL8OHDzeX3SN2dCipPq/uy+rVq81okEDDr5s3b+5asmSJa/78+a4LL7zQZ+iujnTQYa133HGHGWKq26775z+stWjRoq7XXnvN7LOOfgs0rDW7bQl3H0+ePGmGx1avXt0cD+//S/doj4ULF5rRLnq/Duf97LPPzDHr3bu3LfYx1P7pfY8++qgZUaKfyRkzZrhatGhhjtGJEyfyxTH0Hj6t26SjdfzZ+Rjem815wW7fm9ltixUEMhbpGHl9s3VMvA4p0/oIsab/gIEWrSGgduzYYU56ZcuWNR8mreWgHzrvOiRq27ZtrmuvvdbUONAgQYOHU6dO+ayjdU2aNWtm9l9Ppu7XiPZ7pMP5qlSpYp6zWrVq5rqe4N30n/m+++4zwxz1n+qGG24w/7RO2T81bdo0c9w2btzoc7sTj5++TqDPpA7ZdQ8nffbZZ80XvO5Tx44ds+z3oUOHzEmvVKlSZrhn3759zYnHm9bTaNu2rXkO/Vzol72/L7/80lWvXj2zPzpMdPLkyT73W9mWcPdRT+7B/i/dtYGWL19uhtjqySY+Pt7VsGFD18svv+wTCMRyH0Ptn54M9eSmJzU94eoQZK0N4h/wOvkYumnAof9TGpD4s/MxlGzOC3b73rSyLdkp9OeOAwAAOA45MgAAwLEIZAAAgGMRyAAAAMcikAEAAI5FIAMAAByLQAYAADgWgQwAAHAsAhkAAOBYBDIAAMCxCGQAAIBjEcgAAADHIpABAADiVP8Px6kdQHEqaHcAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.plot(lossi);" ] }, { "cell_type": "code", "execution_count": 14, "id": "d02b5d3e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train_loss=1.4996627569198608\n", "val_loss=1.7031011581420898\n" ] } ], "source": [ "train_loss = nn.training_loss(datasets)\n", "val_loss = nn.test_loss(datasets)\n", "print(f\"{train_loss=}\")\n", "print(f\"{val_loss=}\")" ] }, { "cell_type": "markdown", "id": "5b9a1e13", "metadata": {}, "source": [ "### Génération de mots" ] }, { "cell_type": "code", "execution_count": null, "id": "756e5bf7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "aveilles\n", "saienti\n", "pénies\n", "ints\n", "apprés\n", "précieursulte\n", "expulsé\n", "pournile\n", "achement\n", "citressort\n", "résignemention\n", "moyés\n", "soumonial\n", "hement\n", "crivéalis\n", "memblateurs\n", "résent\n", "qu'aucontribunifiencé\n", "prendu\n", "dété\n" ] } ], "source": [ "g = torch.Generator().manual_seed(2147483647 + 10)\n", "for _ in range(20):\n", " word = nn.generate_word(words.itoc, g)\n", " print(word)" ] }, { "cell_type": "markdown", "id": "a3372957", "metadata": {}, "source": [ "## Discussion sur l'inititialisation et la normalisation des couches" ] }, { "cell_type": "markdown", "id": "3e5bb317", "metadata": {}, "source": [ "### Loss initial" ] }, { "cell_type": "markdown", "id": "886bcae8", "metadata": {}, "source": [ "Notre réseau est en fait mal initialisé, avec les valeurs aléatoires que nous avons choisies.\n", "On peut s'en rendre compte en lançant une seule itération d'apprentissage et en regardant le score de _loss_." ] }, { "cell_type": "code", "execution_count": 18, "id": "1327d947", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 1: 30.4901\n" ] } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, 1, mini_batch_size)" ] }, { "cell_type": "markdown", "id": "094c2965", "metadata": {}, "source": [ "Nous avons une valeur de près de 30. Quelle valeur pourrions nous attendre? À l'initialisation, nous aimerions que chacun des 41 caractères aient une équiprobabilité de 1/41 d'apparaître après une suite de 3 caractères donnée, soit une distribution uniforme, car le réseau n'a encore vu aucun example." ] }, { "cell_type": "code", "execution_count": 19, "id": "42500af9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.024390243902439025" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "1/41" ] }, { "cell_type": "markdown", "id": "db5280b9", "metadata": {}, "source": [ "Le _loss_ devrait donc être aux alentours de 3.736:" ] }, { "cell_type": "code", "execution_count": 25, "id": "9df35530", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "3.7135720252990723" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "-torch.tensor(1/41.0).log().item()" ] }, { "cell_type": "markdown", "id": "8aebdb0e", "metadata": {}, "source": [ "Définissons une fonction `comp_loss` qui va nous permettre d'obtenir le _loss_ pour l'élément numéro `item` à partir des logits `l`:" ] }, { "cell_type": "code", "execution_count": null, "id": "fc333bbb", "metadata": {}, "outputs": [], "source": [ "def comp_loss(l:list, item:int):\n", " # Calcule le loss associé à l'item dans la distribution de probabilité\n", " # associée au logits\n", " logits = torch.tensor(l)\n", " probs = torch.softmax(logits, dim=0)\n", " loss = -probs[item].log()\n", " return logits, probs, loss.item()" ] }, { "cell_type": "code", "execution_count": 38, "id": "09cc8ea7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(tensor([0., 0., 0., 0.]), tensor([0.2500, 0.2500, 0.2500, 0.2500]), 1.3862943649291992)\n", "(tensor([1., 1., 1., 1.]), tensor([0.2500, 0.2500, 0.2500, 0.2500]), 1.3862943649291992)\n" ] } ], "source": [ "# Distributions uniformes\n", "print(comp_loss([0.0, 0.0, 0.0, 0.0], 2))\n", "print(comp_loss([1.0, 1.0, 1.0, 1.0], 2))" ] }, { "cell_type": "code", "execution_count": 39, "id": "d50c6032", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(tensor([ 0., 0., 10., 0.]), tensor([4.5394e-05, 4.5394e-05, 9.9986e-01, 4.5394e-05]), 0.00013626550207845867)\n", "(tensor([ 0., 0., 10000., 0.]), tensor([0., 0., 1., 0.]), -0.0)\n" ] } ], "source": [ "# Loss très faible car on sélectionne l'item qui a le logit le plus élevé\n", "print(comp_loss([0.0, 0.0, 10.0, 0.0], 2))\n", "print(comp_loss([0.0, 0.0, 10000.0, 0.0], 2))\n" ] }, { "cell_type": "code", "execution_count": 41, "id": "7e3bb4c1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(tensor([-3., 5., 0., -5.]), tensor([3.3309e-04, 9.9293e-01, 6.6903e-03, 4.5079e-05]), 5.00709342956543)\n", "(tensor([100., 50., 5., -45.]), tensor([1.0000e+00, 1.9287e-22, 5.5211e-42, 0.0000e+00]), 94.99999237060547)\n" ] } ], "source": [ "# Valeurs disparates\n", "print(comp_loss([-3.0, 5.0, 0.0, -5.0], 2))\n", "print(comp_loss([100.0, 50.0, 5.0, -45.0], 2))\n" ] }, { "cell_type": "code", "execution_count": 56, "id": "c00dbcfb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(tensor([13.4427, -0.7880, -7.3284, 1.1521]), tensor([9.9999e-01, 6.6022e-07, 9.5328e-10, 4.5945e-06]), 20.7711124420166)\n" ] } ], "source": [ "print(comp_loss((torch.randn(4) * 10).tolist(), 2))" ] }, { "cell_type": "markdown", "id": "2e6f878d", "metadata": {}, "source": [ "Il nous faut donc une initialisation différente pour ne pas avoir ce loss élevé au début, qui donne une courbe en forme de \"crosse de hockey\" caractéristique." ] }, { "cell_type": "code", "execution_count": null, "id": "216c7a0c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 1: 30.4901\n", "tensor([ 4.6166, 7.7147, 15.4573, 1.0207, -2.7009, -5.7819, -5.3783,\n", " 23.6806, 21.1588, -2.8979, -4.8657, -11.5645, 8.7837, 4.6332,\n", " -13.9403, -13.4483, -3.9130, 1.4592, 8.2284, 10.1325, -16.5248,\n", " -7.4705, -14.2061, -4.3193, 0.8513, -11.2382, 30.9323, 1.3783,\n", " -6.4384, 26.5325, -14.2742, 14.5535, 4.6325, 21.9473, 14.6677,\n", " 18.3815, -1.1816, 1.2932, 7.4166, -7.3696, -19.7648],\n", " grad_fn=)\n" ] } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, 1, mini_batch_size)\n", "print(nn.logits[0])" ] }, { "cell_type": "markdown", "id": "19aa6ae9", "metadata": {}, "source": [ "Dans la classe `BengioFFN` définie ci-dessus, les logits sont obtenus par l'expression:\n", "\n", "```python\n", "self.logits = self.h @ self.W2 + self.b2 # output layer\n", "```\n", "\n", "avec `W2` et `b2` initialisés de la façons suivante:\n", "\n", "```python\n", "self.W2 = torch.randn((self.n_hidden, self.nb_chars), generator=self.g)\n", "self.b2 = torch.randn(self.nb_chars, generator=self.g)\n", "```\n", "\n", "Nous allons donc modifier la méthode `layers()` où se trouve cette initialisation de la façon suivante:" ] }, { "cell_type": "code", "execution_count": 61, "id": "91e82b4a", "metadata": {}, "outputs": [], "source": [ "def layers(self):\n", " self.C = torch.randn((self.nb_chars, self.e_dims), generator=self.g)\n", " self.W1 = torch.randn((self.context_size * self.e_dims, self.n_hidden), generator=self.g)\n", " self.b1 = torch.randn(self.n_hidden, generator=self.g)\n", " self.W2 = torch.randn((self.n_hidden, self.nb_chars), generator=self.g) * 0.01 # Pour l'entropie\n", " self.b2 = torch.randn(self.nb_chars, generator=self.g) * 0\n", "BengioFFN.layers = layers" ] }, { "cell_type": "code", "execution_count": 62, "id": "712ae2d6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 1: 3.7308\n", "tensor([ 0.0420, 0.0911, 0.1608, -0.0033, -0.0175, -0.0748, -0.0536, 0.2406,\n", " 0.2165, -0.0141, -0.0452, -0.1238, 0.0900, 0.0641, -0.1163, -0.1273,\n", " -0.0343, -0.0122, 0.0917, 0.1035, -0.1610, -0.0921, -0.1565, -0.0330,\n", " -0.0077, -0.0974, 0.3215, 0.0067, -0.0524, 0.2585, -0.1674, 0.1523,\n", " 0.0457, 0.2221, 0.1550, 0.1901, -0.0024, 0.0209, 0.0841, -0.0792,\n", " -0.1927], grad_fn=)\n" ] } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, 1, mini_batch_size)\n", "print(nn.logits[0])" ] }, { "cell_type": "markdown", "id": "78b80d70", "metadata": {}, "source": [ "On a maintenant un loss initial qui est proche de ce que nous voulons. Réentrainons complètement le modèle." ] }, { "cell_type": "code", "execution_count": 63, "id": "6f3430b6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 200000: 3.7308\n", " 10000/ 200000: 1.9096\n", " 20000/ 200000: 1.4936\n", " 30000/ 200000: 1.7350\n", " 40000/ 200000: 1.5614\n", " 50000/ 200000: 2.4063\n", " 60000/ 200000: 1.5985\n", " 70000/ 200000: 1.6582\n", " 80000/ 200000: 2.1206\n", " 90000/ 200000: 1.4665\n", " 100000/ 200000: 1.7602\n", " 110000/ 200000: 1.7375\n", " 120000/ 200000: 1.3745\n", " 130000/ 200000: 1.4203\n", " 140000/ 200000: 1.6508\n", " 150000/ 200000: 1.6507\n", " 160000/ 200000: 1.6627\n", " 170000/ 200000: 1.1234\n", " 180000/ 200000: 1.5974\n", " 190000/ 200000: 1.3893\n" ] } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, max_steps, mini_batch_size)" ] }, { "cell_type": "code", "execution_count": 64, "id": "d28a7ee4", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGeCAYAAABhOIBvAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAVSBJREFUeJzt3QlYVOUaB/CXHVEBFQVFFHcld1BEcylxSSsr703N0sy0NMvCFqnU0m7Y5vVWplmZ3dI0y7KrRilqpqIY7mu54sbmAooKCHOf99MZZ2CWM8uZOWfm/3ueeXT2c2aGOe983/u+n5dGo9EQAAAAgEp5u3oDAAAAAOyBYAYAAABUDcEMAAAAqBqCGQAAAFA1BDMAAACgaghmAAAAQNUQzAAAAICqIZgBAAAAVUMwAwAAAKrmS26mvLyczp49S9WrVycvLy9Xbw4AAABIwAsSXL58merVq0fe3laOtWic4OOPP9Y0bNhQExAQoOncubNm27ZtZm9/8eJFzfjx4zUREREaf39/TbNmzTSrVq2S9FynTp3i5RlwwmuAzwA+A/gM4DOAzwCp7zXg47i1ZB+ZWbp0KSUlJdG8efMoPj6eZs+eTf369aPDhw9TnTp1Kt2+pKSE+vTpI677/vvvKTIykk6ePEmhoaGSno9HZNipU6coODjY4fsDAAAAjldYWEhRUVG647g1vDiiIRlxANOpUyf6+OOPddNAvLHPPvssTZ48udLtOeh577336NChQ+Tn52fTixESEkIFBQUIZgAAAFTCnuO3rAnAPMqSmZlJiYmJt5/Q21ucT09PN3qfn3/+mRISEuiZZ56h8PBwat26Nb399ttUVlZm9PbFxcXiBdA/AQAAgOeQNZjJz88XQQgHJfr4fHZ2ttH7HDt2TEwv8f1Wr15NU6ZMoQ8++IDeeusto7dPSUkRkZz2xKM+AAAA4DkUV5rN01CcLzN//nyKjY2lIUOG0GuvvSamn4xJTk4WQ1LaE+fKAAAAgOeQNQE4LCyMfHx8KCcnx+ByPh8REWH0PnXr1hW5Mnw/rVatWomRHJ628vf3N7h9QECAOAEAAIBnknVkhgMPHl1JS0szGHnh85wXY0y3bt3oyJEj4nZaf/31lwhyKgYyAAAAALJPM3FZ9meffUZfffUVHTx4kMaNG0dFRUU0atQocf2IESPEVJEWX3/hwgWaOHGiCGJWrVolEoA5IRgAAACgItn7zHDOS15eHk2dOlVMFbVv355SU1N1ScFZWVkGnf44gffXX3+lF154gdq2bSv6zHBg88orr8i9qQAAAKBCsveZcTb0mQEAAFAfxfaZAQAAAJAbghkAAABQNQQzAAAAoGoIZgAAAEDVZK9mchf5V4ppzvojFOjnQ6/0b+nqzQEAAIBbMDIjUeG1Uvpy8wlatPWk1LsAAACAEyCYAQAAAFVDMAMAAACqhmAGAAAAVA3BDAAAAKgaghkAAABQNQQzAAAAoGoIZgAAAEDVEMwAAACAqiGYsZJGnvcBAAAAbIRgRiIvLy9bX2MAAACQEYIZAAAAUDUEMwAAAKBqCGYAAABA1RDMAAAAgKohmAEAAABVQzADAAAAqoZgBgAAAFQNwQwAAACoGoIZB/hx52lKnPU7Hcu74oiHAwAAACsgmLHS5es3Kl32wtLddCT3Cr38/R5rHw4AAADshGBGooPnCnX/zy64bvQ210rL7H0/AAAAwEoIZiS6dLVU9//8K8WUui+bOv9rLW07dl53uQarUAIAADidr/Of0j08/U2m+PexLzJ0l/2Vc5mul5ZRoJ+PC7cMAADAs2BkRqLcy9eNjsCUlpfr/n+jXEOPfLbVgW8PAAAAWIJgRqK/c6VVKu3IuiT1IQEAAMABEMwAAACAqiGYkfpCeXnp/q+h2/NMSPoFAABwLQQzEt0OZVxDg6gJAADAdcHMnDlzKDo6mgIDAyk+Pp4yMm5XAFW0cOFC8vLyMjjx/VxNb2DG6aMx5eUa+se8dBr1penXDQAAwFPJXpq9dOlSSkpKonnz5olAZvbs2dSvXz86fPgw1alTx+h9goODxfVaHNC4WkSI6wKqE+eLKPPkRfH/G2Xl5OuDATUAAAAt2Y+Ks2bNojFjxtCoUaMoJiZGBDVBQUG0YMECk/fh4CUiIkJ3Cg8PJ1drFRGs+//FqyVOfW79gSAlBHYAAAAeE8yUlJRQZmYmJSYm3n5Cb29xPj093eT9rly5Qg0bNqSoqCgaNGgQ7d+/3+Rti4uLqbCw0OAkB/0Y4tf9ObI8BwAAACgsmMnPz6eysrJKIyt8Pjs72+h9WrRoIUZtVqxYQd988w2Vl5dT165d6fTp00Zvn5KSQiEhIboTB0AAAADgORSXfJGQkEAjRoyg9u3bU8+ePWn58uVUu3Zt+vTTT43ePjk5mQoKCnSnU6dOybJdhtM7WIQJAADAIxKAw8LCyMfHh3JyDKdl+Dznwkjh5+dHHTp0oCNHjhi9PiAgQJzk5o1UFQAAAM8bmfH396fY2FhKS0vTXcbTRnyeR2Ck4GmqvXv3Ut26dcmVvPQ6zRzOvuzU50aLGQAAABeWZnNZ9siRIykuLo46d+4sSrOLiopEdRPjKaXIyEiR+8KmT59OXbp0oaZNm9KlS5fovffeo5MnT9KTTz5JrqQ/y+TK9ZcwQAQAAODkYGbIkCGUl5dHU6dOFUm/nAuTmpqqSwrOysoSFU5aFy9eFKXcfNsaNWqIkZ0tW7aIsm5XQhABAACgTF4aN+uTz6XZXNXEycDcfM9RUvedo6e/2SHptidmDiRHOpJ7hRJn/S7+f+ztAeSNBB4AAHAzhXYcvxVXzaRcGJsBMKW0rJzWHsihgquleJEAwOkQzEh9odwwltl+4gL1eHc9rT+US0rHA4jLd5ymI7nOTb5Wmp1ZF+mpr/+kk+eLSEnmbjhKT/73T3r4U9PNMAEA5IJgRiJXLiNQLtNM4COfbaWsC1dp1MLtdi2C6Qwr95yjpO92U+KsjQ57zIJrpfT6T3sp8+QFUosHP9kiOlA/9XUmKcnPu8+Kfw/neHawCQCugWBG6gtlRSyz9dh5Opp3hRzl2cU7SQ6lZfYFIpevl9Kd76yjyT/sIbntOuX4CrKZvxykb7Zm0eC56htN4CDU3WSdvypGngAArIVgRiJrBmaGzt9KvT+4mbBri1MXrtLohdsp4/gFRf/a/SHzNJ0tuE5LtsvTdVluR/OUNVXj6Xq8t16MPJ3Ix/sCANZBMGND0zy5PfvtTko7lIv8A/BIh7LlWSwWANwXghmpvBybzLrxrzzKKbxu9Pozl6457skAAADcnOxN89yFtwMTgH87cDuB09E9acAzuFd3KAAA+2BkxkmSl++h4Z9vFdU/m4/kO+tpPebg7ayqKjBOzknYwuultO9MAV56ADAJwYyT+sx8m3GKNh85T7tOy7Ou0/krxWL6ytWultwQ1VxlTgwurhTfoK4z19HEJfJUfYFr3fXeBrr3o0206W/8CAAA4xDMyDzNxM3EBs3ZrDsvR8CRfvQ8xb61lsYvkrbcgpxGL/xTVHN9sv6ITffffeoSfZuRZdXrtHrvOcouvE4rdt3sdaI2SghClaTiy3G+qET8u+ZAtms2CAAUD8GMzN5JPSQO0KZcvPVFrfXH33mUd7nYqueYv/Go+PeXfca/7G+UlZOzpB87L/5dtC3Lpvtz4Je8fC+lHXRCV2I7YojrpWUOCUK4q3HHGWso86Tt/VV4O6RuC38euVkiLz+gBFNX7KOUXw66ejMAQOUQzLggJ0D/sX7/K8/guse+yHB4+/tWU1N1AY8p6w7lmMxF2X+2wOi0kZxdkf/OdVzTQUc7nl9ELaek0vNLd9n9WNzV+OLVUqs7+mpuRWIcxAyZv5Ue+GSLpLwhHinccvQ8rVPAEhZnL12j/6afpE9/P0YlN5QRXAGAOiGYkciWA/dfEpvdcZn2Gz/vp+IbZRZvy+vfpB00HngY8+qP+0Sn37dXHzJ7uznrjQc7/Kt54IebaPr/9pu9Px+MHv18G6lJxglpyxis3HPWINhbuPm4+NfctNaFohKRxySdbaM8RSVlorkij/6dM1Hqb4wSRmYQwACAo6A0WyJbBiH6/lvaOkIjFtwcjQkPDrR4W/5FzSf9ku5rpZaDIEtMTXN89sfNA/dX6SfphT7N6dLVUooOq2r0gL/JSJUWBwFV/X0pvnEtUiOe8ptwazmJ4ykDJAW1HCjw1BE7/FZ/CvD1cfh2XS+tHIx4SVhYVC4uXLoMAAAjMy6ZZjLxzX/6YuX1dvIt/Lqf9/tR2nrMMQcpnqbgihEeVTCm/fQ11Ov9DWINHSkH19zC6/TEwj/FNIi9tNMqzlZwzfhrYc6V6zdu3/9qKSnFxG+NV3sdzr4spiFNjQzyPtz9wQaateYvmbcQ7MV/v/9NP4EXEjwOppkkkis/hNdhMmf6/w6YvX7mL+anj6zx064z9OgX26jvv82vK7VDwmKARSU3KNdEIjPf//EvM+iIgvNilODDtL/pya/+lL3Mvd/sjWIa8vNbo3AVfbH5OB3LKxLbA8rGf79TV+ynbbcS8QE8BaaZnNRnRt/XW0/q/v+BhV+7P+8+a7aZmDV4lCesWoDJ63/bfzMvJP9KiajWCfSzfXrkst7oREUPfbJF/Hvy/J+0/sVe5Gz8yzXIX/kffe1IyPpDuZQYEy778+09bbwxXVm5PPk1pkI0FKrbD0uigKfByIxEjhqYGTw33eQv7b9zrlhVpdT2jd8qXf7015kGqw4fPHd70b5/WzFN8Pkfx8iRLl2tPF1z5qLpNai48V7mScfneGQXXBe/XF9cttthj8l9bjhR2p5SbQ4gTSmxIlmXu0ubawWgVMi5AQB7IJiRzEsx1TXswVujGxWl7s+mMf/9UxxYZ681DF6KrSh/lTIFpH8AspTT8uKyPVb1Q+GSdQ78pPTI8bKyW7Axv+w9R6//tNfqKp+RCzJEs0IuL26UvJoaJa8y2e/HEnuDEJ7WG/75NoMmjUrDwRYnvFuaXgUAsAaCGakvlIqqNbhHy8frjtDstYY5Dto+JNzf400LpdaWXL5eSt/oTZdZsv5wLg34cJMItCriKS0eiTEWSJTZMNrBI1P3/OcPs1N0FY1btIO+2ZpF32eetuq5KvYJ4s199ce9t8/falrIybU89L9qzzmT/WA4CDEVbEkddTJH/1mteVml3NZLYkjJwRa3Ikj6znSPHjREBgBrKT9xQCHkbBAnB2O5OMt3nqEHO0ZSyupDdEBv+skW3KX3LyumxXhqjae89Ke9tCZ9t5tW7T1Ho+9s5JAD3Cs/7BHP89y3O+n+dvWsLsU2tw3WfgzKNRrq8e560ZJfOzI25d4YahMZQnENa1S6/fG8ImpTP8Sgrw0/L79+Pk6KqDlBu2aQv9ESfGN42w5L7KmklVNoXZdrAABzEMxIpK5QxjSpHYYtxQ7aZGFH4ECGfbHJeDWNtbiSyhgOcCxNI50ruCamwswFr9YEtj/vOktnK4yYzFh5s0It5aE2Ju/HJe1azyzeQRHBgbThJcNk6Ve+30Ov39uKHImnf7QJ2vq9jComnpfeKKdat5LJuUM0AIArIZiRSGUDM3atq6S0ni889bJwywmjOSZB/tIqrnhqQ9uc0NLq5n4+3jR9UOtbl9j3xp8zM/WzYtcZyY/DC2luO26YU7X0z1PUv3UEOdLfuZZHWLSJ5/ve7EfVAnzJUuU4B5AbDudR5+iaBpebyp9yVU8hAFAv5MxIJDUnQM0Krt0u9eYDVI6J9vj/WnWoUoWNrXkO01bsM3s9P+4MI712eKkAzjHpI7HLMueqSMXrBanF1ZIyyQG3o3NRjDVPNOY/a/8WuVKPfG66eaKUvy5rqroAwLNgZEYiTxiZ0fe/3WfFyRhLXYmtwcskWLL1+HmzIx55Dtyeij7baF+JuiNW1pbK0kfU1hEPS/lV1/QCKmN+3HlzBGr/2UK7/p7++DtfJDlHhFhe9sPTedr3FQBGZiTCl4NyvZt6WLbH5qkcZ9p16iIt3pYluVy+SGL1E1ewSUm6zb9cUilI4Skic8HRsM/sX65CitMXr1GXlDS06weAShDMSOQJ00zOJmXa4FD2ZYPpL2ap94z+e5V7+eYIzlorVhpnvBL15B/2GFw2efkeGv75VlGhJFcV3JQV+w1Ku40lDut7ucI2mtJ15jqLt+EeRfqP12pqqtkV2nk1dWMLlHJ109urD1KqFf12zDUNrOitVQcl3xYAPAOmmSTyRtjnEg8YaQBnzWTJjJUH6aNhHURptDUe/jS90mXf/XmzB82VYumrlBtLXNbiFcgdSWPlbbkqSbsiuCmjv6rcF0jf4LlbjK6gPv/W9BxXRBmbajtZId+mj5H1wLgL9a/7bWtACACeBYdoiTAyY/2BVK4ur7xkw70fbZJ0W1N5P/bIN9OLxho86uRIT32dadXt56w/Iqq8HC1XbzrLXCm8ft6T/lpe6UfPi8aCPAKz/YTlRU0BABDMSIScGfOMrdLLXXXlkHYol1xJDYv48YjJkgzjuTds0daTVGDDyBB3Mrbky823+wXxCI2pESNTieSLtmXRd07OVQIAdUMwAw6x0orSZzkdzy+ivWfQxI1zWSYvN557w7hnjS3TXFLShfSbBFpTEm+skSIAgBQIZtxwbSZPdtf7G1y9CarBCb/OSPK2JmEaAMAWSACWDNGMWnEiKTjGLitX9ja3+rqt652ZWqgT9F5bfF+Bh8HIDLg9lPI6ztD5juspwwuBmmJuMOcGghkAcEUwM2fOHIqOjqbAwECKj4+njAxpix0uWbJE/Hp74IEHyNUwzQRqtP3EBbEqudpcltgMEADAKcHM0qVLKSkpiaZNm0Y7duygdu3aUb9+/Sg313xFyokTJ+jFF1+k7t27K+KdsnVIHMCV/jkvnX7YcbM/jprwIqIAAIoJZmbNmkVjxoyhUaNGUUxMDM2bN4+CgoJowYIFJu9TVlZGw4cPpzfffJMaN25MSoBQBgAAwAODmZKSEsrMzKTExMTbT+jtLc6np1fusKo1ffp0qlOnDo0ePdricxQXF1NhYaHBSQ4YmAEAAPDAYCY/P1+MsoSHhxtczuezs42XhW7atIm++OIL+uyzzyQ9R0pKCoWEhOhOUVFRJAdUBwAox6Wr1i1PAQDuTVHVTJcvX6bHHntMBDJhYWGS7pOcnEwFBQW606lT8nQOxcgMgHIczTNd8g3giU6eLxIJ/55K1j4zHJD4+PhQTo7hyrt8PiIiotLtjx49KhJ/77vvPt1l5eU313bx9fWlw4cPU5MmTQzuExAQIE4A4Dm88evCIXiF86Hz06lRWFV69x/tHPOg4BI937vZMHTNCz2oWXh1j3sXZB2Z8ff3p9jYWEpLSzMITvh8QkJCpdu3bNmS9u7dS7t27dKd7r//frrrrrvE/+WaQpIC350AyuGDXgkOwb/keTFP7YrwoH4HzsmTN0qe3gGYy7JHjhxJcXFx1LlzZ5o9ezYVFRWJ6iY2YsQIioyMFLkv3IemdevWBvcPDQ0V/1a83NlQmg2gHBiZcQxz3ZR5IdKQID8HPROAyoOZIUOGUF5eHk2dOlUk/bZv355SU1N1ScFZWVmiwknpUJoNoBwIZuS1fMdpSvpuNz3Xuxkl9Wku87MBqGRtpgkTJoiTMRs2mF8YcOHChaQEmGYCUP80U27hdXrv18P0aJeG1C7q5qgvVPbqjzdXXP8w7W8EMzIoLSsXP5B9fZT1Q/5GWbnitkkqdW61C6A0G0A5bP2+fen7PbQs8zQNmrPZ0ZsEIDnpuuvMddTtnXWKWjR1z+lL1HJKKs1Zf4TUCMGMREg4BFAOW3PYzK3iDeAM568UU97lYsopLKaCa6WKedGn/LRPLOLKI5dqhGBGotrVUf4NoBQ+mPcFMMrLQ/82EMwAgOogAVheGiOzH/vPFtCBs9LLfnkK5VB2oaKmUsB9IZgBANXJu3Ldpc/PicTpR8/TR2l/08gFGSKhU0kc/eP8WkkZDfxwEw348A8qvlEm6T7v/XaY+s/+g6avPECe4GrJDUrdly3+BedDMAMAqjP5h5vVNq7S+e00GvbZVvpgzV/0+1959Ms+42vNuYvLxaUGgY0UczccFf8u3HLC6PUZxy/QO6mHTAZHHCBmF7g2aNUvVZ+4ZKfZQC5p6W56+ptMemnZHqduG9yEYAYAVOdvGxN55UonKC6VdoBXGldOAD38aboIeBZsOqGr8tH3z3np1CUljXZkXbTp8TUajTg5AvfcWbHrLC3JML32X+r+mwHtqr3nyF2U3CgXI5BSR+NcCcEMALgdztVYuj3LYQczrfd/PUzd311X6fLSMo14vlMXrorz+s/LlSvXVRbsODPIOZZ3hf7KuUxt3vhVTNtp7Tp1Sfz7fab1Sy3w6z90/lZ67IsMh34GLnrYau3Tft4vRiCTl7t2JFQxTfMAAJyJczVYoJ8PDWof6bDH/dhED47P/zhGx/KLRAuHP19LpM5vr6WmdarTZyNi6c531otqyO2vJZJqmDn+r9xzTjQddKQZKw/Q1ZIyMW33bO9mdj8elz1vO35zBenC6zcopIoylmVYkpFFy3eeMXk9T6356TVRmr/xqFg369sxXSRX1HqR43ybkSX+Xb7jjPhbeqV/S8W8lhVhZAYAVOlcwTUqKjafbGlN9Y09OJDRTpX8N/2kGKk5eK5Q9O7Qjs64i9dv7ZNaOHJq0d5BnsnL94pcIWMOZ1+m5q//Qim/HNRd9vbqQ6I3EndidrXF27Jopt62KQ2CGQBQpYSUdRT31lq7HoPLhi8UlYg27o6qSNLoDWts/Dvf5O240+qKXaZ/paulA/off+fR7ltTQs7O59j4V57khGTt1BVPmfB7rjTcrI6DpU9/P1bpuhvlyqiW+zbjFC3adpKUCMEMAKjWNb1clG3HztMTC7dT1vmbeStSfqWPW5RJHWesoaav/UIJKWmVklDlwgd/PnhNXLLLrsfhfJBVe87R8VsjQ3JY9ucpk3knPDrGeSlyLg9hKox6a9UBGrEgQ1QZSfXAnM1i6mTKCmWMLkn9tBVeu6GYJNzXflTGa1cRghkAULV/rTogWsQPmb+V1h3KpWe/3SF5hOHX/Tm6/+dfKRGP4wyOGhn47UAOPbN4B931vvkFe+3B61lxJY8xZy9VLp3mPisf/Ca9Jb6t4SNP52lfA2sdvVUNx0Ga/pIC209coMRZv9OWI6ZH1FyBK6S6zVwnguCjeYaVfByA/51zWXf+8nXP7HODYAYAVO2zP47TK3p9Z84YOcC6q51Zjpve0a+40p8qY7tPS3+e/6z9mz5ap47FCjlQa/fmb2JUT1suzjkqj3y+zaGB15lL1+zumcPBNo+A9f7gd4PLJ/+wh/r8e2OlFc89DYIZAFC9vWecn7NhS36JsWDhznfW0WoF9CbpZCb/SMp6P9plCw6cs5x0nXbQ+tEUfdZOB/43/QS98fP+Spdry761VWoOruQXOKeHR1W4Z44ceBV4R1D7ohMIZgAAZCKlkOb0xWs0ftEO2nu6QPb3gadVOM/FmMvFN2jlnrM2H9SHfrZV0u24amf0V39Kflz9OIr7+HDidNs3frVq26au2G/QibikrJym/8/8MguXrztmRet8G6cutT2LQBoEMwDgVsr0Kj8+3XjM5EKH0ZNXSW7qtuWo9BwKU5Unl66WiF/pvD1PLKx8MB/91XaD819uPk6TvtttdPt5pIEPxgu3HCdr8OgEV4FxvxNjpLbif+7bnfRD5mmRfKxlquS4Ii5Zr0hq8NT93fUicbrIQgXTwezbz2GsYeGxvCJasNn8a/e/3a4dLdtppkJMjmRgjcqHZtA0DwDcysWrhr+ofzvAi/+VUdv6IRbv+8Fvf1FpeTl98M92uqmVu2/lKKQ+313S85vKF2k/fQ0F+HrTigndjF7P26jvzVsjB/e2rUt3tayju/xiUYkYaZCKy86v3yinagG+9NWtpNmpP++noZ0bVDqAcXUYdzm+YWEa5+fdZ8XJFs8vNV/BNfa/f9IDHexrdDjqy9uBIQdvUpyQUhGmkCN+mzd+o91T+1IVfx9Xb4piIJgBALe2YPMJ3ahBw1pBZm+79M+ba+880a0RtY40DH7u/9j+8uPiG+VWHw+vVGgMqF+OLgUnh3LpdubriQY9Wnj6I7vwuuQux1JwoGWtiq8HVycZq1Daf7ZAtvwa3obJyw1Hpb7jknTS0PB467sd8+vNo3Ax9YJJDvz+8esRF13T5scovlFGLy7bQ72a16bBsfVJ7TDNBACqZy5AkDr9UTHoMHYAsZalEQ5zQUvFA7KtORTaHjTck0UfN5wr1CtLNkdqE11bAiEOGLjax/Rzexm01pdLUXFZpQZ73FOlYoDGXae5RNrcmk9cKj/gwz8sluDrd7CWaxFUU77dlkX/232WJi3bTe4AwQwAqN55BXZ0dSSeeuF8kdR9N1dmtsX+Cks75F4upukrzSfBWsvaUSPtuj/2Nv3jhSrlor9PHL489MkWUSJtqveOvrOXrtHXW013zB08dwttPXae1h/KpTUVRqMcvUiqpelYtUMwAwCqJ7VU96SZ7sDG/P5XHrnas9/upLRDubqkYCmklFL/ZGbBw4q494oUUkd6HI1fF27Uxw375Hb4VuD0o8TXb/7GyssT6AeUvLr3qIWGyd9SmHuLF2/L0nXC5qCIXxc+cYWWfoM9d4KcGQAAE0ZWmJrxVHzQlYJX1G4eXo1csWYQ4zyV1++NsekxLl61fXTPVDL0Kz9Iqw5ztFdvNc57qEOk0VW6v386gdwNRmYAAJzI3twIU/fXX5mbf43vOX3JbK8UngKRo1fKXznSRnGkqtiN2BwpDftMPQf3+zF+nbRSdSlTe8623MTo0a/7DacrK67Kve+M/D2PHA3BDACAG1i6/XaC7G/7c0T11cAPN5m8faET1/DhnJjC66U06kvrR7p+2mlbCbij7NJbMsJZldnSlj5wXMbwrDV/GZy/72PTnxulQjADAFDBZxuPGVSaqMH7v/1lsDAhy1JIF1mu7klauovWH7Y+B4mrvHiU6Zut8lYzmcILeZrbNt4vR+PGgHK6WlJmNlnbUtCmPwqoFMiZAQCoIHV/NtVc7S/L67L9xEWT193/8SYa17MJRdU03w9HjdYevJnEbAtrevzYsvK51ATnS9cM82o+XnfE5FSOPUx1ka7I1sUrF22zLzCcumIfzX00lpQEwQwAgBFcMiuHKT/tM3ndntMFNG6R6ZEAsGzuhqNWv0w5hdICoIqjQ9km1rlyBHNVUKy0rNyhi1dqrMhNMtcXyFUQzAAAGMHr9yjNtuMXJLfn90Rbjp6nyNAqTnkuOcv2eTHOQ9nmS6jVNg0qN+TMAACA21iWeZrUzlIgI0fH4H1nXFt5ZS8EMwAAAKBqCGYAAABA1RDMAAAAqIyUtaHsweXwppy/ory10BDMAAAAqIzcwUy2mbJvrmY6mufYTs+qCGbmzJlD0dHRFBgYSPHx8ZSRYboL5PLlyykuLo5CQ0OpatWq1L59e/r666+dsZkAAABAlhdl/eVWY0aPCWaWLl1KSUlJNG3aNNqxYwe1a9eO+vXrR7m5xhso1axZk1577TVKT0+nPXv20KhRo8Tp119/lXtTAQAAgIj+tfqgql4H2YOZWbNm0ZgxY0RAEhMTQ/PmzaOgoCBasGCB0dv36tWLHnzwQWrVqhU1adKEJk6cSG3btqVNm9S3VgQAAACoPJgpKSmhzMxMSkxMvP2E3t7iPI+8WMIrv6alpdHhw4epR48eRm9TXFxMhYWFBicAAADwHLIGM/n5+VRWVkbh4eEGl/P57GzDJcj1FRQUULVq1cjf358GDhxIH330EfXp08fobVNSUigkJER3ioqKcvh+AAAAgHIpspqpevXqtGvXLtq+fTv961//Ejk3GzZsMHrb5ORkEfxoT6dOnXL69gIAAHgSjfSlnNS/NlNYWBj5+PhQTk6OweV8PiIiwuT9eCqqadOm4v9czXTw4EExAsP5NBUFBASIEwAAADjHB2v+omd7NyOPGJnhaaLY2FiR96JVXl4uzickJEh+HL4P58YAAAAAOH3VbJ4iGjlypOgd07lzZ5o9ezYVFRWJ6iY2YsQIioyMFCMvjP/l23IlEwcwq1evFn1m5s6dK/emAgAAgArJHswMGTKE8vLyaOrUqSLpl6eNUlNTdUnBWVlZYlpJiwOd8ePH0+nTp6lKlSrUsmVL+uabb8TjAAAAAFTkpeH6ZzfCpdlc1cTJwMHBwQ597OjJqxz6eAAAAGp1YuZAxRy/FVnNBAAAACAVghkAAABQNQQzAAAAoGoIZgAAAEDVEMwAAACAqiGYAQAAAFVDMAMAAACqhmAGAAAAVA3BjBX+EVtfvncCAAAAbIJgxgojEhra9ioDAACAbBDMAAAAgKohmLGCe61iBQAA4B4QzAAAAICqIZixgpeXfG8EAAAA2AbBDAAAAKgaghkrIGcGAABAeRDMAAAAgKohmAEAAABVQzADAAAAqoZgBgAAAFQNwYwV0DMPAABAeRDMAAAAgKohmAEAAABVQzBjBTQABgAAUB4EM1ZAzgwAAIDyIJgBAAAAVUMwAwAAAKqGYAYAAABUDcGMFTRYaRIAAEBxEMwAAACAqiGYAQAAAFVDMAMAAACqhmAGAAAAVA3BjBXqhVaR750AAAAAmyCYsUJ4cCA91CHStlcaAAAA1BvMzJkzh6KjoykwMJDi4+MpIyPD5G0/++wz6t69O9WoUUOcEhMTzd7e2RKa1HL1JgAAAIAzg5mlS5dSUlISTZs2jXbs2EHt2rWjfv36UW5urtHbb9iwgYYNG0br16+n9PR0ioqKor59+9KZM2fk3lQAAABQIdmDmVmzZtGYMWNo1KhRFBMTQ/PmzaOgoCBasGCB0dsvWrSIxo8fT+3bt6eWLVvS559/TuXl5ZSWlib3pgIAAIAKyRrMlJSUUGZmppgq0j2ht7c4z6MuUly9epVKS0upZs2aRq8vLi6mwsJCgxMAAAB4DlmDmfz8fCorK6Pw8HCDy/l8dna2pMd45ZVXqF69egYBkb6UlBQKCQnRnXhaCgAAADyHoquZZs6cSUuWLKEff/xRJA8bk5ycTAUFBbrTqVOnnL6dAAAA4Dq+cj54WFgY+fj4UE5OjsHlfD4iIsLsfd9//30RzKxdu5batm1r8nYBAQHiBAAAAJ5J1pEZf39/io2NNUje1SbzJiQkmLzfu+++SzNmzKDU1FSKi4sjJaldHYETAACAR00zcVk294756quv6ODBgzRu3DgqKioS1U1sxIgRYqpI65133qEpU6aIaifuTcO5NXy6cuUKKUHP5rWpb4xhDhAAAAC46TQTGzJkCOXl5dHUqVNFUMIl1zziok0KzsrKEhVOWnPnzhVVUP/4xz8MHof71Lzxxhvkal5eXjR/RBxFT17l6k0BAAAAPjZrNBqNO70SXJrNVU2cDBwcHCzb8yCYAQAAT3Zi5kDFHL8VXc0EAAAAYAmCGQAAAFA1BDMAAACgaghmAAAAQNUQzAAAAICqIZgBAAAAVUMwAwAAAKqGYAYAAABUDcGMjR7qGOnYdwIAAABsgmDGRg1rVrX1rgAAAOBACGZspCG3WgUCAABAtRDMAAAAgKohmAEAAABVQzADAAAAqoZgBgAAAFQNwQwAAACoGoIZG2lQzAQAAKAICGYAAABA1RDMAAAAgKohmAEAAABVQzADAAAAqoZgxkZ3Ngtz7DsBAAAANkEwY6NO0TVp+fiu9GLf5rY+BAAAADgAghk7dGxQg8b1auqI9wEAAABshGDGTj7eXvY+BAAAANgBwQwAAACoGoIZAAAAUDUEMwAAAKBqCGYAAABA1RDMAAAAgKohmJFBevLdcjwsAAAAGIFgRgZ1Q6rI8bAAAABgBIIZmcx8qI1cDw0AAAB6EMzIJC66hlwPDQAAAM4OZubMmUPR0dEUGBhI8fHxlJGRYfK2+/fvp8GDB4vbe3l50ezZs52xiQAAAKBSsgczS5cupaSkJJo2bRrt2LGD2rVrR/369aPc3Fyjt7969So1btyYZs6cSREREaQGVfx8Kl2m0bhkUwAAADyO7MHMrFmzaMyYMTRq1CiKiYmhefPmUVBQEC1YsMDo7Tt16kTvvfceDR06lAICAkgNtr+eSH1jwl29GQAAAB5J1mCmpKSEMjMzKTEx8fYTenuL8+np6Q55juLiYiosLDQ4OVu1AF9qUDPI6c8LAAAAMgcz+fn5VFZWRuHhhqMWfD47O9shz5GSkkIhISG6U1RUlEMeFwAAANRB9dVMycnJVFBQoDudOnXK1ZsEAAAATuQr54OHhYWRj48P5eTkGFzO5x2V3Mt5NUrIrakaIOtLCQAAAK4YmfH396fY2FhKS0vTXVZeXi7OJyQkkDsZ06MxdWlck2YMusPVmwIAAOBRZB9O4LLskSNHUlxcHHXu3Fn0jSkqKhLVTWzEiBEUGRkpcl+0ScMHDhzQ/f/MmTO0a9cuqlatGjVt2pSUipOAl4w1HaAN7liffthxWvy/ZUR1OpR92YlbBwAA4L5kD2aGDBlCeXl5NHXqVJH02759e0pNTdUlBWdlZYkKJ62zZ89Shw4ddOfff/99cerZsydt2LCB3AEHM3/nXqGycjSjAQAAsJdTEj0mTJggTsZUDFC486/GzTrO/TCuKy3eluXqzQAAAHBLqq9mUoOODUJdvQkAAABuC8GMk9zbrq7B+bta1DF524+G3Z5mAwAAAPMQzMik4kRZxeClXmig0fuFBwfQfe3qybVZAAAAbgfBjBPw6t9S/fHy3bJuCwAAgLtBMOMCNar6k6nwxt8XbwkAAIA1cOR0ojmPdKTEVnXo+d7Nnfm0AAAAbg09+J1oYNu64iRX074rxTdkeWwAAAAlw8iMm3jrgdau3gQAAACXQDCjYi3Cq+v+Xy+0iku3BQAAwFUQzCjA412jxb+Pdmmgu+ylfi0s3i/QT9lvX90Q4+XnAAAAjoScGQWYcm8MPdghku6oF6y77Jm7mlJxaRl9uO4IqVXNqv50ruC6qzcDAADcnLJ/2nsIH28vahcVSr4+hm9HtUDzsWa3pmG6/7vbelYAAABSIZhRsEe7NKQujWuavP6hjvUd9lwfP+L4JRSs6BUIAABgMwQzChbk70tLxiaYvN7by7Yuw8afy8eu+wMAALgKghkAAABQNSQAy8TSSEfF/BhrtK0fIprkNQqravNjAAAAuAsEMzKpXyOIXuzbnKoH+hm9fnyvJrTuUC4N7hhp9WN/MryjeHxzfL29aGLvZmKtp5RfDln9HAAAAGqBYEZGE+5uZvK6WtUCaP2LveR8enq2983nd3Qw88fLd1H3d9dbvB0KrAAAwBmQMwOCl8l1vCvz9UGZkr3mPdpRMZ+8+EamK+YAANQAwQxYLdjE1JlUnz4WW6nkvFZVf495J+rXqEJ3taxDSvFI/O3O0wAAaoRgxk3J2UKvaoAvrXz2Tlr9XHezt2tYyzCvp139ENr3Zj/qd0dEpZLzpU+ZLkE39XhqFuDrQ+8MbkPTB91BMXVvd34GAADrIZgBk17p35IiQ6tQxmu9dZfNuLU6d+vIEIrRW35B39KxXWh4fAOa1LfC+lJeXqIKy5imdapZfCdWPNPNoe/Wu4Pbiu15qkdjq+7XMuL2Ap/2GNKpAY1IiKbVE80HhUry3N1NXb0JAACVIJgBk8b1akKbJ99NdarfXjCyc7Tl/Ir4xrXoXw+2oeomAhdbhQY5diqqa9NatGdaXxrU3rqKsuoWlpmwJDxYvgU4n7mrCckpqW8L+s/Q9rI+BwCAtRDMQCXdmtaixU/GW/3KtI4Mpn8PaaeqV9Rbv42yk8weUjkY6NG8tkMee/Sd1o0yAQC4AwQzbsKRC00uerILddVbxFKq6YNa04MdHLdelNzkLh3vExNu9PKompVzf8KqeU4CtDF1Q+QbrQIA94dgRgW6N7M+sOjfOsLo5V+P7kxvP9jGAVvl/jjvxxpV/Fy/vpVaV0+/w0T+FQCAFAhmVOCrUZ1FozprBJvI6+jerLZdpbgRMuZ72GL+Y7GyPfaUe2Mk9eThKq23HmhNv73QgzyBt5OWQ6/pQeX6AGAfBDMqyeuo4eIvdg6m1ib1oJAg+3rM6Hu8a7TdjyHnOESgxJEWLrN+tEtDo9NHUt3Xtp7u/4M7KnuqjqfP5Ew14imn0Xc2op/GO7Z6DQDcF4IZN+Vvx0KWxvCBummdyiXJUTWr2PyYAX6u/fiZKhN3hV4tbicA83paSsZBXurz8o1CRYQEilGxBm7UVwgA5KXsb02we10myWz8pa2fJ1LxIZScvdEmMsTkaBdPGa2b1NPu57AmfcXLwtTNqG7Wj2Lx6IYUHaJqWP3YAABKgmDGTVQ8GIZVC3DK897d0njFjqPXg3J0jsy9beuavK59VCg1rm25iZ8zNbBhCstSMNW4dlXa+NJdVC/UeXlQvFr8/e1uT6kBADgCghmwy/OJVo4A2anvHRG0c0ofUhSVrrsZUsXP6VM5L/dvSR8O6+DU5wQA94dgBpySJCt1+QQpXJ0M/WxvqS39NYqr/nJUDGZv/GZsaYo7b/U2GpHQ0M5HBwBPg2DGQy0eY32HX3tZqoDh5ROUjBe6nDu8oyhv3yFxdGjpU10Mqras7V1jLf2ZJY2Tspa6Nqll9X3aRYUaJI/zlNiXozqJijn9xouPdakc2EQjMRgAKkAwoxIBehUujlijqGuTMIePGphLYuXtf2dwW5ID57jIrUntqvT7S3fRPW3qWtUDpWGtqvTG/Xfozoc6sLTdXpaSjs1tr36YNO2+2/tnDz8fb6MVcwAAighm5syZQ9HR0RQYGEjx8fGUkZFh9vbLli2jli1bitu3adOGVq9eTZ6Ov+i5Kduq5+60uqSYu/5yE71Phnc0uHzsrdWi7zHRLdhe+gmo217tTc3D5TlQff90gljZe1hneUc9pOjYQP7AytHVVFKCGldR8KYBgCcFM0uXLqWkpCSaNm0a7dixg9q1a0f9+vWj3Nxco7ffsmULDRs2jEaPHk07d+6kBx54QJz27dtHno6DgTvqhVjdxp6nRXZP60sDbo0q6Jf7/vp8D/rICQmZtlQuSeXr431rZW/p0yr2NLgzZ8nYBFkeV//tlbJyuTV87OyAh4ADANw+mJk1axaNGTOGRo0aRTExMTRv3jwKCgqiBQsWGL39f/7zH+rfvz+99NJL1KpVK5oxYwZ17NiRPv74Y7k31a0Z+/XNl7WIqC6CAbXhERA/Hy+DfA2pIxEv929hdjTK2ONwjocp+q+sfsM7Y48T19CxgYgncsVK5wCgbLIexUpKSigzM5MSExNvP6G3tzifnp5u9D58uf7tGY/kmLp9cXExFRYWGpzA/X3/dFfa92Y/qh5oPgelZ/PaBvktreoG0/heTSVPrXBC6nN3N7U7L4SXg/h8RJxBp1+wzcv9pFW9AYDnkDWYyc/Pp7KyMgoPN2ysxuezs7ON3ocvt+b2KSkpFBISojtFRUU5cA88R00HJBVbW2UysmtD8vX2ooc6Rtr065zXRLKE84syXu1NtuKE1KS+LURPFnvwtFZiTLhD8lOcVaVkL6kVX9aqX8P2JTQAwD2pb36hguTkZCooKNCdTp065epNUiWeRpnUpzn9Z2h7q+/LybfcSdbaKqu6IVXo4Iz+NOth65/TGvrTaOZyi9yBud27r109k6uvJzS2vrzaEh4Re+auJnRHvWDRKC/NzBIRvVrWkdxBWC2+GBnn6k0A8BiyrrQXFhZGPj4+lJOTY3A5n4+IMJ6zwJdbc/uAgABxAhes53TLzeTbygL1FpL09fEyWaUF2tfCi0rLNNSzeRh9m5Hl8MReXo3aVOLzV090pqwLRTR1xX7acvS87nJ7x5Fe6tdSnCx5RGIlGi9CqRZ3SwzQ4KYBbSJo9V7jI/AAlsh6JPH396fY2FhKS0vTXVZeXi7OJyQYr/rgy/Vvz9asWWPy9qBcPFLDizbOfKgNVXXCCtXOHnTpf8fNAPupnjdL3O21efLdooy+3x0RlPm6Yd6YNXg0pGVEdRrSSfqUKycuV+zxwiMrn0sYXaihNyLHjQVdUVGlRDylOOeRjhabRcLNnDJ+rTinDcAWsh9huCx75MiRFBcXR507d6bZs2dTUVGRqG5iI0aMoMjISJH7wiZOnEg9e/akDz74gAYOHEhLliyhP//8k+bPny/3pqoep2MobRblUSMdXN0F9+3Ju1JM4Q5qPsgjXNpRrlp2LBSqHQ05ffGqXdvDAZWUHJ/a1QPEAqBB/r43p5YW76CX+rWweD8pn9VB7SPpv+kndec50MstLCa1GNi2Lv3xdxQt2Y7pb3O0I4ZLxnShdtN/c9K7435CqvhRwbVS8kSyBzNDhgyhvLw8mjp1qkjibd++PaWmpuqSfLOyskSFk1bXrl1p8eLF9Prrr9Orr75KzZo1o59++olat24t96aqXpCfDxWVlLl6MzwGJyGbC2ScEVdytdXuUwU0adluhz+2uUCmYuNGXgBUa/2LvRy2DbENa4h8LA6YyjUaMcKnhmBm8ZPOXy7EHYQoqEM2qIv8Y/9ENGHCBHEyZsOGDZUu++c//ylO4HmU8MuieqBT/ixspj+iwVND+tM85lQMTeyZ/agXWoWm3BvjlNfKWSt7N61TjY7kXnHIY3W9tWimp6kR5EfdmobRyj3nXL0p4GGQfelGOjasIf6t7oT8FLmsntidYvTmzfUXaXQES1Mbk+9pSdFhVR36nO5q9J2N6OE457dCkKs0faSDP2tqWC5CjnL81pHGu5SDc33/dILI6zPX8NOdIJhxI/8e0l4ko/787J2kVpGhVURAoxVTT3pCYHAV+4O4p3sqp/RXm1jMQYO79aBxle7N5BsxiW9kvLvzRBurBNWIA7eHOljfN0ptfp7QjZQuLromzXssliJD1VMBaA8EM270JRJWLYCS72lFjdxoZIEXyJRqwl3NqFvTWvT+P9uRO3ilX0uxuOhrA1qZvZ0n/fK35Ilu5gM/fxlbAXD+1JF/3UPHUwaotpzcEeoEB9KhGf1tvj830lQy/o5pW999Rju63loSpklt88eNBxUepCKYcRPu1gyOA5LXB7ai+jWCrEoeXPRkF/pHbH1ylwRjXlzU1rWIlP6RsDUGM7dfjyWYr54z95J4OahBoyOCy6ialbscNw+vRq7yxn0xVt0+0M9yd25Tqpn5AbN4DBKrHe2jYR1E9SF/d1oa+a8YqCsJghlQJA5Inuze2K4vRTAPAzqVNZBpNfWp91oXDDh75XlLhhppasi5GPrTdve2rSv7dvjqVb5CZZ+PjKPQID+a9bC00ekfxnUVbSCeuauppBFEJY8C45MBisYVJiMSGtILic1le44q/uoKmGxtQKjwgRpFrDHGBwJjnZntfX1NdcBWM17EdeGoziaT9c0tX6Fv6VjzIwLuPAItpTrMGp2ia9LOKX3ooY6WR6d5SpRbH0jx7j/aktIhmAHFmz6oNU1MdEwSpbEEWc4xGtO9Eb3c33KjNyme691MdNR9pb88qztzV2Ve72j2EMevaaX2YwVPNXIVh/5SGlLxj06e1tPHXWnbuKA6xxWjMJx8rzWss/VVajX0VqdnTWpbnhZbm9ST4mVYF8wSrlpUA1s6Y3tJHD3RX7POEmNVi9xlXEkQzAAQ0WsDY2h8r6YOeS2S+jSnA2/2oxYy/bFzt9RVz3WnBxSekCcXjYQqjraRtiVoThkYU6mDr7ULqJpaVNUenaKl/YK2R9UAH9E7qFmdavRCn+YOPVjVc0IS9PB4aet7aftZeaJaVf2pS+OaDpn2VNpriGAGQAbW/OpRusa3qhz0+/+o1TALC1rqd6DVflnPeKC15OF4UxJb1aHnE5vRgsdtW0n7sxFxBqOKrSPleS+4DcCapJ5iWQ1H5KvxFBKPgnz0SAeS278ebCP5tl4yjF6uVGhLjNp6S6NwRdKSsQn0hIl2D/Vr2Bd0u5L7fOMCeMA0ijGhVfwoulaQSF6tVdXxK8h/PTqexvVqQl/YeCBW0nvRsUEo9TaxmrV2aoenqeIa1qBFt5Yk4OkXTpS0Bw/9P5/YnO5ueXMZF2vc0zqi0uhQl0bOn5qxBU8hce+m2IY1HdIA09RHRmpek5y4WaCS+lRpJfVpQZ2ja9LANnXpRQtrpilt6sgaCGYAVI5Lt9Mm9RJrIsmx+jQfzDn/x96pEkfhwM0edYIDLE5TfT+uq2ydbHmo3xYcEFijXX3n5vpYCk6N9zHRmH2c8b1uBwf3t6tn8jnkzjHiqRmpU8z2juJJScbVn7az9HkKCfKj755OoDnDOzq0OlRpSe0IZsCteUpHXA5iKgYy3ERRqaW+9tAfpegbY/1Ih2M6ytheacMjXZYOjsZyOPWnliQtveCEMlr+yHF5NpdpN5ahWefLekn0xnrvyKFi8vijXRrQt2OkVVxx4n/vVsZH/hyFk3Ej9aaDtCuOO8uMQXeIYGrGIGUt/oxgBsBNccl5neqOn3ZSEle1vTBWwi0VL9HBeQv24AMYd9nlkQBX4umz/z7RmX4a39Xm5o6WgsJJfZqLFg1jut9c3sMRzI2e+FXoZcNBv5L7qzj759pjCdG0Jbk3NZZQreZMCGbAo3jGOI17JPTZw89X3oNP+uTeDn08HtXQnzr4p14Xa1MDPjxl4GWhD4wlHRrY3pZfO73BB3o5D/bP9m4mSrh5RE5/pPURK6qXtDJe7U2pz3enZhVK8EH9EMwAgNupYiI3wFQw27BWEA1qX8+qfAF71xB65q7b+SDcGFLf8PiGdk+V8uO/Z6HZmf4ohNSEbG59P7hjfYuVYRUNaOPYDsGxDWrYtG5UywjnVuVZWlvNGrZWw3kCBDNuwtNGHABspT+I0O+OcEp5qA1teLGXVZ2VHVGJNalPC/phXAItH9+1Ug6M/pSNrc8V4OtD/zTS7Mxe97WtRx883E7kh1iD2+b/b8KdDnsd72snPfh0pUSb8rqMs6UazlPY1hcdAFTHtmRZ9/bpY677pcsBi6kKJf2AS84fKg93iqKMExcs3s5bb4PsmcKzNgBy1mMpQc2q/lRcWkZFJWWkpBW1J/V1TGd0ubnXpwEAzJYcuyNeSZkTRL97yr6kWms5Oyc0tIr9nYgrLlMwuKPpLtL6yeMcODx3d1PRVM+lJfouGIK29n02NcVpzN43+ur+z1VhnBiuDdJ42RJXqh7gS4vHdJG91NxREMwAuLGGtRxfLqskXGnStUmYSBDt3KimSw9wvMq7HLRTMf1bR9DQTlEGi/5Zc6ANq+ZPKQ+1NejXYy5xlzsfc4PBhaM6ifNJfVuI5Q7sUTH/R+OAkQP2cCfLCytK8cvE7pUu075CvO/8GvJaceYM7SQ9l6h6oGFVXJv6IfTXW/fQsbcH0KNdDPOowDxMM7kJ5RYOKku7+rZXb7hSmI0l1vwFzMerf8ZKy53whL48jvhb8auwXMXuaX1lX6uG+wjNHNzWaXseHhxIXzx+M5BRqvkj4ij96HkxqmHL38aMlQcMLmtlZskOHpV6olu0xcotbonAPXFOXbhGtnJUmbsnQTADHmFtUg/6eddZerKHPL+e5catyHecvEhxVi44yPPwsx52/OraaiXXEgryBjKO2Wiu2HK396NagC/10csF44q0FbvOSrovByebj+TTukO5kp/P2f1mnL3kh0bFv2UwzQQeoWmd6mKYPLjCsK5a8K/yN+6/g+5ta38FR8CtOfluTa3/NetId99aIykiWP4VldXGy84DzH+G3g5gf3qmG93bti59OMz4Yo81bjUA7CGhL40zmNtdPwtJv1yZZquHzOQPyYGnrMBxMDID4GHSJvUUv0gf7OCYPANbJd/TSvT80AY1zqpo4YPWom1Zql5Uz9Io0aD2tw/M7aNC6eNHOpoMGlZP7C5GJ7h3jNJxb5n+d0RQI6PrPBEF+ftSt6a1aPOR81Y/tj0jmLZ8Dv89xLrnMxWMYkLqJgQzAB6mfo0gGmJFkqJcOLfAli6u2vVhvko/SZPvub12j1RcDv3Hy3dZXHBSKawdmLF2sVGuTtJv0icnSaNMGvO5JPMeiyWlWTepZ6Xuy99szTI6+sLXXbpWSgmNrVv5nBfadKa6oeoaMUUwAwCqw+vD8MlW9izOV9XfcSsP28tYDoeafqk7I+E8sVU4rT2YI9vjx9QNFj8Q9L06oBW1CK9OvVuFU/GNcoPr3hncliL0lq6wxf3tI2n7iYuiJYFc5j2qvKDRHOTMAABIwJ16ufx7qRP62XD3Xv0kV5CHtrTbnqRfjYnpLg6264Va7snz7N1NRU8XaxYNHd65geivxJ9JuTRW2EKSluCvBAA8hj0jAR0b1JC1MZ/+QVM/B6NaoDxf02oawZHL412jRY5RFyunfByJO+y+kNjcqnJsvi33V4LbMDIDAGCDj4bdTKp94z77GsmBvMwNrPj6eIv1q+yZdnQEe/rKqLia2qEwMuMm8IEGcC7uyHtoRn8KtKJ9PQDIAyMzAOB2nNX8S85Apm39EIvrJ6lNxXWL1NykzVa+PpjgkwNGZgAAFOiHcV0p73Kx2SRSY1Mozl4A0xrRYVXp6Z5NaN7vR532nEp6OZ7o1ojCqqmjJYDaYGQGADz6l7JS8fpPUqph1MZSbyB7P4KhQfZ11k0e0MrOLTD32Nb3RQJpEMwAgOdAsOb2L8XUe2Osbkinjxs5piffTZ7ohVvl4SMS1LdiN4IZAHBrtnQJBvXi1b6/HdvFrsfgrsieqHVkiEhqnz6oNamNbMHMhQsXaPjw4RQcHEyhoaE0evRounLlitn7zJ8/n3r16iXuwz0XLl26JNfmAYCHGKaApRvA9Kruah+B0rjZ3GygSqvzZAtmOJDZv38/rVmzhlauXEkbN26ksWPHmr3P1atXqX///vTqq6/KtVkAAOBii56Mp08fixWjKHILV8kaXMZUl6lhojuS5ZU6ePAgpaam0vbt2ykuLk5c9tFHH9GAAQPo/fffp3r1jC+Y9fzzz4t/N2zYIMdmuTUlZewDgPO9O7gtvfm//fTJcOWvqdOtaZjsIx2fjYijv3IuU3StqmLRRzWa+2gsJX23m17sK32pA08lSzCTnp4uppa0gQxLTEwkb29v2rZtGz344IMOe67i4mJx0iosLHTYY4P61bCzsgFAybz0fsY83CmK/hFb365usu6kT0y4OK3ee47UqlXdYPplYndXb4bnTjNlZ2dTnTp1DC7z9fWlmjVriuscKSUlhUJCQnSnqKgohz4+qNMnwzvS3S3r0It9W7h6U8DFAv1vf81VD/Qjd4ZAxvGU3LcHbAxmJk+eLBJzzZ0OHTpEzpScnEwFBQW606lTp5z6/KBMA9rUpQWPd6IaTkgwBGXjFaj51+2q5+6kKv7qTG4EAAdOM02aNIkef/xxs7dp3LgxRUREUG5ursHlN27cEBVOfJ0jBQQEiJOnc698egDHD9crXWRoFTpz6Rp5OnyXgezBTO3atcXJkoSEBFFWnZmZSbGxN5PR1q1bR+Xl5RQfH2/ThgIAuLNZQ9rRm/87QGO6N3bK87lZRbHHCsPos3w5M61atRIl1mPGjKGMjAzavHkzTZgwgYYOHaqrZDpz5gy1bNlSXK/F+TS7du2iI0eOiPN79+4V53lEBwDAndWvESQqcDo3qunqTQEVeevB1tStaS36fMTtghtPJFufmUWLFolgpXfv3qIk+8477xRN8bRKS0vp8OHDoreM1rx586hDhw4iCGI9evQQ53/++We5NhMA9ODHurogOVV+Vf2NT2Dc0/pmysRTPZ0zkmauW/GiJ7tQYkw4eTLZOvJw5dLixYtNXh8dHV2pn8Abb7whTgAAAK6cTpsx6A763+5zNNZEsPLxIx3pzMVr1KBWkH0bCA6B9oIAAAAVPJYQLU6m+Hh7IZBRECw0CQAAIJO6IbeXbPDBvKBsMDIDAAAg48KNu6f2JR8fLzQ1lBGCGQAAlWoeXs3VmwAShAS5d+dpJcA0EwC4na63FjJ091H9u1rUoZSH2tCKZ7qRu2gTGSL+DfTD4Qmkw8gMALidkQkNqUaQn9v3bOElZIZ1bmDVfZrUrkpH84ronjaO7cbuKB8N60Bzfz9Kw+MbunpTQEUQzACA2/H18aaHOtZ39WYo0rKnu9KWo/liRWklqhMcSNPuu8PVmwEqg2AGAMCD1KzqT/e2vdmJHcBdYFLSTbSMqO7qTQAAAHAJjMyo3J43+tL1kjIKDfJ39aYAACgOEok9A4IZlQsO9BMnAACorFeLOpTYKlxXJQXuCcEMANyGlSbBzfCyA5+P9OwVpT0BcmYAAABA1RDMAAAAgKohmAEAAABVQzADAAAAqoZgBgAAAFQNwQwAAACoGoIZAAAAUDUEMwAAAKBqCGYAAABA1RDMAAAAgKohmAEAAABVQzADADpdmtS6+cXghRcFANQDC00CgM4zdzWhOtUDqGfz2nhVAEA1EMwAgE6Arw892qUhXhEAUBVMMwEAAICqIZgBAAAAVUMwAwAAAKqGYAYAAABUDcEMAAAAqBqCGQAAAFA1BDMAAACgaghmAAAAQNUQzAAAAICqyRrMXLhwgYYPH07BwcEUGhpKo0ePpitXrpi9/bPPPkstWrSgKlWqUIMGDei5556jgoICOTcTAAAAVEzWYIYDmf3799OaNWto5cqVtHHjRho7dqzJ2589e1ac3n//fdq3bx8tXLiQUlNTRRAEAAAAYIyXRqPRkAwOHjxIMTExtH37doqLixOXcWAyYMAAOn36NNWrV0/S4yxbtoweffRRKioqIl9fy0tJFRYWUkhIiBjN4REhAAAAUD57jt+yjcykp6eLqSVtIMMSExPJ29ubtm3bJvlxtDtlKpApLi4WL4D+CQAAADyHbKtmZ2dnU506dQyfzNeXatasKa6TIj8/n2bMmGF2aiolJYXefPPNSpcjqAEAAFAP7XHblgkjq4OZyZMn0zvvvGNxiskROzVw4EAxVfXGG2+YvF1ycjIlJSXpzp85c0bcJyoqyu5tAAAAAOe6fPmymG6SNZiZNGkSPf7442Zv07hxY4qIiKDc3FyDy2/cuCEqlvg6SzvSv39/ql69Ov3444/k5+dn8rYBAQHipFWtWjU6deqUuK+Xlxc5EgdYHCTx47tjPo67758n7CP2T/3wHqqbu79/cu4jj8jw8V9qTq1dwUzt2rXFyZKEhAS6dOkSZWZmUmxsrLhs3bp1VF5eTvHx8WZfpH79+okA5eeff6bAwECrto9zcurXr09y4jfPXT+knrB/nrCP2D/1w3uobu7+/sm1j9aOyMieANyqVSsxujJmzBjKyMigzZs304QJE2jo0KG6qIunhFq2bCmu1wYyffv2FZVLX3zxhTjP+TV8Kisrk2tTAQAAQMVkSwBmixYtEgFM7969xYjJ4MGD6cMPP9RdX1paSocPH6arV6+K8zt27NBVOjVt2tTgsY4fP07R0dFybi4AAACokKzBDFcuLV682OT1HJzoZy336tXLpixmZ+Gpr2nTphnk6LgTd98/T9hH7J/64T1UN3d//5S6j7I1zQMAAABwBiw0CQAAAKqGYAYAAABUDcEMAAAAqBqCGQAAAFA1BDMAAACgaghmJJozZ44oJeeOxNzBWNvoz5V4kc1OnTqJpRt4Uc8HHnhA9O3Rx+XuvKyD/unpp582uE1WVpZYBysoKEg8zksvvSSWntC3YcMG6tixoyjF4x5ACxcudMprxOtyVdx+brSodf36dXrmmWeoVq1aYikL7mWUk5Ojmv3jx6u4f3zifVLr+7dx40a67777RHNM3t6ffvrJ4HouoJw6dSrVrVuXqlSpQomJifT3338b3IaXPRk+fLjoLhoaGkqjR4+mK1euGNxmz5491L17d7G93Fr93XffrbQty5YtE58Xvk2bNm1o9erVVm+LNfvHvbNeeeUV8VxVq1YVtxkxYgSdPXvW4vs+c+ZMxe8f4+VsKm47N0hVy/snZR+N/U3y6b333lPFe5gi4digpO9OKdtiEZdmg3lLlizR+Pv7axYsWKDZv3+/ZsyYMZrQ0FBNTk6OS1+6fv36ab788kvNvn37NLt27dIMGDBA06BBA82VK1d0t+nZs6fY3nPnzulOBQUFuutv3Lihad26tSYxMVGzc+dOzerVqzVhYWGa5ORk3W2OHTumCQoK0iQlJWkOHDig+eijjzQ+Pj6a1NRU2V+jadOmae644w6D7c/Ly9Nd//TTT2uioqI0aWlpmj///FPTpUsXTdeuXVWzf7m5uQb7tmbNGm6VoFm/fr1q3z/ehtdee02zfPlysS8//vijwfUzZ87UhISEaH766SfN7t27Nffff7+mUaNGmmvXrulu079/f027du00W7du1fzxxx+apk2baoYNG6a7nl+D8PBwzfDhw8Xn/9tvv9VUqVJF8+mnn+pus3nzZrGf7777rtjv119/XePn56fZu3evVdtizf5dunRJvBdLly7VHDp0SJOenq7p3LmzJjY21uAxGjZsqJk+fbrB+6r/d6vU/WMjR44U74/+tl+4cMHgNkp+/6Tso/6+8Yn/Lry8vDRHjx5VxXvYT8KxQUnfnZa2RQoEMxLwl9EzzzyjO19WVqapV6+eJiUlRaMkfGDkP8zff/9ddxkfDCdOnGjyPvwB9fb21mRnZ+sumzt3riY4OFhTXFwszr/88ssioNA3ZMgQ8Qcj92vEwQx/KRrDBw7+w1+2bJnusoMHD4rXgA8iati/ivi9atKkiaa8vNwt3r+KBwrer4iICM17771n8D4GBASIL3vGX4p8v+3bt+tu88svv4iDyZkzZ8T5Tz75RFOjRg3dPrJXXnlF06JFC935hx9+WDNw4ECD7YmPj9c89dRTkrfF2v0zJiMjQ9zu5MmTBgfCf//73ybvo+T942Bm0KBBJu+jpvfP1D5WxPt79913G1ymlvfQ2LFBSd+dUrZFCkwzWVBSUiIWy+ShPS1emoHPp6enk5IUFBToOi9XXFYiLCyMWrduTcnJybrlIxjvAw9thoeH6y7jhT55Xaz9+/frbqO//9rbaPdf7teIh1R5OJhXY+ehax76ZPycPKyv/7w8XNugQQPd86ph/7T4eb755ht64oknDFZ8V/v7V3FZEl5rTf+5eGE5HnrWf894aiIuLk53G749b5N2uRO+TY8ePcjf399gn3go/eLFi5L2W8q2OOrvkt9P3id9PCXBw+odOnQQ0xf6w/dK3z+eWuBphxYtWtC4cePo/PnzBtvuTu8fT3esWrVKTJVVpJb3sKDCsUFJ351StsXlyxm4g/z8fLHIpf4byvj8oUOHSCl4NfLnn3+eunXrJg56Wo888gg1bNhQBAM8f8vz+fzHtHz5cnE9/6EY2zftdeZuwx/qa9euiT9MuV4j/qPlOVj+0jx37hy9+eabYg563759Yrv4i6LiQYKf19K2K2X/9PG8Pa80zzkJ7vL+VaTdJmPPpb+9fKDU5+vrK76I9W/TqFGjSo+hva5GjRom91v/MSxti704F4Dfs2HDhhmsLvzcc8+JPAPepy1btogglT/fs2bNUvz+cX7MQw89JLbv6NGj9Oqrr9I999wjDjw+Pj5u9f6xr776SuSe8D7rU8t7WG7k2KCk704p2yIFghk3wclTfIDftGmTweVjx47V/Z+jbE4i44U/+UuoSZMmpHT8JanVtm1bEdzwwf27774TyXDuhFeK5/3VrirvDu+fJ+Nfmw8//LBI4Jw7d67BdUlJSQafa/4yf+qpp0TippLWuzFm6NChBp9J3n7+LPJoDX823c2CBQvEiDAnr6rxPXzGxLHB3WCayQIe3udfGxUzq/l8REQEKQGvTL5y5Upav3491a9f3+xtORhgR44cEf/yPhjbN+115m7DvzQ5oHDma8TRe/PmzcX282PzMCaPZph6XrXs38mTJ2nt2rX05JNPuvX7p308c8/F/+bm5hpcz8P3XCHjiPdV/3pL22JvIMPv65o1awxGZUy9r7yPJ06cUMX+6ePpX/4M6X8m1f7+af3xxx9iJNTS36VS38MJJo4NSvrulLItUiCYsYCj7djYWEpLSzMYtuPzCQkJ5Er8i48/rD/++COtW7eu0pCmMbt27RL/8i98xvuwd+9egy8f7ZdvTEyM7jb6+6+9jXb/nfkacXknj0rw9vNz+vn5GTwvf/FwTo32edWyf19++aUYmucySHd+//gzyl9Q+s/FQ9KcS6H/nvEXG8+la/Hnm7dJG8zxbbi8loMG/X3i6Ugevpey31K2xZ5AhnO9OEDlnApL+H3lXALt9IyS96+i06dPi5wZ/c+kmt+/iqOl/LfRrl07Vb2HGgvHBiV9d0rZFkkkpwp7MC4t4+zxhQsXikz9sWPHitIy/SxvVxg3bpwo2duwYYNBeeDVq1fF9UeOHBGlg1zqdvz4cc2KFSs0jRs31vTo0aNS+V3fvn1FCR+X1NWuXdto+d1LL70kssznzJljtPxOjtdo0qRJYv94+7mMkcsEuTyQs/O1JX1ccrhu3TqxnwkJCeKklv3TZvfzPnClgz61vn+XL18WpZx84q+YWbNmif9rq3m41JQfm/dnz549olLEWGl2hw4dNNu2bdNs2rRJ06xZM4PSXq6A4LLXxx57TJSf8vbzPlYse/X19dW8//77Yr+5Ms5Y2aulbbFm/0pKSkTpbP369cX7of93qa0A2bJli6iC4eu51Pebb74R79mIESMUv3983YsvviiqTPgzuXbtWk3Hjh3F+3P9+nVVvH+W9lG/tJq3iSt4KlL6ezjOwrFBad+dlrZFCgQzEnH9PL/YXC/PpWbcP8HV+I/Q2In7C7CsrCxx4KtZs6b4MHGvB/7Q6fcpYSdOnNDcc889ogcCBwocQJSWlhrchvuetG/fXuw/H1C1zyH3a8RlfnXr1hWPGRkZKc7zQV6L/6DHjx8vSiD5j+rBBx8Uf7Rq2T/266+/ivft8OHDBper9f3j5zL2ueSSXm256ZQpU8QXPe9X7969K+37+fPnxcGvWrVqohR01KhR4gCkj3tu3HnnneIx+LPBX/oVfffdd5rmzZuLfeIS0lWrVhlcL2VbrNk/PsCb+rvU9g7KzMwU5bd8sAkMDNS0atVK8/bbbxsEA0rdPz4Y8sGND2p80OXyZO4bUjHoVfL7Z2kftTjo4L8pDkoqUvp7SBaODUr77pSyLZZ43dpxAAAAAFVCzgwAAACoGoIZAAAAUDUEMwAAAKBqCGYAAABA1RDMAAAAgKohmAEAAABVQzADAAAAqoZgBgAAAFQNwQwAAACoGoIZAAAAUDUEMwAAAEBq9n8SWQq5/8iW5QAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.plot(lossi);" ] }, { "cell_type": "code", "execution_count": 65, "id": "3e327893", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train_loss=1.4785451889038086\n", "val_loss=1.6728813648223877\n" ] } ], "source": [ "train_loss = nn.training_loss(datasets)\n", "val_loss = nn.test_loss(datasets)\n", "print(f\"{train_loss=}\")\n", "print(f\"{val_loss=}\")" ] }, { "cell_type": "markdown", "id": "33dddb0b", "metadata": {}, "source": [ "### Squashing via tanh" ] }, { "cell_type": "markdown", "id": "594a4125", "metadata": {}, "source": [ "Il y a potentiellement un problème avec la couche cachée `h` et sa fonction d'activation `tanh` qui \"squashe\" les valeurs.\n", "\n", "```python\n", "self.hpreact = self.embcat @ self.W1 + self.b1 # hidden layer pre-activation\n", "self.h = torch.tanh(self.hpreact) # hidden layer\n", "```" ] }, { "cell_type": "code", "execution_count": 66, "id": "29640a29", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 1: 3.7308\n" ] }, { "data": { "text/plain": [ "tensor([[-1.0000, -0.9998, -0.9985, ..., -0.9964, 1.0000, 0.9995],\n", " [-1.0000, 0.9432, -0.9935, ..., 0.9990, 0.9751, -0.9991],\n", " [-1.0000, -0.9998, -0.9985, ..., -0.9964, 1.0000, 0.9995],\n", " ...,\n", " [-1.0000, -0.9995, 0.9973, ..., -0.9959, 0.9997, -0.9979],\n", " [-1.0000, 1.0000, -0.9998, ..., -0.3458, 1.0000, 1.0000],\n", " [ 0.9631, -0.9770, 1.0000, ..., -0.8631, 0.9987, -0.8641]],\n", " grad_fn=)" ] }, "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, 1, mini_batch_size)\n", "nn.h" ] }, { "cell_type": "markdown", "id": "a8c69dc9", "metadata": {}, "source": [ "On peut voir qu'il y a beaucoup de valeurs à 1 ou -1. C'est `tanh` qui \"squashe\" des valeurs. Est-elle trop active?" ] }, { "cell_type": "code", "execution_count": 69, "id": "7dde5e09", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIZZJREFUeJzt3QnUVNVhB/DLIosLICpbBdQYFRVRsUFatSZyQCTGHO1piNYlIdpYsFUMCq1BY9KgYNTUEs1GSM7RuOSoScUgijEmBjdaqhLlqIWCVSAurCrr67n3nJl+8/GxfPBtd77f75zHMPPuzLw79715/++9e9+0KYqiCAAAGWnb3AsAAFBfAgwAkB0BBgDIjgADAGRHgAEAsiPAAADZEWAAgOwIMABAdtqHKrV169bw9ttvh/322y+0adOmuRcHANgF8fq6a9euDX369Alt27ZtfQEmhpe+ffs292IAALth2bJl4eCDD259ASYeeSl9AF26dGnuxQEAdsGaNWvSAYjSfrzVBZjSaaMYXgQYAMjLzrp/6MQLAGRHgAEAsiPAAADZEWAAgOwIMABAdgQYACA7AgwAkB0BBgDIjgADAGRHgAEAsiPAAADZEWAAgOwIMABAdgQYACA77Zt7AQCAluWQibN2WmbJTaNCc3IEBgDIjgADAGTHKaQqPbQGANXMERgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAoLoDzJQpU8Kf//mfh/322y/06NEjfP7znw+LFi2qKPPxxx+HsWPHhgMOOCDsu+++4bzzzgsrVqyoKLN06dIwatSosPfee6fXmTBhQti8eXNFmaeeeiqceOKJoWPHjuHwww8PM2fO3JN6AgBVpF4B5re//W0KJ88++2x4/PHHw6ZNm8Lw4cPD+vXry2Wuuuqq8O///u/hgQceSOXffvvtcO6555bnb9myJYWXjRs3hj/84Q/hpz/9aQonkydPLpdZvHhxKvPpT386LFiwIFx55ZXhK1/5Snjssccaqt4AQMbaFEVR7O6T//SnP6UjKDGonHbaaWH16tXhoIMOCvfcc0/467/+61TmtddeCwMGDAjz5s0LJ598cvj1r38dPvvZz6Zg07Nnz1TmrrvuCtdee216vQ4dOqT/z5o1K7zyyivl9xo9enRYtWpVmD179i4t25o1a0LXrl3TMnXp0iU0pEMmztppmSU3jWrQ9wSApnJIM+7ndnX/vUd9YOKLR927d0+38+fPT0dlhg0bVi5z1FFHhX79+qUAE8XbgQMHlsNLNGLEiLTACxcuLJep+RqlMqXXqMuGDRvSa9ScAIDqtNsBZuvWrenUzl/+5V+GY489Nj22fPnydASlW7duFWVjWInzSmVqhpfS/NK8HZWJoeSjjz7abv+cmNhKU9++fXe3agBAtQaY2BcmnuK59957Q0swadKkdESoNC1btqy5FwkAaCTtd+dJ48aNC4888kh4+umnw8EHH1x+vFevXqlzbuyrUvMoTByFFOeVyjz//PMVr1capVSzTO2RS/F+PBfWuXPnOpcpjlaKEwBQ/ep1BCb2943h5aGHHgpPPvlkOPTQQyvmDx48OOy1115h7ty55cfiMOs4bHro0KHpfrx9+eWXw8qVK8tl4oimGE6OPvrocpmar1EqU3oNAKB1a1/f00ZxhNEvf/nLdC2YUp+V2OckHhmJt2PGjAnjx49PHXtjKLniiitS8IgjkKI47DoGlQsvvDBMnTo1vcZ1112XXrt0BOWrX/1q+Ld/+7dwzTXXhC9/+cspLN1///1pZBIAQL2OwNx5552pf8npp58eevfuXZ7uu+++cpnbbrstDZOOF7CLQ6vj6aAHH3ywPL9du3bp9FO8jcHmb//2b8NFF10UbrzxxnKZeGQnhpV41GXQoEHhO9/5TvjRj36URiIBAOzRdWBaMteBAYDdU/XXgQEAaA4CDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAoPoDzNNPPx3OPvvs0KdPn9CmTZvw8MMPV8y/5JJL0uM1pzPPPLOizPvvvx8uuOCC0KVLl9CtW7cwZsyYsG7duooyL730Ujj11FNDp06dQt++fcPUqVN3t44AQGsPMOvXrw+DBg0K06dP326ZGFjeeeed8vTzn/+8Yn4MLwsXLgyPP/54eOSRR1Iouuyyy8rz16xZE4YPHx769+8f5s+fH6ZNmxZuuOGG8IMf/KC+iwsAVKH29X3CyJEj07QjHTt2DL169apz3quvvhpmz54dXnjhhXDSSSelx+64445w1llnhVtuuSUd2bn77rvDxo0bw4wZM0KHDh3CMcccExYsWBBuvfXWiqADALROjdIH5qmnngo9evQIRx55ZLj88svDe++9V543b968dNqoFF6iYcOGhbZt24bnnnuuXOa0005L4aVkxIgRYdGiReGDDz6o8z03bNiQjtzUnACA6tTgASaePvrZz34W5s6dG26++ebw29/+Nh2x2bJlS5q/fPnyFG5qat++fejevXuaVyrTs2fPijKl+6UytU2ZMiV07dq1PMV+MwBAdar3KaSdGT16dPn/AwcODMcdd1z4xCc+kY7KnHHGGaGxTJo0KYwfP758Px6BEWIAoDo1+jDqww47LBx44IHhjTfeSPdj35iVK1dWlNm8eXMamVTqNxNvV6xYUVGmdH97fWtiv5s4qqnmBABUp0YPMG+99VbqA9O7d+90f+jQoWHVqlVpdFHJk08+GbZu3RqGDBlSLhNHJm3atKlcJo5Yin1q9t9//8ZeZACg2gJMvF5LHBEUp2jx4sXp/0uXLk3zJkyYEJ599tmwZMmS1A/mnHPOCYcffnjqhBsNGDAg9ZO59NJLw/PPPx+eeeaZMG7cuHTqKY5Ais4///zUgTdeHyYOt77vvvvCd7/73YpTRABA61XvAPPiiy+GE044IU1RDBXx/5MnTw7t2rVLF6D73Oc+F4444ogUQAYPHhx+97vfpVM8JXGY9FFHHZX6xMTh06ecckrFNV5iJ9w5c+akcBSff/XVV6fXN4QaANitTrynn356KIpiu/Mfe+yxnb5GHHF0zz337LBM7Pwbgw8AQG1+CwkAyI4AAwBkR4ABALIjwAAA2RFgAIDsCDAAQHYEGAAgOwIMAJAdAQYAyI4AAwBkR4ABALIjwAAA2RFgAIDsCDAAQHYEGAAgOwIMAJAdAQYAyI4AAwBkR4ABALIjwAAA2RFgAIDsCDAAQHYEGAAgOwIMAJAdAQYAyI4AAwBkR4ABALIjwAAA2RFgAIDsCDAAQHYEGAAgOwIMAJAdAQYAyI4AAwBkR4ABALIjwAAA2RFgAIDsCDAAQHYEGAAgOwIMAJAdAQYAyI4AAwBkR4ABALIjwAAA2RFgAIDsCDAAQHYEGAAgOwIMAJAdAQYAyI4AAwBkR4ABALIjwAAA2RFgAIDsCDAAQHYEGAAgOwIMAJAdAQYAyI4AAwBkR4ABALIjwAAA2RFgAIDsCDAAQHYEGAAgOwIMAFD9Aebpp58OZ599dujTp09o06ZNePjhhyvmF0URJk+eHHr37h06d+4chg0bFl5//fWKMu+//3644IILQpcuXUK3bt3CmDFjwrp16yrKvPTSS+HUU08NnTp1Cn379g1Tp07d3ToCAK09wKxfvz4MGjQoTJ8+vc75MWj867/+a7jrrrvCc889F/bZZ58wYsSI8PHHH5fLxPCycOHC8Pjjj4dHHnkkhaLLLrusPH/NmjVh+PDhoX///mH+/Plh2rRp4YYbbgg/+MEPdreeAEAVaV/fJ4wcOTJNdYlHX26//fZw3XXXhXPOOSc99rOf/Sz07NkzHakZPXp0ePXVV8Ps2bPDCy+8EE466aRU5o477ghnnXVWuOWWW9KRnbvvvjts3LgxzJgxI3To0CEcc8wxYcGCBeHWW2+tCDoAQOvUoH1gFi9eHJYvX55OG5V07do1DBkyJMybNy/dj7fxtFEpvESxfNu2bdMRm1KZ0047LYWXkngUZ9GiReGDDz5oyEUGAFrDEZgdieElikdcaor3S/PibY8ePSoXon370L1794oyhx566DavUZq3//77b/PeGzZsSFPN01AAQHWqmlFIU6ZMSUd7SlPs+AsAVKcGDTC9evVKtytWrKh4PN4vzYu3K1eurJi/efPmNDKpZpm6XqPme9Q2adKksHr16vK0bNmyBqwZAFC1ASae9okBY+7cuRWncmLflqFDh6b78XbVqlVpdFHJk08+GbZu3Zr6ypTKxJFJmzZtKpeJI5aOPPLIOk8fRR07dkzDsmtOAEB1qneAiddriSOC4lTquBv/v3Tp0nRdmCuvvDJ861vfCr/61a/Cyy+/HC666KI0sujzn/98Kj9gwIBw5plnhksvvTQ8//zz4Zlnngnjxo1LI5Riuej8889PHXjj9WHicOv77rsvfPe73w3jx49v6PoDAK2hE++LL74YPv3pT5fvl0LFxRdfHGbOnBmuueaadK2YONw5Hmk55ZRT0rDpeEG6kjhMOoaWM844I40+Ou+889K1Y0piH5Y5c+aEsWPHhsGDB4cDDzwwXRzPEGoAIGpTxIu3VKF46ioGodgfpqFPJx0ycdZOyyy5aVSDvicANJVDmnE/t6v776oZhQQAtB4CDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsCDACQHQEGAMiOAAMAZEeAAQCyI8AAANkRYACA7AgwAEB2BBgAIDsNHmBuuOGG0KZNm4rpqKOOKs//+OOPw9ixY8MBBxwQ9t1333DeeeeFFStWVLzG0qVLw6hRo8Lee+8devToESZMmBA2b97c0IsKAGSqfWO86DHHHBOeeOKJ/3+T9v//NldddVWYNWtWeOCBB0LXrl3DuHHjwrnnnhueeeaZNH/Lli0pvPTq1Sv84Q9/CO+880646KKLwl577RW+/e1vN8biAgCZaZQAEwNLDCC1rV69Ovz4xz8O99xzT/jMZz6THvvJT34SBgwYEJ599tlw8sknhzlz5oQ//vGPKQD17NkzHH/88eGb3/xmuPbaa9PRnQ4dOjTGIgMArb0PzOuvvx769OkTDjvssHDBBRekU0LR/Pnzw6ZNm8KwYcPKZePppX79+oV58+al+/F24MCBKbyUjBgxIqxZsyYsXLhwu++5YcOGVKbmBABUpwYPMEOGDAkzZ84Ms2fPDnfeeWdYvHhxOPXUU8PatWvD8uXL0xGUbt26VTwnhpU4L4q3NcNLaX5p3vZMmTIlnZIqTX379m3oqgEA1XoKaeTIkeX/H3fccSnQ9O/fP9x///2hc+fOobFMmjQpjB8/vnw/HoERYgCgOjX6MOp4tOWII44Ib7zxRuoXs3HjxrBq1aqKMnEUUqnPTLytPSqpdL+ufjUlHTt2DF26dKmYAIDq1OgBZt26deHNN98MvXv3DoMHD06jiebOnVuev2jRotRHZujQoel+vH355ZfDypUry2Uef/zxFEiOPvroxl5cACADDX4K6Wtf+1o4++yz02mjt99+O1x//fWhXbt24Ytf/GLqmzJmzJh0qqd79+4plFxxxRUptMQRSNHw4cNTULnwwgvD1KlTU7+X6667Ll07Jh5lAQBo8ADz1ltvpbDy3nvvhYMOOiiccsopaYh0/H902223hbZt26YL2MWRQ3GE0fe+973y82PYeeSRR8Lll1+egs0+++wTLr744nDjjTdqLQCgcQLMvffeu8P5nTp1CtOnT0/T9sSjN48++mhDLxoAUCX8FhIAkB0BBgDIjgADAGRHgAEAsiPAAADZEWAAgOwIMABAdgQYACA7AgwAkB0BBgDIjgADAGRHgAEAsiPAAADZEWAAgOwIMABAdgQYACA7AgwAkB0BBgDIjgADAGRHgAEAsiPAAADZEWAAgOwIMABAdgQYACA7AgwAkB0BBgDIjgADAGRHgAEAsiPAAADZEWAAgOwIMABAdgQYACA7AgwAkB0BBgDIjgADAGRHgAEAsiPAAADZEWAAgOwIMABAdgQYACA7AgwAkJ32zb0AAEDTOWTirKr4uB2BAQCyI8AAANkRYACA7AgwAEB2BBgAIDtGITVjL+8lN41qrLcHgKrmCAwAkB0BBgDIjgADAGRHgAEAsiPAAADZMQqpGRmpBEBT71eqhSMwAEB2BBgAIDsCDACQHQEGAMiOTrwtnI6+ALAtR2AAgOw4AlMFHKUBqH6taYj0rhBgAKCZCSf1J8C0Eru6cSy5aVSjLwsA7CkBhgpORwE0LEdXGocAQ6vYYB1ZAlrDd11r0qIDzPTp08O0adPC8uXLw6BBg8Idd9wRPvWpTzX3YrV61brBNlS9diUsNeV70To15dHUHLcd8tdiA8x9990Xxo8fH+66664wZMiQcPvtt4cRI0aERYsWhR49ejT34pEZX3pN8xk21A6oWoNZSwuuLW27aGnLQ8vWYgPMrbfeGi699NLwpS99Kd2PQWbWrFlhxowZYeLEic29eNAivoSbMlQ0pZa2PEDL0yIDzMaNG8P8+fPDpEmTyo+1bds2DBs2LMybN6/O52zYsCFNJatXr063a9asafDl27rhwwZ/TWgs/a56oCrfqzXzOdMSNMb+tebrFkWRX4B59913w5YtW0LPnj0rHo/3X3vttTqfM2XKlPCNb3xjm8f79u3baMsJAK1V19sb9/XXrl0bunbtmleA2R3xaE3sM1OydevW8P7774cDDjggtGnTpkGTYQxFy5YtC126dAnVqNrrqH7504b504Z5W9OI+4l45CWGlz59+uywXIsMMAceeGBo165dWLFiRcXj8X6vXr3qfE7Hjh3TVFO3bt0abRljg1Xjzr011VH98qcN86cN89alkfYTOzry0qJ/zLFDhw5h8ODBYe7cuRVHVOL9oUOHNuuyAQDNr0UegYni6aCLL744nHTSSenaL3EY9fr168ujkgCA1qvFBpgvfOEL4U9/+lOYPHlyupDd8ccfH2bPnr1Nx96mFk9TXX/99ducrqom1V5H9cufNsyfNsxbxxawn2hT7GycEgBAC9Mi+8AAAOyIAAMAZEeAAQCyI8AAANkRYOrwL//yL+Ev/uIvwt57773LF8OLfaHjiKnevXuHzp07p99tev311yvKxCsDX3DBBemiP/F1x4wZE9atWxeaWn2XY8mSJelqxnVNDzzw/799U9f8e++9NzS13fmcTz/99G2W/atf/WpFmaVLl4ZRo0al9SL+IvqECRPC5s2bQ3Oobx1j+SuuuCIceeSRaf3s169f+Id/+Ifyb4Y1dxtOnz49HHLIIaFTp07p1+eff/75HZaP691RRx2Vyg8cODA8+uij9d4em1p96vjDH/4wnHrqqWH//fdPU1z+2uUvueSSbdrqzDPPDDnUb+bMmdsse3xeNbVhXd8pcYrfIS2tDZ9++ulw9tlnpyvfxmV4+OGHd/qcp556Kpx44olpFNLhhx+e2nRPt+t6i6OQqDR58uTi1ltvLcaPH1907dp1lz6em266KZV9+OGHi//6r/8qPve5zxWHHnpo8dFHH5XLnHnmmcWgQYOKZ599tvjd735XHH744cUXv/jFJv/467scmzdvLt55552K6Rvf+Eax7777FmvXri2Xi6vTT37yk4pyNevfVHbnc/6rv/qr4tJLL61Y9tWrV1d8Bscee2wxbNiw4j//8z+LRx99tDjwwAOLSZMmFc2hvnV8+eWXi3PPPbf41a9+VbzxxhvF3Llzi09+8pPFeeedV1GuOdrw3nvvLTp06FDMmDGjWLhwYWqHbt26FStWrKiz/DPPPFO0a9eumDp1avHHP/6xuO6664q99tor1bE+22NTqm8dzz///GL69OlpXXv11VeLSy65JNXnrbfeKpe5+OKL03pQs63ef//9Iof6xXWsS5cuFcu+fPnyijK5t+F7771XUb9XXnklrbex7i2tDR999NHin//5n4sHH3wwfQc89NBDOyz/3//938Xee++d9pFxG7zjjjtS3WbPnr3bn9fuEGB2IK5ouxJgtm7dWvTq1auYNm1a+bFVq1YVHTt2LH7+85+n+7GR44rxwgsvlMv8+te/Ltq0aVP87//+b9FUGmo5jj/++OLLX/5yxWO7suK31PrFAPOP//iPO9zA27ZtW/Ele+edd6Yv4Q0bNhRNqaHa8P77709fMJs2bWrWNvzUpz5VjB07tnx/y5YtRZ8+fYopU6bUWf5v/uZvilGjRlU8NmTIkOLv/u7vdnl7bGr1rWNtMUDvt99+xU9/+tOKnd8555xTtAT1rd/OvlursQ1vu+221Ibr1q1rkW1Yn++Aa665pjjmmGMqHvvCF75QjBgxosE+r13hFFIDWLx4cbrYXjzEWfN3HOIhs3nz5qX78TYe6o9XFi6J5du2bRuee+650FQaYjnmz58fFixYkE5b1DZ27Nj0W1bx6skzZszY6c+ht6T63X333WnZjz322PTjoB9++GHF68ZTFTUvpDhixIj0g2YLFy4MTamh1qV4+iiegmrfvn2zteHGjRvT+lRz24n1iPdL205t8fGa5UttUSq/K9tjU9qdOtYW18VNmzaF7t27b3MYP57OjKcGL7/88vDee++FXOoXT3n2798//SDgOeecU7EdVWMb/vjHPw6jR48O++yzT4trw/ra2TbYEJ9X1lfizUnc0KLaVwmO90vz4m1cSWuKO474hVQq01TLuqfLETfEAQMGpH5CNd14443hM5/5TOojMmfOnPD3f//36Usq9rVo6fU7//zz05dpPAf80ksvhWuvvTYsWrQoPPjgg+XXrat9S/OaUkO04bvvvhu++c1vhssuu6xZ2zAux5YtW+r8bF977bU6n7O9tqi5rZUe216ZprQ7dawtro9x3ay5Q4h9Jc4999xw6KGHhjfffDP80z/9Uxg5cmTaQcQfw20qu1O/uLOO4fi4445LQfqWW25J3ycxxBx88MFV14ax78crr7ySvjtrailtWF/b2wbjH3QfffRR+OCDD/Z4nd8VrSbATJw4Mdx88807LPPqq6+mjoHVXL89FVfOe+65J3z961/fZl7Nx0444YT021XTpk1rkJ1fY9ev5o48HmmJHQfPOOOM9KXyiU98IlRTG8YvmdiR8Oijjw433HBDk7Uhu+emm25KHanjX+o1O7rGv+ZrrrMxDMR1NZaL625LFn+Ut+YP88bwEv8o+v73v5+CdbWJwSW2UTyqWVPObdgStJoAc/XVV6ce3zty2GGH7dZr9+rVK92uWLEi7fhK4v34G06lMitXrqx4XhzBEkeHlJ7fFPXb0+X4xS9+kQ5nX3TRRTstGw/3xi+jDRs27PHvZTRV/Woue/TGG2+kL5T43No96GP7Rg3Rfk1Vx7Vr16a/+vbbb7/w0EMPhb322qvJ2rAu8VRV/Euz9FmWxPvbq0t8fEfld2V7bEq7U8eSeGQiBpgnnngi7dx2tm7E94rrbFPu/PakfiVxPYyBOS57tbVh/CMgBtB4dHNnmqsN62t722A8JR1HjMXPak/XiV3SYL1pqlB9O/Hecsst5cfiCJa6OvG++OKL5TKPPfZYs3Xi3d3liJ1da49c2Z5vfetbxf777180pYb6nH//+9+n14mjH2p24q3Zg/773/9+6sT78ccfFznUMa6TJ598cmrD9evXt5g2jJ39xo0bV9HZ78/+7M922In3s5/9bMVjQ4cO3aYT7462x6ZW3zpGN998c1q/5s2bt0vvsWzZsrQO/PKXvyxyqF/tTspHHnlkcdVVV1VVG5b2I3G533333RbdhvXtxBtHZdYUR0HW7sS7J+vErhBg6vA///M/afhiaahw/H+cag4ZjhtbHHJWc8hfHCIWV7yXXnop9Syvaxj1CSecUDz33HNpBxmHsTbXMOodLUccqhnrF+fX9Prrr6eNK454qS0Oz/3hD3+YhrLGct/73vfSMLs4JL2l1y8OK77xxhtTIFi8eHFqw8MOO6w47bTTthlGPXz48GLBggVpuOBBBx3UrMOo61PH+OUfR+oMHDgw1bfmsM1Yt+ZswzjcMn7Bz5w5M4Wzyy67LG1LpRFfF154YTFx4sSKYdTt27dPO7c4xPj666+vcxj1zrbHplTfOsbljyPEfvGLX1S0Vek7KN5+7WtfS+EmrrNPPPFEceKJJ6b1oKkD9e7UL363xtD95ptvFvPnzy9Gjx5ddOrUKQ23rZY2LDnllFPSCJ3aWlIbrl27tryfiwEmXkYk/j/uC6NYr1i/2sOoJ0yYkLbBOOS/rmHUO/q8GoIAU4c4tC02Yu3pN7/5zTbXyyiJfzF8/etfL3r27Jka7YwzzigWLVq0zXUB4k4mhqL4l9WXvvSlilDUVHa2HHFjql3fKO6s+/btm5J0bTHUxKHV8TX32WefdI2Su+66q86yLa1+S5cuTWGle/fuqe3iNVXihlnzOjDRkiVLipEjRxadO3dO14C5+uqrK4Ygt+Q6xtu61uk4xbLN3YbxOhL9+vVLO+34l1u8vk1JPGIUt8naQ8CPOOKIVD4O55w1a1bF/F3ZHptaferYv3//OtsqhrXoww8/TGE6hugY3mL5eJ2Nhtw5NGb9rrzyynLZ2EZnnXVW8R//8R9V1YbRa6+9ltptzpw527xWS2rD32zn+6FUn3gb61f7OfH7In4W8Q++mvvDXfm8GkKb+E/DnZACAGh8rgMDAGRHgAEAsiPAAADZEWAAgOwIMABAdgQYACA7AgwAkB0BBgDIjgADAGRHgAEAsiPAAADZEWAAgJCb/wNrTfv2s9XCQgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Histogramme des valeurs de la couche `h`\n", "plt.hist(nn.h.view(-1).tolist(), 50);" ] }, { "cell_type": "code", "execution_count": 72, "id": "c5348d9e", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAG4hJREFUeJzt3XuMlNXhP+DDcpPbgigXKVBobQQKaoo3qjVVKWhXowUbmxiKhmhLkZRLLdAqFauBQCJWI2KbFvhDi6EJGqDaUlT8A/CCtQoq0UQKFgGN5SIt9/nmnF9mfzsIhYVd9uzu8yQvs+/7np15d87uzIdzmyaFQqEQAAAyUlbXFwAAcCQBBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7zUI9dPjw4bBly5bQrl270KRJk7q+HADgBMS1YXfv3h26desWysrKGl5AieGkR48edX0ZAMBJ2Lx5c+jevXvDCyix5aT4A5aXl9f15QAAJ2DXrl2pgaH4Pt7gAkqxWyeGEwEFAOqXExmeYZAsAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsNKvrCwDqh16Tlx23zMYZFaflWoCGTwsKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsmGYMZMeUZkALCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDvN6voCgLrXa/Kyur4EgJprQZkxY0Zo0qRJGDduXOWxvXv3hjFjxoSzzjortG3bNgwfPjxs27at5Ps2bdoUKioqQuvWrUPnzp3D3XffHQ4ePHgqlwIANCAnHVBee+218MQTT4Tzzz+/5Pj48ePDkiVLwqJFi8LKlSvDli1bwrBhwyrPHzp0KIWT/fv3h1WrVoUFCxaE+fPnh6lTp57aTwIANO4uns8//zzceuut4Xe/+1144IEHKo/v3Lkz/P73vw9PPfVUuPrqq9OxefPmhb59+4Y1a9aEyy67LPz1r38N77zzTvjb3/4WunTpEi688MLw61//OkyaNCncd999oUWLFjX30wHZ0Z0E1FoLSuzCia0ggwcPLjm+du3acODAgZLjffr0CT179gyrV69O+/F2wIABKZwUDR06NOzatSusX7/+qI+3b9++dL7qBgA0XNVuQVm4cGF44403UhfPkbZu3ZpaQDp06FByPIaReK5Ypmo4KZ4vnjua6dOnh2nTplX3UgGAxtCCsnnz5vDTn/40PPnkk+GMM84Ip8uUKVNS91Fxi9cBADRc1QoosQtn+/bt4Rvf+EZo1qxZ2uJA2EceeSR9HVtC4uDXHTt2lHxfnMXTtWvX9HW8PXJWT3G/WOZILVu2DOXl5SUbANBwVauL55prrglvv/12ybHbb789jTOJg1x79OgRmjdvHlasWJGmF0cbNmxI04oHDRqU9uPtgw8+mIJOnGIcLV++PIWOfv361dxPBpx2BsACdRJQ2rVrF/r3719yrE2bNmnNk+LxUaNGhQkTJoSOHTum0DF27NgUSuIMnmjIkCEpiIwYMSLMnDkzjTu555570sDb2FICAFDjK8nOnj07lJWVpRaUOPsmztCZM2dO5fmmTZuGpUuXhtGjR6fgEgPOyJEjw/333682AICkSaFQKIR6Jk4zbt++fRowazwKNM6umY0zKur6EoBafP/2YYEAQHYEFAAgOwIKAJAdAQUAaPizeIC81McBsABaUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2LHUPNNgl/DfOqDgt1wLUPC0oAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2mtX1BQAnr9fkZZ4+oEHSggIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZKdZXV8AQG3pNXnZcctsnFGhAiBDWlAAgOwIKABAdgQUACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAAD1e6G2xx9/PG0bN25M+1//+tfD1KlTw3XXXZf29+7dGyZOnBgWLlwY9u3bF4YOHRrmzJkTunTpUnkfmzZtCqNHjw4vvvhiaNu2bRg5cmSYPn16aNbMmnFQ3UXGABqqarWgdO/ePcyYMSOsXbs2vP766+Hqq68ON954Y1i/fn06P378+LBkyZKwaNGisHLlyrBly5YwbNiwyu8/dOhQqKioCPv37w+rVq0KCxYsCPPnz08hBwCgqEmhUCiEU9CxY8cwa9ascPPNN4dOnTqFp556Kn0dvffee6Fv375h9erV4bLLLgvPPfdcuP7661NwKbaqzJ07N0yaNCl88sknoUWLFif0mLt27Qrt27cPO3fuDOXl5ady+ZAtLSinh6Xu4fSpzvv3SY9Bia0hsStnz549YdCgQalV5cCBA2Hw4MGVZfr06RN69uyZAkoUbwcMGFDS5RO7geIFF1thjiZ2F8UyVTcAoOGq9sCPt99+OwWSON4kjiFZvHhx6NevX3jzzTdTC0iHDh1KyscwsnXr1vR1vK0aTorni+eOJY5RmTZtWnUvFeC4fKAg5KnaLSjnnXdeCiOvvPJKGuwaB7m+8847oTZNmTIlNQcVt82bN9fq4wEA9awFJbaSnHvuuenrgQMHhtdeey385je/Cbfccksa/Lpjx46SVpRt27aFrl27pq/j7auvvlpyf/F88dyxtGzZMm0AQONwyuugHD58OI0RiWGlefPmYcWKFZXnNmzYkKYVxy6hKN7GLqLt27dXllm+fHkaKBO7iQAAqt2CErta4ponceDr7t2704ydl156KfzlL39Jo3JHjRoVJkyYkGb2xNAxduzYFEriDJ5oyJAhKYiMGDEizJw5M407ueeee8KYMWO0kAAAJxdQYsvHD3/4w/Dxxx+nQHL++eencPKd73wnnZ89e3YoKysLw4cPL1morahp06Zh6dKlaexKDC5t2rRJY1juv//+6lwGANDAnfI6KHXBOig0BtZByYe1UqAerYMCAFBbBBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMhOs7q+AIDc9Zq87LhlNs6oOC3XAo2FFhQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDvN6voCABqCXpOXHbfMxhkVp+VaoCHQggIAZEdAAQCyI6AAANkxBgUyHa8A0JhpQQEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDIjnVQoIZZ4wTg1GlBAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQDqd0CZPn16uPjii0O7du1C586dw0033RQ2bNhQUmbv3r1hzJgx4ayzzgpt27YNw4cPD9u2bSsps2nTplBRURFat26d7ufuu+8OBw8erJmfCABoXAFl5cqVKXysWbMmLF++PBw4cCAMGTIk7Nmzp7LM+PHjw5IlS8KiRYtS+S1btoRhw4ZVnj906FAKJ/v37w+rVq0KCxYsCPPnzw9Tp06t2Z8MAKi3mhQKhcLJfvMnn3ySWkBiELnyyivDzp07Q6dOncJTTz0Vbr755lTmvffeC3379g2rV68Ol112WXjuuefC9ddfn4JLly5dUpm5c+eGSZMmpftr0aLFcR93165doX379unxysvLT/byoVb4LB6OZeOMCk8Ojdquarx/n9IYlPgAUceOHdPt2rVrU6vK4MGDK8v06dMn9OzZMwWUKN4OGDCgMpxEQ4cOTRe9fv36oz7Ovn370vmqGwDQcJ10QDl8+HAYN25cuPzyy0P//v3Tsa1bt6YWkA4dOpSUjWEkniuWqRpOiueL54419iUmruLWo0ePk71sAKAhB5Q4FmXdunVh4cKFobZNmTIltdYUt82bN9f6YwIAdafZyXzTXXfdFZYuXRpefvnl0L1798rjXbt2TYNfd+zYUdKKEmfxxHPFMq+++mrJ/RVn+RTLHKlly5ZpAwAah2q1oMTxtDGcLF68OLzwwguhd+/eJecHDhwYmjdvHlasWFF5LE5DjtOKBw0alPbj7dtvvx22b99eWSbOCIqDZfr163fqPxEA0LhaUGK3Tpyh8+yzz6a1UIpjRuK4kFatWqXbUaNGhQkTJqSBszF0jB07NoWSOIMnitOSYxAZMWJEmDlzZrqPe+65J923VhIAoNoB5fHHH0+33/72t0uOz5s3L9x2223p69mzZ4eysrK0QFucfRNn6MyZM6eybNOmTVP30OjRo1NwadOmTRg5cmS4//771QgAcOrroNQV66CQM+ugcCzWQaGx23W61kEBAKgNAgoAkB0BBQDIjoACAGRHQAEAGsZKstBYmaEDcHpoQQEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7zer6AgAai16Tlx23zMYZFaflWiB3WlAAgOwIKABAdnTxQDWa3wE4PbSgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkJ1mdX0BAPx/vSYvO+7TsXFGhaeMBk8LCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkxi4dG4URmRgBQj1tQXn755XDDDTeEbt26hSZNmoRnnnmm5HyhUAhTp04N55xzTmjVqlUYPHhweP/990vKfPbZZ+HWW28N5eXloUOHDmHUqFHh888/P/WfBgBonAFlz5494YILLgiPPfbYUc/PnDkzPPLII2Hu3LnhlVdeCW3atAlDhw4Ne/furSwTw8n69evD8uXLw9KlS1PoufPOO0/tJwEAGowmhdjkcbLf3KRJWLx4cbjpppvSfryr2LIyceLE8LOf/Swd27lzZ+jSpUuYP39++MEPfhDefffd0K9fv/Daa6+Fiy66KJV5/vnnw3e/+93w0Ucfpe8/nl27doX27dun+46tMHA8unhobCzmRo6q8/5do4NkP/zww7B169bUrVMUL+TSSy8Nq1evTvvxNnbrFMNJFMuXlZWlFhcAgBodJBvDSRRbTKqK+8Vz8bZz584l55s1axY6duxYWeZI+/btS1vVBAYANFz1Yprx9OnTU0tMcevRo0ddXxIAUF8CSteuXdPttm3bSo7H/eK5eLt9+/aS8wcPHkwze4pljjRlypTUX1XcNm/eXJOXDQA05IDSu3fvFDJWrFhR0h0Tx5YMGjQo7cfbHTt2hLVr11aWeeGFF8Lhw4fTWJWjadmyZRpMU3UDABquao9BieuVfPDBByUDY9988800hqRnz55h3Lhx4YEHHghf+9rXUmC5995708yc4kyfvn37hmuvvTbccccdaSrygQMHwl133ZVm+JzIDB4AoOGrdkB5/fXXw1VXXVW5P2HChHQ7cuTINJX45z//eVorJa5rEltKrrjiijSN+Iwzzqj8nieffDKFkmuuuSbN3hk+fHhaOwUA4JTXQakr1kGhuqyDQmNjHRRyVGfroAAA1AQBBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7zer6AgCoeb0mL6uR+9k4o6JG7geqS0Ch3r/IegEFaHh08QAA2RFQAIDsCCgAQHYEFAAgOwIKAJAds3io92pqOiUA+dCCAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2fFZPACc0mddbZxR4RmkxmlBAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHdOMyXr6IgCNkxYUACA7WlAAOCUWc6M2aEEBALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAd04ypFRZhA+BUaEEBALIjoAAA2RFQAIDsCCgAQHYMkgWg1vm8HqpLQKHazNABoLbp4gEAsiOgAADZEVAAgOwYgwJAFgykpSotKABAdgQUACA7AgoAkB0BBQDIjkGyjYgBaADUF1pQAIDsNCkUCoW6evDHHnsszJo1K2zdujVccMEF4dFHHw2XXHLJcb9v165doX379mHnzp2hvLz8tFxr7iw/D3DiNs6o8HTVgeq8f9dZF8/TTz8dJkyYEObOnRsuvfTS8PDDD4ehQ4eGDRs2hM6dO9fVZQFAolu8btVZQHnooYfCHXfcEW6//fa0H4PKsmXLwh/+8IcwefLk0Fj4AwCATLp49u/fH1q3bh3+9Kc/hZtuuqny+MiRI8OOHTvCs88+W1J+3759aSuKTUM9e/YMmzdvrpUunv6/+stxy6ybNrRG7geAhq2m3i/WncD9nKjT/XhVu3h69OiR3utjV092LSiffvppOHToUOjSpUvJ8bj/3nvvfaH89OnTw7Rp075wPP6QdaX9w3X20ADUIzX1ftH+NL/v1Obj7d69O8+AUl1TpkxJ41WKDh8+HD777LNw1llnhSZNmoQcFFNhbbXqoC7qG38TeVAP+VAXIcROmxhOunXrdtznq04Cytlnnx2aNm0atm3bVnI87nft2vUL5Vu2bJm2qjp06BByFMOJgJIHdZEH9ZAH9ZCPxl4X7Y/TclKn66C0aNEiDBw4MKxYsaKkVSTuDxo0qC4uCQDISJ118cQumzgo9qKLLkprn8Rpxnv27Kmc1QMANF51FlBuueWW8Mknn4SpU6emhdouvPDC8Pzzz39h4Gx9EbugfvWrX32hKwp10Vj5m8iDesiHuqhHK8kCAByNz+IBALIjoAAA2RFQAIDsCCgAQHYElFO0cePGMGrUqNC7d+/QqlWr8NWvfjXN5omfN1TVW2+9Fb71rW+FM844I604O3PmzFN9aI7w4IMPhm9+85vpc56OtZDfpk2bQkVFRSoTPzX77rvvDgcPHvRc1oLHHnss9OrVK/3Ox08sf/XVVz3Ptejll18ON9xwQ1qhM66w/cwzz5Scj/Mh4qzJc845J71WDR48OLz//vvqpIbFj2a5+OKLQ7t27dJrTPy8uQ0bNpSU2bt3bxgzZkxaDb1t27Zh+PDhX1i4FAHllMXPDoqLzD3xxBNh/fr1Yfbs2emTmX/xi1+ULG88ZMiQ8OUvfzmsXbs2zJo1K9x3333ht7/9rd/BGhRD4fe///0wevToo56Pn/8Uw0kst2rVqrBgwYIwf/789KJNzXr66afTWkcxrL/xxhvhggsuCEOHDg3bt2/3VNeSuI5UfJ5jMDya+J+iRx55JL0+vfLKK6FNmzapTuKbJTVn5cqVKXysWbMmLF++PBw4cCC9/sf6KRo/fnxYsmRJWLRoUSq/ZcuWMGzYMNVwpDjNmJo1c+bMQu/evSv358yZUzjzzDML+/btqzw2adKkwnnnneeprwXz5s0rtG/f/gvH//znPxfKysoKW7durTz2+OOPF8rLy0vqhlN3ySWXFMaMGVO5f+jQoUK3bt0K06dP9/SeBvGlffHixZX7hw8fLnTt2rUwa9asymM7duwotGzZsvDHP/5RndSi7du3p/pYuXJl5fPevHnzwqJFiyrLvPvuu6nM6tWr1UUVunhqwc6dO0PHjh0r91evXh2uvPLKtMR/UfyfS2z2+/e//10bl8BRxHoYMGBAyWKAsR5iC1ds/aJmxBaq2FIYuxCKysrK0n6sA06/Dz/8MC2IWbVO4uehxK43dVL77wdR8T0h/m3EVpWqddGnT5/Qs2dPdXEEAaWGffDBB+HRRx8NP/rRjyqPxReGI1fILe7Hc5we6uH0+PTTT1N32tF+5/2+143i865OTq/Y/T9u3Lhw+eWXh/79+1fWRfzP6pHj5Px9fJGAcgyTJ09OA83+1xbHn1T1r3/9K1x77bVpHMQdd9xxrLumlusBIAdxLMq6devCwoUL6/pS6qU6+yye3E2cODHcdttt/7PMV77ylcqv4yCnq666Ks0iOXLwa9euXb8wQru4H89Rc/Xwv8Tn+siZJOqh5p199tmhadOmR/2d9/teN4rPe6yDOIunap3Ez0Gj5t11111h6dKlaXZV9+7dS+oidoPu2LGjpBXF38cXCSjH0KlTp7SdiNhyEsPJwIEDw7x581J/e1WDBg0Kv/zlL1O/Y/PmzdOxOLr7vPPOC2eeeeYJPUZjVZ16OJ5YD3EqcpxJEqf/FeuhvLw89OvXr0Yeg5Car+PfwooVK9IUy2JTd9yPL9qcfnEZhPjGGOugGEji2Ks4m+dYs944OXGM8tixY8PixYvDSy+9lJ77quLfRnwfiHURpxdHcTxiXAIhvkZRRdURs1TfRx99VDj33HML11xzTfr6448/rtyK4qjtLl26FEaMGFFYt25dYeHChYXWrVsXnnjiCU95DfrnP/9Z+Pvf/16YNm1aoW3btunruO3evTudP3jwYKF///6FIUOGFN58883C888/X+jUqVNhypQp6qGGxd/xOENk/vz5hXfeeadw5513Fjp06FAyg4qaFX/Pi7/z8aX9oYceSl/Hv4toxowZqQ6effbZwltvvVW48cYb02zD//73v6qiBo0ePTrNInzppZdK3g/+85//VJb58Y9/XOjZs2fhhRdeKLz++uuFQYMGpY1SAkoNTGmNLwZH26r6xz/+UbjiiivSi/aXvvSl9GJBzRo5cuRR6+HFF1+sLLNx48bCddddV2jVqlXh7LPPLkycOLFw4MABVVELHn300fQi3KJFizTteM2aNZ7nWhR/z4/2+x//LopTje+99970n6X4OhT/U7VhwwZ1UsOO9X4Q3yuKYij8yU9+kpafiP9Z/d73vlfyn1r+nybxn6otKgAAdc0sHgAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgCE3Pwfo2nqBY9r5YkAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Histogramme des valeurs de la couche `h`\n", "plt.hist(nn.hpreact.view(-1).tolist(), 50);" ] }, { "cell_type": "markdown", "id": "d5dd11d0", "metadata": {}, "source": [ "La distribution de valeurs est trop large. Beaucoup de nombres prennent des valeurs extrêmes, ce qui a un impact sur la qualité finale du réseau: beaucoup de valeurs passant \"à travers\" cette couche du réseau en remontant lors de la propagation arrière sont supprimées.\n", "\n", "Que se passe t'il dans le réseau quand tanh vaut -1 ou 1? En reprenant le code que nous avions implémenté pour la rétropropagation:\n", "\n", "```python\n", "def tanh(self):\n", " x = self.data\n", " t = (math.exp(2*x) - 1) / (math.exp(2*x) + 1)\n", " out = Value(t, children=(self,), op='tanh')\n", " def _backward():\n", " self.grad = (1 - t**2) * out.grad\n", " out._backward = _backward\n", " return out\n", "```\n", "\n", "La valeur calculée $t$ est comprise entre $-1$ et $1$ ($\\mathrm{tanh}$). Lors du calcul du gradient local, quand $t$ est proche de $-1$ ou $1$, on va \"tuer\" le gradient puisque $1-t^2$ sera égal à 0 et on va donc stopper la propagation arrière ici.\n", "\n", "On peut améliorer cette situation, tout d'abord en visualisant tous les cas où les valeurs de `h` sont dans une région \"plate\" de la fonction $tanh$ (proches de $-1$ ou $1$), les cases blanches étant celles où c'est le cas:" ] }, { "cell_type": "code", "execution_count": 73, "id": "184ebdd9", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABj0AAAElCAYAAAC/JSDoAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAANqZJREFUeJzt3QuwXVV9MPB1QiCgci9GHiElUBAEKpW2KA9RCkKJ0KEi0CraCpSipUAHokXp8JDWaQRapSqETqcFnRqxzPAY7RRHUEIdgdY4jNUWRigUGAgqTu6FUAIl55u9v+/my4Xknnuz7spae53fb+aIuefsvddee732+Z+1V6/f7/cDAAAAAABAx83JnQAAAAAAAIDZIOgBAAAAAABUQdADAAAAAACogqAHAAAAAABQBUEPAAAAAACgCoIeAAAAAABAFQQ9AAAAAACAKgh6AAAAAAAAVRD0AAAAAAAAqjA3FGbdunXhySefDNtvv33o9Xq5kwMAAAAAAGTU7/fDs88+GxYuXBjmzJmTJ+hxzTXXhKuuuiqsWrUqHHjggeHzn/98OPjggwdu1wQ8Fi1alCpZAAAAAABABz3++ONht9122/JBj69+9athyZIl4brrrguHHHJIuPrqq8PixYvDgw8+GHbeeecpt21meEwkfmRkJNRkdHQ0avuxsbFkx47Zd+yxB5kqbanzdKr9x2w7ne1z7Xs6+485ds58GSRlPRgkpqzValjbzNRpK7Wsdfl6x2zf5XYrp9h8K7Ufi5Vy3DJITDnPOQ7tch3KOV6L2XdOJfc1KY8dK+XYYVjLWq3jlpLLQ8l9Uc5yXmrautxmxshZv3Mq+bu9YTVaaB0cHx9vJ0tMxA+m0us380JmWRPoeNvb3ha+8IUvrH9kVZOg8847L3ziE58YmPgmY5vMqS3oEfu4rphLNejYCYrBtI89yFRpS52nU+0/ZtvpbJ9r39PZf8yxc+bLICnrwSAxZa1Ww9pmpk5bqWWty9c7Zvsut1s5pXwEas5+LFbKccsgMeU85zi0y3Uo53gtZt85ldzXpDx2rJRjh2Eta7WOW0ouDyX3RTnLealp63KbGSNn/c6p5O/2hlWv0Do4k7jBrC9k/uKLL4aVK1eGY4455v8fZM6c9t/33HPPqz6/du3aNsEbvgAAAAAAAGZq1oMeP/vZz8LLL78cdtlll0l/b/7drO/xSkuXLm0jNBMv63kAAAAAAABFBD1m6qKLLmqnpEy8mrU8AAAAAAAAZmrWFzLfcccdw1ZbbRWefvrpSX9v/r1gwYJXfX7evHntCwAAAAAAoKiZHttss0046KCDwp133rn+b81C5s2/DzvssNk+HAAAAAAAQJqZHo0lS5aE0047Lbz1rW8NBx98cLj66qvDmjVrwhlnnJF8BflBq8MPWn0+1eryqfcde+xB+RKz75zHjjXV/lOXpZh8SSn2vGqtYyWLKaupy2HKaxZz3inbrdxtasrrnTPfUo4dBonZPnWepTzvmGOnrgcpdXlcE3PsWs+ry2La89hxbFfb81LHHdPZvtR74Nh9l1xHc/ahU8l5vUtuG3LqcptaaruYc4ycsw52tU2cjlrPO2UfmrIs9jN+l7ulvptPEvR43/veF37605+GSy+9tF28/Fd+5VfC7bff/qrFzQEAAAAAAGZLr19YOG18fDyMjo62i5qPjIx06lcuXZbz13tdvSYlz/RIeexSr0fXpczzrs70KHmWSc6ZHoMM60yPlL9yqfWXg12e6VGyWvvvro7XhlXJvxyMVWp7nvOX3qnvgbXnWz7PY5R87JS6POOp1HYtVlfzfJBhrf+pxxYplfo9cpfHa72E3ztMte104gbJ1vQAAAAAAADIQdADAAAAAACogqAHAAAAAABQBUEPAAAAAACgCnNDB8UstlLqwjnTkXORp1IXMyt50Z/YPM+5wHPOBcNKXgSqq7raZsbqaptZ80KJpS5cn7Ndy3neqXV1bFHy4pQp5RyvdXVBz9xyLsKaU6ljxdR5Wup4LvVCqCmPnXv/m6vkcUvO8pBTl9vUlHUwZ3vd1Xax5HFLyWP7Uq9nzn2XXAe3VDk20wMAAAAAAKiCoAcAAAAAAFAFQQ8AAAAAAKAKgh4AAAAAAEAVBD0AAAAAAIAqCHoAAAAAAABVEPQAAAAAAACqMDcUanR0dJPv9fv9ZMft9XpTvp/y2DmlPO8u51mX057qvAeVldiyNGj7XPtO3TaUWtZSn3fK6x1z7JRlKVbJZS2mvMTmec6yFJP2nGUtdt8pzzv22KW2ezn7yNTHztkupjyvnGUtRsl1bFjvS7qcp6W2e6nbnZRlratjyS7XoZzjtVLv9XLX71Tb5tbV+p373iFm21LHoanviVLqJ7w32FLnbaYHAAAAAABQBUEPAAAAAACgCoIeAAAAAABAFQQ9AAAAAACAKgh6AAAAAAAAVRD0AAAAAAAAqiDoAQAAAAAAVGFuKNTY2FgYGRnZ6Hu9Xm+T2/X7/Sn3O9W2qQ1KW63H7vJ5pSwvJe97qnxNXcem2v+gfQ96P6a85CxrXW47cra5g3Q5X4dRl/MsZ9pj6mBsumPa8y5f75Ri+uDY/rvkMXaMlOc1aN858zRlHUw5FpyN/cccO6Wc49iuHjt1WUuZZ13t51K39V0dn+fsS3Lef+dU8liyq/W7y+cVU09Sl6WcfU1KXTgvMz0AAAAAAIAqCHoAAAAAAABVEPQAAAAAAACqIOgBAAAAAABUQdADAAAAAACogqAHAAAAAABQBUEPAAAAAACgCnNne4ef/OQnw+WXXz7pb/vuu2944IEHZrSf0dHRTb7X7/c3+V6v15tyv1NtW7pB55brvGPzPOd5xaQ9Jt2xUpfjlOcWk+cpy9Kg7VPnecz+U7Z7qdvUlHUsZ9vS1Ta3y33kICnPO2VZK7ksxsjZf8eq9dixfWxXz3uQkvvQUo9dcpuac/xe8n1qqX1F7v47Zx2ttY7lHKeWfM+Usm3K2d6n/L6l5HYrp5zfcQ1r/Y6R+l5wKiXXsdnK81kPejTe/OY3hzvuuOP/H2RuksMAAAAAAACslyQa0QQ5FixYkGLXAAAAAAAAW25Njx//+Mdh4cKFYa+99gof/OAHw2OPPbbJz65duzaMj49PegEAAAAAAGQPehxyyCHhhhtuCLfffntYtmxZeOSRR8I73/nO8Oyzz27080uXLm3X75h4LVq0aLaTBAAAAAAADIFeP/HKJKtXrw577LFH+MxnPhPOPPPMjc70aF4TmpkegwIfw7rYkYXMX81C5uWVtVilLiBXctvR5UX5UrbnJS/SGKPkcl5yH9zVhczp3mKFOdu1ktuerh47xrAubBtL/z1cSl7Qu6vj1C7Xsa6OU3Oed1fPa9D2Xe37c7OQeV0LmafUK3hsPx1jY2NhZGRkys8kX2F8hx12CG9605vCQw89tNH3582b174AAAAAAABiJA96PPfcc+Hhhx8Ov/d7vxdK0OVocc4IfVdnDQzilyZ15XmXo+Sl/lK85DZxkJJnguS8Jil/UVXyr9hy1pOY8tDl/rureZ7y2Cnrb83jtRg561DO613y9Sx5NlVOJc86KLWelDw2SHlNco7XUrctOdu9nPdjMdt3+R4457G7OqtokJz3YznVOj5Pqd/RmXvNE6Ka5TGyrOnxsY99LKxYsSI8+uij4bvf/W5473vfG7baaqtw6qmnzvahAAAAAAAA0s30eOKJJ9oAxzPPPBN22mmn8I53vCPce++97f8HAAAAAADoTNDjxhtvnO1dAgAAAAAAbPnHWwEAAAAAAOQg6AEAAAAAAFRB0AMAAAAAAKiCoAcAAAAAAFCFWV/IfEvo9XqbfK/f72/2ttN5P8agtMVIme7UYvJl0HnHloepto89dmzaunpNch479pqk2jb3ecfIeexYpaYtNl0pr0nO693lchzTl8SK2X/Kspi6Dyy1H0td1nJe72HtS1IqeXwfU79zjoFzti3Mfv1Pea83ne1TylnWSi3nw3o9Sm7XlNPZ1+XvuFIee5CS70O7Oj7vZ/yOa7aY6QEAAAAAAFRB0AMAAAAAAKiCoAcAAAAAAFAFQQ8AAAAAAKAKgh4AAAAAAEAVBD0AAAAAAIAqCHoAAAAAAABVmBsKNTY2FkZGRmZ9v/1+P2r7Xq+32fueatvpbJ/SoLTFKPm8YtKW+rym2n/JZSnlsVOfd0z9zlkHa73eJef5ILH9QantWso8ixWTLynbjtj9x6YtZvvY8yq1fYg9r672wYPSHVu/c45bSm6bYuQsSzmVfN4xY8WSzyum3cvZTw3r+DunlH1FrJT9d8n3oTWP32scj5Ws5HSn7GtKvjdImbZ+wffAs8VMDwAAAAAAoAqCHgAAAAAAQBUEPQAAAAAAgCoIegAAAAAAAFUQ9AAAAAAAAKog6AEAAAAAAFRB0AMAAAAAAKjC3FCZXq8XtX2/3496P2bb2LTnOnZMnkzHVGmLPfag8059bqmuSex5pczzGDmvd+q2JaeU9T9nu5Zy+5TnNUjJdTBnm1pyHZtKyf1Y6rSllLIfq7WslTwOzVmOB8k5Zoo5dmyep+xDu3q9U46/c593zBi55Daz5Hu9UtOWui/pav+ds02tdexQcx3r6v1Wl8eKXW1buvwdV6+A7xXN9AAAAAAAAKog6AEAAAAAAFRB0AMAAAAAAKiCoAcAAAAAAFAFQQ8AAAAAAKAKgh4AAAAAAEAV5s50g7vvvjtcddVVYeXKleGpp54Kt9xySzjxxBPXv9/v98Nll10W/vZv/zasXr06HH744WHZsmVhn332CVtCc/yUer1esrTFpD0mXSXka65jp8zz2HSXmuepy1pKMXkaez1iysugbXNek9T1IOWxp9o+tr1Oeb1jj93VdqlkOa/JIDmPnbMvyVlWY9qWnG1qSjWPmaaSsz2PrWMpr1nJfWhKsdckZZuacsw0SJfvLVJKmee1Xq+c9b/L12RY7x26WsdStrmp74Fj5PxeIaec1ztWCd9xz3imx5o1a8KBBx4Yrrnmmo2+f+WVV4bPfe5z4brrrgv33XdfeO1rXxsWL14cXnjhhdlILwAAAAAAwOzM9DjuuOPa16aiOFdffXW4+OKLw3ve8572b1/60pfCLrvsEm699dbw/ve/f6aHAwAAAAAA2PJrejzyyCNh1apV4Zhjjln/t9HR0XDIIYeEe+65ZzYPBQAAAAAAEDfTYypNwKPRzOzYUPPvifdeae3ate1rwvj4+GwmCQAAAAAAGBKzOtNjcyxdurSdDTLxWrRoUe4kAQAAAAAAwx70WLBgQfvfp59+etLfm39PvPdKF110URgbG1v/evzxx2czSQAAAAAAwJCY1aDHnnvu2QY37rzzzkmPq7rvvvvCYYcdttFt5s2bF0ZGRia9AAAAAAAAkq/p8dxzz4WHHnpo0uLl999/f5g/f37Yfffdw/nnnx8+9alPhX322acNglxyySVh4cKF4cQTTwyzpdfrbfK9fr8/a8eZ6f6nSlfp550636YSk2+p0536muY69qB8S1nWBp1XTB2LOa9BYvfd1TzPqeR0l9w25MyXUstSyYa1fudsc2Pr76C0pezHciq5zU2ZtpzHzjmuSanWepC6bUm1bez+S76eXS1Lg/I1Z1mLlbK8pBw75LwXHCRnH5hTbJ6XXA9SyjluqXW8VXId7BfcD25uupvJFc3yGEmCHt/73vfCUUcdtf7fS5Ysaf972mmnhRtuuCFceOGFYc2aNeHDH/5wWL16dXjHO94Rbr/99rDtttvO9FAAAAAAAADT1usXFvaZiNg063ts6lFXOWd65Iz+lXresUr9VUPutJnpMfM8MdNj88pSqe1HyW1qyl9cDOuvOWpmJtfM8qTrMz262h6nTFvO6x2ry9esq+nuah0b1v475/i81jwt+V4wta625yXfp9barpXcPw9S6vXOqcv3/sNaB0uOGyRZ0wMAAAAAACAXQQ8AAAAAAKAKgh4AAAAAAEAVBD0AAAAAAIAqCHoAAAAAAABVmBsK1azEnkOv10u28n3svqd6P2W6p7P/GLFpSykmz1NKnWcpz3tQ2ksta7F1LHUdjTFV2lK3HSXX/1xS50lX267UbU8uKccGg/afM09ytpkln3fJ9bfktiNlOc/ZbqUcr6Uc16Rst4a5D42RehxbqpzlPFbK/ee834ppm0puz0uuQzFpL7lNTTmmKvW+IbWSv/Mo+XuFlOW45LLYK/iaTDDTAwAAAAAAqIKgBwAAAAAAUAVBDwAAAAAAoAqCHgAAAAAAQBUEPQAAAAAAgCoIegAAAAAAAFUQ9AAAAAAAAKowN3cCuqbX623yvX6/v0XTsiWPHbP/qfIsdvtB6Rp07NjtU5oqbbHpSlleurrvWCnLWupy3tXzTqnLeZqybSm5Dqbsp1LmeWo5r3dM/S65rA2Ssj0vWcpxS0xZzdmu5bzeXa5DXe1jU1/vUu9LUu875rxLHjOlbNdSqvl7h67ex6Zs11LnaUw5T1kPcp53zbp67z9IzeP7rjPTAwAAAAAAqIKgBwAAAAAAUAVBDwAAAAAAoAqCHgAAAAAAQBUEPQAAAAAAgCoIegAAAAAAAFWYmzsBpen3+2EY9Xq9zc6XmG2ns33O6xlz3rGm2n/qPI3ZPvbYKctaSrHXJGVZy5lvXT126vo9ldT1OybPcx47pVLTNRtKbc9rHreUWp5i0zXovHOO51IeO+X1zllHYtNWav3OWZZKblty1pFYMe15ymuS8j4z5bbTMdV5l3wPXHKe13pfEivm3Eodb5V+3l0dr3V1PDZI6vuKXsI2NeWxZ6ucmukBAAAAAABUQdADAAAAAACogqAHAAAAAABQBUEPAAAAAACgCoIeAAAAAABAFQQ9AAAAAACAKgh6AAAAAAAAwxn0uPvuu8MJJ5wQFi5cGHq9Xrj11lsnvX/66ae3f9/w9e53v3s20wwAAAAAABAf9FizZk048MADwzXXXLPJzzRBjqeeemr96ytf+cpMDwMAAAAAADAjc2f28RCOO+649jWVefPmhQULFsx01wAAAAAAAGWt6XHXXXeFnXfeOey7777h7LPPDs8888wmP7t27dowPj4+6QUAAAAAAJA96NE82upLX/pSuPPOO8MVV1wRVqxY0c4Mefnllzf6+aVLl4bR0dH1r0WLFs12kgAAAAAAgCHQ6/f7/c3euNcLt9xySzjxxBM3+Zn/+q//Cm984xvDHXfcEY4++uiNzvRoXhOamR4xgY+I0xlqzbXc3HyN2XY628fsO6XY847Zf2yeprwmsVKWtRip8zTmvGPF5Fts2kq9ZjnLWs4873JZ0/9vXr6lVGv9TlkWU16vlOOSLqct5/VOqeT+eZCSx3sxSm5bSm23Uo9bYq5JqXW/62OHWtvrnPeCJZfVrn4X1OXzLvXYOb/jSl2WuprnJY+/G2NjY2FkZGTLP95qQ3vttVfYcccdw0MPPbTJ9T+aRG74AgAAAAAAmKnkQY8nnniiXdNj1113TX0oAAAAAABgiM2d6QbPPffcpFkbjzzySLj//vvD/Pnz29fll18eTj755LBgwYLw8MMPhwsvvDDsvffeYfHixbOW6K5OYyt5umXJ5Is8m25ZUMdmX8mPDcg59T5nWev6NNRUxy5ZqVP3S65jKXW5LMVcs9h2q9bykjLdOfvQ2H13VcmPaeyynGOHUh9fl/ORQ6nHoV19FGpKXX60tXvkLS9l21LyIyBjx5K5lPx4+JLrbz/TfWqzLEazJniSoMf3vve9cNRRR63/95IlS9r/nnbaaWHZsmXhBz/4QfjiF78YVq9eHRYuXBiOPfbY8Od//uftY6wAAAAAAABSmXHQ48gjj5wy4vKNb3wjNk0AAAAAAADlrekBAAAAAACwJQh6AAAAAAAAVRD0AAAAAAAAqiDoAQAAAAAAVEHQAwAAAAAAqMLcUKixsbEwMjKy0fd6vd4mt+v3+6FUsWmLOe+ptp3O9l016LwHicmX2GPnlLI8xORLbDmO2T51HcnZrsVck9i0TbV96uudct858zT3/nPJ2c+V3N7Xer0HKbkOpjx2V9OecrxWcl8SK+a8SxZzTXLWgZLvO7rcF6RMe8nlJed9SU4p266U9aSr6c6dtpT9WM7rnbPdiklb6vMqdUzV1fFz7P57Bd8/z9axzfQAAAAAAACqIOgBAAAAAABUQdADAAAAAACogqAHAAAAAABQBUEPAAAAAACgCoIeAAAAAABAFebmTkBper3elO/3+/1s+57q/ZTpHrT/QftOnbZS911yWUppUNpixJ5XyXme85rlPHYX05U6bbFlTfsw+3K257Hbd3Xs0OU6mFPKPI8pqzFj3OkcO+fYI2Wex0h97JTlIWbfOdu1QXIeO6VhLWsl32/lHCumPHbJ3yvkTFvK8hDbf5f8PVPOMdNU+885Ri55jJtTyfeCpbbXsWZrzGOmBwAAAAAAUAVBDwAAAAAAoAqCHgAAAAAAQBUEPQAAAAAAgCoIegAAAAAAAFUQ9AAAAAAAAKog6AEAAAAAAFRhbqhMr9eb8v1+v1/ssQdtHyN23zH5ljLPS74mJe875TWptY7FnlfJacvVLsUq+bxjrnfOspazvY6tIynrUGy+5LzeOcWkLWV7H3u9c/axKaU8r5Lb1JKvZ8o+OOd9ySAl5/lUaUudZzHHLrl+59TV+/OSxw6xeVbqOLXk73pSlofUfWSpbWrKbbvcj3X5epeq5LI0SK52bXx8PIyOjk5rP2Z6AAAAAAAAVRD0AAAAAAAAqiDoAQAAAAAAVEHQAwAAAAAAqIKgBwAAAAAAUAVBDwAAAAAAoAqCHgAAAAAAQBXmzuTDS5cuDTfffHN44IEHwnbbbRfe/va3hyuuuCLsu+++6z/zwgsvhI9+9KPhxhtvDGvXrg2LFy8O1157bdhll11mlLDR0dFNvtfv90MqMfuOTVfK8xqk1+sVe+xSr0lsnpVajged26B9p8yX2GPH5EvsvnOmLUbO88p9TWL2nXL72POOaddy9lOxunreOev3sPbfJV/vmPrd5XFJSiWP31P2FSWPY3PW0ZixQ8ljppLrYE4574FziimLqct5V69Jyfd6JV/vYSwrsX1JyfnW5WuS854p5bFjlHxPlGSmx4oVK8I555wT7r333vDNb34zvPTSS+HYY48Na9asWf+ZCy64IHzta18LN910U/v5J598Mpx00kmzklgAAAAAAIBN6fUjQjc//elPw84779wGN4444ogwNjYWdtppp7B8+fJwyimntJ9pZoXsv//+4Z577gmHHnrowH2Oj49POcuj4Zcqs6/Lv9bOpavp7vJMj0HM9Jh9Jf9qscszPWJ0uZzn2veg/acuxznb1K7O9EipyzP3Ss3Tmn8x2VWpf21Zcv+eUqkzPXL2YyUreSxY8rFLHUuWWveno9Q8nc72ufZd8rFj5SznJd+vxRx7WPO8Vr1p5FkTgxgZGUm3pkdzgMb8+fPb/65cubKd/XHMMces/8x+++0Xdt999zbosTHNI7CaQMeGLwAAAAAAgJna7KDHunXrwvnnnx8OP/zwcMABB7R/W7VqVdhmm23CDjvsMOmzzXoezXubWiekmdkx8Vq0aNHmJgkAAAAAABhimx30aNb2+OEPf9guWB7joosuameMTLwef/zxqP0BAAAAAADDae7mbHTuueeGr3/96+Huu+8Ou+222/q/L1iwILz44oth9erVk2Z7PP300+17GzNv3rz2BQAAAAAAsMVmejQLqDQBj1tuuSV861vfCnvuueek9w866KCw9dZbhzvvvHP93x588MHw2GOPhcMOOywqoQAAAAAAALM206N5pNXy5cvDbbfdFrbffvv163Q0a3Fst9127X/PPPPMsGTJknZx82YV9fPOO68NeBx66KGhhhXim8BPV1e2jzmvqfafOk9izi1l2mLyLHb72PNKWc5j8yVnWZtK7LFjto+t34OkvN4ppa6DMWKO3dV+aDauScy+B4lpW2KvSanXO/Wxtedl5fkgOduWlPmSeozcVSWXtZx9SVf74NTpKvW8S25bYst5jJzluOS+JKVazzvldxqD5PyeKef3LanHLTnbvZh9xyr1fqzkcUkvY/3PEvRYtmxZ+98jjzxy0t+vv/76cPrpp7f//7Of/WyYM2dOOPnkk8PatWvD4sWLw7XXXjubaQYAAAAAAHiVXr+wn22Mj4+3M0a6GsnKpcszPVJG/0r+xUXJeZ6zjnU1Cp5SyTM9SlZyOS/1lyY1//I/50yPXPse5l/vlcpMjzT5UutYMkbNZa2rY8lax6klG9brWXLaco7XUir1/nk6Ut6XlNxel/p9S8kzVGttU0u+B+5yvvQSt11jY2PtE6ZmbU0PAAAAAACAUgl6AAAAAAAAVRD0AAAAAAAAqiDoAQAAAAAAVGFu6KCUCw6VvPBOykVgci6OU+sibykX9eny4nQlL3YWs+9YORd4LlWXF3GL0eVF+3L2sTHHHtY+MPXicyUvrJdSqQtjlnzeJY/fS14kvavtecnHLnnfORd4LrU9T71Iaql5XnJfkFOtdazm65ny+5YYsXWs1O+Rci9cXWo5L7kPLXmh81z92Pj4eBgdHQ3TYaYHAAAAAABQBUEPAAAAAACgCoIeAAAAAABAFQQ9AAAAAACAKgh6AAAAAAAAVRD0AAAAAAAAqiDoAQAAAAAAVGFu6KB+v7/Z2/Z6vWT7jtl2OmmLMShtMceOTXfKtJWs5POe6tix5TxG6rIWc+zYfJlq+9THLjXPU9eRmHKes/6mLGuDxJbFlHkeI3X/nbJ+5zx2V/uSQWLzpeSymlJMWYvZd6yUdazLZS2lkstDzjHXIDn7khhdvv8utT0f1nFLajmvSY3ldDrb5xwrltqH5qxjOe8zu1wPcn53V/J59xOWteky0wMAAAAAAKiCoAcAAAAAAFAFQQ8AAAAAAKAKgh4AAAAAAEAVBD0AAAAAAIAqCHoAAAAAAABVEPQAAAAAAACqMDdUptfrTfl+v98PXZUy7Tn3HXPNcl7vQccuWUy+pM7zlGmruX1IVVZj62/KY+es37HHLrX9yJnng6i/G+eazL7Ydi/lvnOmbZCpjp0y3amPHbN9bJuacowcK6a85OxDuzy2yFnOY+RMW87xVup2L6WUact5H1pqH5n62DFS53mp51br9wKDlNwuxSq1DsbmecqxwyBdqCdmegAAAAAAAFUQ9AAAAAAAAKog6AEAAAAAAFRB0AMAAAAAAKiCoAcAAAAAAFAFQQ8AAAAAAKAKgh4AAAAAAEAV5s7kw0uXLg0333xzeOCBB8J2220X3v72t4crrrgi7Lvvvus/c+SRR4YVK1ZM2u4jH/lIuO6660INer3eJt/r9/ubvW2s2H3HpD32vAdtn1JM2kpOd8o8T33eKetJjJzlPGdZy5m21GVhqrSXWg5Lb3tK3Xfq/afs/4f1eg8675h8qbl+58yXlP1czmtWcnmpdXw+yLD236X2B6nLWqn1v+bzTlkOS257Sj126rKUcuyQc98ll7Va+4qSvzvoah8as+8u5+lsndeMZno0wYxzzjkn3HvvveGb3/xmeOmll8Kxxx4b1qxZM+lzZ511VnjqqafWv6688spZSSwAAAAAAMCszPS4/fbbJ/37hhtuCDvvvHNYuXJlOOKII9b//TWveU1YsGDBTHYNAAAAAACQb02PsbGx9r/z58+f9Pcvf/nLYccddwwHHHBAuOiii8Lzzz+/yX2sXbs2jI+PT3oBAAAAAAAknemxoXXr1oXzzz8/HH744W1wY8IHPvCBsMcee4SFCxeGH/zgB+HjH/94ePDBB9u1QDa1Tsjll1++uckAAAAAAABo9fqbufLI2WefHf75n/85fOc73wm77bbbJj/3rW99Kxx99NHhoYceCm984xs3OtOjeU1oZnosWrQo2yJPJS9knnLRvq4uZD6si1vlXGw0tZyL9pW6iFssiyiXt2BgjK6WpdRKvial1sHU1zPneK3GfmaYxxYp61DJ7XXO8y65rA7jecVKOa6xoPeWz/NhXci8y/f+XT3vnGODLl/vrsp5bxCr5Po9FWPFzc/T5ulTIyMjsz/T49xzzw1f//rXw9133z1lwKNxyCGHtP/dVNBj3rx57QsAAAAAACDG3JlGoM4777xwyy23hLvuuivsueeeA7e5//772//uuuuuM0rYVBGbmF+alCwm7anPu9RfHnT5F88p015rPUhd1mLallJ/OdBlqX/1EJOvJc/cS6nk2ZI5f0HX1TY59a9Mc86OLfVXqjn3nVqp16zLvxTt6r5rntnT5XuPUu9TS50NOUitbUOXZ8fmrCO1jt9z1rGSx6mxuvqdZskzNWotq12eNdhL9P1a84So0dHR2Q96nHPOOWH58uXhtttuC9tvv31YtWpV+/fmYNttt114+OGH2/ePP/748IY3vKFd0+OCCy4IRxxxRHjLW94yk0MBAAAAAADMyIyCHsuWLWv/e+SRR076+/XXXx9OP/30sM0224Q77rgjXH311WHNmjXt2hwnn3xyuPjii2eWKgAAAAAAgNSPt5pKE+RYsWLFTNMAAAAAAAAQbU78LgAAAAAAAPIT9AAAAAAAAKog6AEAAAAAAFRB0AMAAAAAABi+hcxLMdWC6r1eL9m+p/N+jEFpjzl2yn2nzJNBaU997K6ed86yNEhM2nOed6yYfCs53SVfk5xlLUbJeRp77Jj+O/a8Yq5Z6noQs+9BYsp5zrKYs92LlfJ615rnOdvUlNvn7ENztqmpy3HOOhazbc6y2GU5xzUpdfXeP/WxS21zc35PlHoMXGpfMkjKY+dsz1O3S1Mdu+bv12odr+Xsx7owbjHTAwAAAAAAqIKgBwAAAAAAUAVBDwAAAAAAoAqCHgAAAAAAQBUEPQAAAAAAgCoIegAAAAAAAFUQ9AAAAAAAAKowN1Sm3+9Hbd/r9ZLtf9C2g44ds21svqSUM20xx47N85xlLaWUx46pI6mVXAdTpq3L5x2jy3VsqnwZtO+U7VqX8zRm/4PyLLYcxxy75LFDTNpz9iWxedrVNjW2bYmRs5yX3K6lrAddbltqLYu19iWp2/Oc7UPMeG2Qkq9ZzrJWallNfV45v4fKOV5LeeyU90Qpv4fqctsSo+bxWj/jd9glXG8zPQAAAAAAgCoIegAAAAAAAFUQ9AAAAAAAAKog6AEAAAAAAFRB0AMAAAAAAKiCoAcAAAAAAFAFQQ8AAAAAAKAKgh4AAAAAAEAVBD0AAAAAAIAqCHoAAAAAAABVEPQAAAAAAACqIOgBAAAAAABUQdADAAAAAACogqAHAAAAAABQhbkz+fCyZcva16OPPtr++81vfnO49NJLw3HHHdf++4UXXggf/ehHw4033hjWrl0bFi9eHK699tqwyy67hK7o9/ubvW2v14vad8yxY7aNFXveMfvPed45pc7zlPselPYYXS5rMcfOWR5ixZSHlG1qbJ6mLOc58yX22DmlTFvKclzysUu+3l0dO5TcR+be/+ZK3Yd2dexQ8tgi5bG7PIYu9dg5+5KSy/EgOceKg6Rs10oeQ8dIme7U5bir/XdOOfupWvM19ry7OjbI2eb1Eud5CfdjM5rpsdtuu4VPf/rTYeXKleF73/teeNe73hXe8573hB/96Eft+xdccEH42te+Fm666aawYsWK8OSTT4aTTjopVdoBAAAAAADW6/Ujwyvz588PV111VTjllFPCTjvtFJYvX97+/8YDDzwQ9t9//3DPPfeEQw89dFr7Gx8fD6Ojo2FsbCyMjIyELin5lyQpDetMj9S/BJ9q+y6XtZJ/xZpztkWpMz1yRv9j9x2j5pkeDNdMj1rLQ6ltx7D2kbOx/1xKnukxrG1LV2d61FpHuqzka5KyfufsS3KOobt6PWNpW4Zr1kHKY8dK+R3XICU/5SHleQ/Sz5jnqcbIM4kbbPaaHi+//HL7GKs1a9aEww47rJ398dJLL4Vjjjlm/Wf222+/sPvuu7dBDwAAAAAAgGLW9Gj8+7//exvkaNbveN3rXhduueWW8Eu/9Evh/vvvD9tss03YYYcdJn2+Wc9j1apVm9xfs/ZH89owYgMAAAAAADBTM57pse+++7YBjvvuuy+cffbZ4bTTTgv/8R//ETbX0qVL22kpE69FixZt9r4AAAAAAIDhNeOgRzObY++99w4HHXRQG7A48MADw1//9V+HBQsWhBdffDGsXr160ueffvrp9r1Nueiii9rncE28Hn/88c07EwAAAAAAYKht9poeE9atW9c+nqoJgmy99dbhzjvvXP/egw8+GB577LH2cVibMm/evHbhkQ1fAAAAAAAASdf0aGZlHHfcce3i5M8++2xYvnx5uOuuu8I3vvGN9tFUZ555ZliyZEmYP39+G7w477zz2oDHoYceOuOENfvbHDErwMfKeexYvV4v27kNOnatBuXpVPkSs+1sbJ/SVGlLna6YY6e8JrHnHXPs2PMquW1Jeeyc55WzrKXMl5zXc5CSj11qe57z2Dnb85rFlLXYPEt57K72JbF1v9ZyXHKbmbKs5dx3rJzjlpj7sUFKbTsGnXfq+5Ja2+uU4/Muj0tyjh1y7bvmMVPO77hKbs9Lvd79jGVpS+17RkGPn/zkJ+FDH/pQeOqpp9qgxFve8pY24PEbv/Eb7fuf/exnw5w5c8LJJ5/czv5YvHhxuPbaa2cloQAAAAAAAFPp9QsL+46Pj2/2LI9GYafTGSl/eVBy1DNG6l9UmOmx5fN8c6/HdI49rL++T3nsktu1lFLmeZfLmv5/8/ItpVrrd1d/MV3yr7lK/tViV9uekvvnQUoe78UouW0ptd1KPW5J+avkLiv1SQxdbq9z3guWXFa7+l1QrFJ/+Z/z2DmfZjKsM3v6BZel6ey7WRd80BIZ0Wt6AAAAAAAAlEDQAwAAAAAAqIKgBwAAAAAAUAVBDwAAAAAAoApzQ2FiF1JpFkJn9sXka8prUvL1Lvm8a823nOfV5Wvi2N3Ks1i1Xu8uX5NadbWsdbk972K6ulxWZmP7UpV8XiWnratlreQ8LfW8S86zWpVch2ot5yWr9bxqHjOl3Het7XnJeZ772NOJH/T6qZepn6EnnngiLFq0KHcyAAAAAACAgjz++ONht91261bQY926deHJJ58M22+/fej1em30pwmCNCczMjKSO3mQhXoA6gHoC0BfAO4NwD0y6AsYVv1+Pzz77LNh4cKFYc6cOd16vFWT4I1FapqAh6AHw049APUA9AWgLwD3BuAeGfQFDKPR0dFpfc5C5gAAAAAAQBUEPQAAAAAAgCoUH/SYN29euOyyy9r/wrBSD0A9AH0B6AvAvQG4RwZ9AQxW3ELmAAAAAAAAVc70AAAAAAAAmA5BDwAAAAAAoAqCHgAAAAAAQBUEPQAAAAAAgCoUH/S45pprwi/+4i+GbbfdNhxyyCHhX//1X3MnCZJYunRpeNvb3ha23377sPPOO4cTTzwxPPjgg5M+c+SRR4Zerzfp9Yd/+IeuCNX45Cc/+aoyvt9++61//4UXXgjnnHNOeMMb3hBe97rXhZNPPjk8/fTTWdMMs60Z97yyHjSvpuw39AXU6O677w4nnHBCWLhwYVveb7311knv9/v9cOmll4Zdd901bLfdduGYY44JP/7xjyd95uc//3n44Ac/GEZGRsIOO+wQzjzzzPDcc89t4TOB2a8DL730Uvj4xz8efvmXfzm89rWvbT/zoQ99KDz55JMD+49Pf/rTLgnV9AWnn376q8r4u9/97kmf0RdQez3Y2H1C87rqqqvWf0Z/AIUHPb761a+GJUuWhMsuuyx8//vfDwceeGBYvHhx+MlPfpI7aTDrVqxY0X6hde+994ZvfvOb7c3NscceG9asWTPpc2eddVZ46qmn1r+uvPJKV4OqvPnNb55Uxr/zne+sf++CCy4IX/va18JNN93U1pnmZv+kk07Kml6Ybf/2b/82qQ40fULjt3/7t9d/Rl9AbZrxTjPWb37wtDHNeOdzn/tcuO6668J9993XfvHb3Bc0wfAJTcDjRz/6UVtnvv71r7dfGnz4wx/egmcBaerA888/394PX3LJJe1/b7755vbHUb/1W7/1qs/+2Z/92aQ+5LzzznNZqKYvaDRBjg3L+Fe+8pVJ7+sLqL0ebFj+m9ff//3ft0GP5geBG9IfMOzmhoJ95jOfaW/qzzjjjPbfzU3OP/3TP7UV+hOf+ETu5MGsuv322yf9+4YbbmhnfKxcuTIcccQR6//+mte8JixYsEDuU625c+dutIyPjY2Fv/u7vwvLly8P73rXu9q/XX/99WH//fdvg4WHHnpohtTC7Ntpp50m/bv5le4b3/jG8Ou//uvr/6YvoDbHHXdc+9qYZpbH1VdfHS6++OLwnve8p/3bl770pbDLLru0v358//vfH/7zP/+zHUs1QcO3vvWt7Wc+//nPh+OPPz785V/+ZftrSehqHRgdHV0fAJ/whS98IRx88MHhscceC7vvvvv6vzezxt0rUGM9mDBv3rxNlnF9AcNQD15Z/m+77bZw1FFHhb322mvS3/UHDLtiZ3q8+OKL7Ze9zdT1CXPmzGn/fc8992RNG2wJzRe8jfnz50/6+5e//OWw4447hgMOOCBcdNFF7S+/oCbN40qaL6eaQVvzS63mZr7R9AnNDKgN+4Xm0VfNjb5+gVo146F/+Id/CL//+7/f/oJrgr6AYfLII4+EVatWTWr/my+Bm0ffTrT/zX+bR1pNBDwazeeb+4dmZgjUeK/Q9AtNuX9loLx5DOiv/uqvto86+d///d9saYQU7rrrrvbHgfvuu284++yzwzPPPLP+PX0Bw6Z51HPz4/DmkZ6vpD9g2BU70+NnP/tZePnll9tfcG2o+fcDDzyQLV2wJaxbty6cf/754fDDD2+DGxM+8IEPhD322KP9QvgHP/hB+2zfZmp7M8UdatB8gdXMcmpuYpqpupdffnl45zvfGX74wx+2X3hts802r7q5b/qF5j2oUfMr9tWrV7fPsJ6gL2DYTLTxG7svmHiv+W/zJdgrZw42Px7RR1Cb5rFuzX3Aqaee2q5hM+GP//iPw6/92q+15f673/1u+wOpZjzVPEEBatA82qp5tO2ee+4ZHn744fCnf/qn7S/im2DHVlttpS9g6Hzxi19sZ3S88pHP+gMoOOgBw6xZ26P5knfDtQwaGz6XulnIsFnM8+ijj24HfM2jT6DrNpzG+5a3vKUNgjSBvn/8x39sF66FYdM80q2pFxs+mkdfADC8mlmvv/M7v9M+9m3ZsmWT3mvWw9xwHNX8WOQjH/lIWLp0aftIIOi65nGGG94PN+W8uQ9uZn8098UwbJrH/zdPR9h2220n/V1/AAU/3qp5fE8TqW+mam2o+bdnlFKzc889t11889vf/nbYbbfdpvxs84Vw46GHHtpCqYMtq5nV8aY3vakt403b3zzqp/nV+4b0C9Tqv//7v8Mdd9wR/uAP/mDKz+kLqN3E2H+q+4Lmvz/5yU8mvd881ufnP/+5eweqC3g0/UOzxseGszw21T809eDRRx/dYmmELal5HG7z3dHE/bC+gGHyL//yL+2TPwbdKzT0BwyjYoMeza9SDjrooHDnnXdOeuRP8+/DDjssa9oghebXWk3A45Zbbgnf+ta32im7g9x///3tf5sZH1Cj5557rp3J1JTxpk/YeuutJ/ULzSCvWfNDv0CNrr/++vZxPb/5m7855ef0BdSuGRM1X2Rt2P6Pj4+3a3VMtP/Nf5ugeLP+04RmPNXcP0wEBqGGgEez9lkTEG/W7Rik6R+adW1e+eg3qMUTTzzRrukxcT+sL2DYZoQ398gHHnjgwM/qDxhGRT/eqpmOddppp7ULEh588MHh6quvDmvWrAlnnHFG7qRBkkdaLV++PNx2223tMxknnj/dLNTZPNan+eK3ef/4449vb3KaNT0uuOCCcMQRR7TTeqEGH/vYx8IJJ5zQPtLqySefDJdddlk76695ZnVTF5oF2pq+oXlWdfPrxvPOO6+9uTn00ENzJx1mVfNFbRP0aMZBzboEE/QF1Bzk3nDmarN4eXOD3rT3u+++e7vW2ac+9amwzz77tEGQSy65pH3s24knnth+fv/992+f9X7WWWeF6667rv2CuPkxSfMolA0fDwddrAPNF7qnnHJK+P73v9/OCG/Wvpy4V2jeb34w2Kxp0AQCjzrqqPZeovl3c6/wu7/7u+H1r399xjOD2akHzatZ7+/kk09uA+HNmOjCCy8Me++9d1i8eHH7eX0BwzAmmvjxx0033RT+6q/+6lXb6w/g/+kX7vOf/3x/991372+zzTb9gw8+uH/vvffmThIk0VTHjb2uv/769v3HHnusf8QRR/Tnz5/fnzdvXn/vvffu/8mf/El/bGzMFaEa73vf+/q77rpr2+b/wi/8Qvvvhx56aP37//M//9P/oz/6o/7rX//6/mte85r+e9/73v5TTz2VNc2Qwje+8Y22D3jwwQcn/V1fQK2+/e1vb3QcdNppp7Xvr1u3rn/JJZf0d9lll3YcdPTRR7+qfjzzzDP9U089tf+6172uPzIy0j/jjDP6zz77bKYzgtmrA4888sgm7xWa7RorV67sH3LIIf3R0dH+tttu299///37f/EXf9F/4YUXXAqqqAfPP/98/9hjj+3vtNNO/a233rq/xx579M8666z+qlWrJu1DX0DtY6LG3/zN3/S32267/urVq1+1vf4A/q9e8z8TARAAAAAAAICuKnZNDwAAAAAAgJkQ9AAAAAAAAKog6AEAAAAAAFRB0AMAAAAAAKiCoAcAAAAAAFAFQQ8AAAAAAKAKgh4AAAAAAEAVBD0AAAAAAIAqCHoAAAAAAABVEPQAAAAAAACqIOgBAAAAAABUQdADAAAAAAAINfg/pWt+/uTk9WMAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.figure(figsize=(20,10))\n", "plt.imshow(nn.h.abs() > 0.99, cmap='gray', interpolation='nearest');" ] }, { "cell_type": "markdown", "id": "c28a836d", "metadata": {}, "source": [ "Si nous avions une colonne avec uniquement des cases blanches, cela signifierait que le neurone est mort et qu'il ne participe plus au fonctionnement général (aucun exemple ne l'a jamais activé), il n'apprendra plus (\"_permanent brain damage_\"). Cela ne semble pas être le cas ici.\n", "\n", "Ce comportement n'est pas propre à la fonction $tanh$ et se retrouve dans toutes les [fonctions d'activation](https://fr.wikipedia.org/wiki/Fonction_d'activation).\n", "\n", "Même si cette manière d'initialiser le réseau donne un _loss_ acceptable, il peut être amélioré en faisant en sorte qu'il y a ait moins de cases blanches: une manière de faire ceci est de réduire la plage de valeurs prises par la fonction de pré-activation `hpreact` et de faire en sorte que ces valeurs soient plus proches de 0." ] }, { "cell_type": "code", "execution_count": 102, "id": "958e7c05", "metadata": {}, "outputs": [], "source": [ "def layers(self):\n", " self.C = torch.randn((self.nb_chars, self.e_dims), generator=self.g)\n", " self.W1 = torch.randn((self.context_size * self.e_dims, self.n_hidden), generator=self.g) * 0.2\n", " self.b1 = torch.randn(self.n_hidden, generator=self.g) * 0.01 # un peu d'entropie\n", " self.W2 = torch.randn((self.n_hidden, self.nb_chars), generator=self.g) * 0.01 # Pour l'entropie\n", " self.b2 = torch.randn(self.nb_chars, generator=self.g) * 0\n", "BengioFFN.layers = layers" ] }, { "cell_type": "code", "execution_count": 82, "id": "cf45e7b6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 1: 3.7371\n", "tensor([[-0.9646, -0.6130, -0.8121, ..., -0.6439, 0.7729, 0.7591],\n", " [-0.8748, 0.5015, -0.7546, ..., 0.5581, 0.2301, -0.5421],\n", " [-0.9646, -0.6130, -0.8121, ..., -0.6439, 0.7729, 0.7591],\n", " ...,\n", " [-0.7323, -0.5527, 0.2453, ..., -0.6356, 0.5811, -0.4778],\n", " [-0.9943, 0.9848, -0.8656, ..., -0.2016, 0.9336, 0.9865],\n", " [ 0.5922, -0.2423, 0.9416, ..., -0.3742, 0.4832, -0.0968]],\n", " grad_fn=)\n" ] } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, 1, mini_batch_size)\n", "print(nn.h)\n" ] }, { "cell_type": "code", "execution_count": 83, "id": "aeb90ee5", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIeZJREFUeJzt3QmQ1NWBP/A33KgcQeVaAVGjYMQjmCAb47KBEpEYDWxtjMaooWDjgruK8SCrKCS7oLGitRZKNhXFVEkOUx5RCVHRaEzQKAlRUClhZcGVI+pyGpHj96/36t9T0zAcM8zxZubzqWqa7t+bnt/r9+vu77zfe68riqIoAgBARlo19g4AAOxOQAEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDITpvQBO3atSu8++67oVOnTqGioqKxdwcAOABxbdjNmzeH3r17h1atWjW/gBLDSZ8+fRp7NwCAWli9enU46qijml9AiT0npQp27ty5sXcHADgAmzZtSh0Mpc/xZhdQSqd1YjgRUACgaTmQ4RkGyQIA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyE6bxt4BAKBhHX3DE/sts3Lm6NCY9KAAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAGjaAWXGjBnhM5/5TOjUqVPo3r17uOCCC8KyZcvKygwbNixUVFSUXb75zW+WlVm1alUYPXp0OOSQQ9LjXHvttWHHjh11UyMAoMlrU5PCzz33XJg4cWIKKTFQfPvb3w5nn312eP3118Ohhx5aWW78+PFh+vTplbdjECnZuXNnCic9e/YMv//978OaNWvC17/+9dC2bdvwH//xH3VVLwCgpQSU+fPnl92eM2dO6gFZtGhROOuss8oCSQwg1XnyySdToHn66adDjx49wqmnnhq+853vhOuvvz7ccsstoV27drWtCwDQTBzUGJSNGzem627dupXd/8ADD4QjjjginHTSSWHKlCnhww8/rNy2cOHCMGjQoBROSkaOHBk2bdoUli5dWu3v2bZtW9pe9QIANF816kGpateuXeGqq64Kn/vc51IQKbnoootCv379Qu/evcOrr76aekbiOJWHHnoobV+7dm1ZOIlKt+O2vY19mTZtWm13FQBoKQEljkVZsmRJeOGFF8runzBhQuX/Y09Jr169wvDhw8OKFSvCscceW6vfFXthJk+eXHk79qD06dOntrsOADTHUzyTJk0Kjz/+eHj22WfDUUcdtc+yQ4YMSdfLly9P13Fsyrp168rKlG7vbdxK+/btQ+fOncsuAEDzVaOAUhRFCicPP/xweOaZZ0L//v33+zOLFy9O17EnJRo6dGh47bXXwvr16yvLPPXUUyl0nHjiiTWvAQDQsk/xxNM6c+fODY8++mhaC6U0ZqRLly6hY8eO6TRO3H7uueeGww8/PI1Bufrqq9MMn5NPPjmVjdOSYxC55JJLwm233ZYe48Ybb0yPHXtKAABq1INyzz33pJk7cTG22CNSuvzsZz9L2+MU4Th9OIaQAQMGhGuuuSaMHTs2PPbYY5WP0bp163R6KF7H3pSvfe1raR2UquumAAAtW5uanuLZlzhwNS7mtj9xls+8efNq8qsBgBbEd/EAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDttGnsHmqqjb3hiv2VWzhzdIPsCAM2NHhQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2LNRWy0XYAID6owcFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAGjaAWXGjBnhM5/5TOjUqVPo3r17uOCCC8KyZcvKynz00Udh4sSJ4fDDDw+HHXZYGDt2bFi3bl1ZmVWrVoXRo0eHQw45JD3OtddeG3bs2FE3NQIAWlZAee6551L4ePHFF8NTTz0Vtm/fHs4+++ywdevWyjJXX311eOyxx8KDDz6Yyr/77rthzJgxldt37tyZwsnHH38cfv/734f7778/zJkzJ0ydOrVuawYANFkVRVEUtf3hv/zlL6kHJAaRs846K2zcuDEceeSRYe7cueEf/uEfUpk333wzDBw4MCxcuDCcccYZ4Ve/+lX44he/mIJLjx49UpnZs2eH66+/Pj1eu3bt9vt7N23aFLp06ZJ+X+fOnUOu38WzcuboOnkcAGjoz7n6+Ayryef3QY1Bib8g6tatW7petGhR6lUZMWJEZZkBAwaEvn37poASxetBgwZVhpNo5MiRaaeXLl16MLsDALT0bzPetWtXuOqqq8LnPve5cNJJJ6X71q5dm3pAunbtWlY2hpG4rVSmajgpbS9tq862bdvSpSSGGQCg+ap1QIljUZYsWRJeeOGFUN/i4Nxp06aFpqaxutAAoKmr1SmeSZMmhccffzw8++yz4aijjqq8v2fPnmnw64YNG8rKx1k8cVupzO6zekq3S2V2N2XKlHQ6qXRZvXp1bXYbAGiOASWOp43h5OGHHw7PPPNM6N+/f9n2wYMHh7Zt24YFCxZU3henIcdpxUOHDk234/Vrr70W1q9fX1kmzgiKg2VOPPHEan9v+/bt0/aqFwCg+WpT09M6cYbOo48+mtZCKY0ZiSNyO3bsmK7HjRsXJk+enAbOxiBx5ZVXplASZ/BEcVpyDCKXXHJJuO2229Jj3HjjjemxYxABAKhRQLnnnnvS9bBhw8ruv++++8Jll12W/n/HHXeEVq1apQXa4sDWOEPn7rvvrizbunXrdHroiiuuSMHl0EMPDZdeemmYPn261gAAah5QDmTJlA4dOoRZs2aly97069cvzJs3rya/GgBoQWo9iwcAyM/RdbTYaGPzZYEAQHb0oABAI7Nu1p70oAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALJjoTYAaAKObiZL2B8oPSgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgO20aewcAoKk6+oYn9ltm5czRDbIvzY0eFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYs1NYEWAgIgJZGDwoAkB0BBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdgQUACA7AgoAkB0BBQDIjqXum8Ay9gDQ0uhBAQCyI6AAANkRUACA7BiDAgD1yFjD2tGDAgA0/YDy/PPPh/POOy/07t07VFRUhEceeaRs+2WXXZbur3o555xzysp88MEH4eKLLw6dO3cOXbt2DePGjQtbtmw5+NoAAC0zoGzdujWccsopYdasWXstEwPJmjVrKi8/+clPyrbHcLJ06dLw1FNPhccffzyFngkTJtSuBgBAs1PjMSijRo1Kl31p37596NmzZ7Xb3njjjTB//vzw8ssvh9NPPz3dd9ddd4Vzzz033H777alnBgBo2eplDMpvfvOb0L1793DCCSeEK664Irz//vuV2xYuXJhO65TCSTRixIjQqlWr8NJLL1X7eNu2bQubNm0quwAAzVedB5R4eufHP/5xWLBgQbj11lvDc889l3pcdu7cmbavXbs2hZeq2rRpE7p165a2VWfGjBmhS5culZc+ffrU9W4DAM15mvGFF15Y+f9BgwaFk08+ORx77LGpV2X48OG1eswpU6aEyZMnV96OPShCCgA0X/W+DsoxxxwTjjjiiLB8+fIUUOLYlPXr15eV2bFjR5rZs7dxK3FMS7xQ/3PxV84c7WkGoPmvg/LOO++kMSi9evVKt4cOHRo2bNgQFi1aVFnmmWeeCbt27QpDhgyp790BAJpjD0pcryT2hpS8/fbbYfHixWkMSbxMmzYtjB07NvWGrFixIlx33XXhuOOOCyNHjkzlBw4cmMapjB8/PsyePTts3749TJo0KZ0aMoMHAKhVD8orr7wSTjvttHSJ4tiQ+P+pU6eG1q1bh1dffTV86UtfCscff3xagG3w4MHht7/9bdkpmgceeCAMGDAgnfKJ04vPPPPM8F//9V9aBACoXQ/KsGHDQlEUe93+61//er+PEXta5s6dW9NfDQC0EL6LBwDIjoACAGRHQAEAsiOgAADZEVAAgJa3kiz5rBILAE2FHhQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyYxYPAFTD7MjGpQcFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQu10WiLF62cOdqzD0C1BBQAWhyrxObPKR4AIDsCCgCQHQEFAMiOMSgA1Pt4DoPiqSk9KABAdvSgkPXIdn+ZAbRMAgoAWfAHCVUJKDSa3HprAMiHMSgAQHYEFAAgO07xQA04Rw7QMAQUAJoV49uaB6d4AIDs6EGhyXPaBaD5EVAAqHdOu1BTTvEAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOacaAtWSA7OhBAQCyowcFoIWyCjM504MCAGRHDwoATYYl81sOAQUypfud+j5+IGcCCjRzPqhw/NAUGYMCAGRHQAEAsuMUDy2C8RwATYuAAhwQIQ9oSE7xAADZ0YMC1Jnm2suSW73MzKIlqHEPyvPPPx/OO++80Lt371BRUREeeeSRsu1FUYSpU6eGXr16hY4dO4YRI0aEt956q6zMBx98EC6++OLQuXPn0LVr1zBu3LiwZcuWg68NANAye1C2bt0aTjnllPCNb3wjjBkzZo/tt912W/jP//zPcP/994f+/fuHm266KYwcOTK8/vrroUOHDqlMDCdr1qwJTz31VNi+fXu4/PLLw4QJE8LcuXPrplYAGcqtJwaaVUAZNWpUulQn9p7ceeed4cYbbwznn39+uu/HP/5x6NGjR+ppufDCC8Mbb7wR5s+fH15++eVw+umnpzJ33XVXOPfcc8Ptt9+eemYAgJatTsegvP3222Ht2rXptE5Jly5dwpAhQ8LChQtTQInX8bROKZxEsXyrVq3CSy+9FL785S/v8bjbtm1Ll5JNmzbV5W4DZMP4EqiHWTwxnESxx6SqeLu0LV537969bHubNm1Ct27dKsvsbsaMGSnolC59+vSpy90GADLTJKYZT5kyJWzcuLHysnr16sbeJQCgqQSUnj17put169aV3R9vl7bF6/Xr15dt37FjR5rZUyqzu/bt26cZP1UvAEDzVadjUOKsnRgyFixYEE499dTK8SJxbMkVV1yRbg8dOjRs2LAhLFq0KAwePDjd98wzz4Rdu3alsSrQEhhnAFDHASWuV7J8+fKygbGLFy9OY0j69u0brrrqqvDd7343fPKTn6ycZhxn5lxwwQWp/MCBA8M555wTxo8fH2bPnp2mGU+aNCkNoDWDBwCoVUB55ZVXwt///d9X3p48eXK6vvTSS8OcOXPCddddl9ZKieuaxJ6SM888M00rLq2BEj3wwAMplAwfPjzN3hk7dmxaOwUAoFYBZdiwYWm9k72Jq8tOnz49XfYm9rZYlA0AaNKzeACAlkVAAQCy49uMoY6ZoQNw8PSgAADZEVAAgOw4xQNNmNNJnkNorgQU+P982APkwykeACA7elAA6oAeOKhbelAAgOwIKABAdgQUACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAADZsVAbkN2CZitnjm6QfQHyJaAA2RFiAKd4AIDs6EEBmi3fjwNNlx4UACA7AgoAkB0BBQDIjoACAGRHQAEAsiOgAADZEVAAgOwIKABAdizUBjRJFmGD5k0PCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgDQ/APKLbfcEioqKsouAwYMqNz+0UcfhYkTJ4bDDz88HHbYYWHs2LFh3bp1db0bAEATVi89KJ/61KfCmjVrKi8vvPBC5barr746PPbYY+HBBx8Mzz33XHj33XfDmDFj6mM3AIAmqk29PGibNqFnz5573L9x48bwox/9KMydOzd84QtfSPfdd999YeDAgeHFF18MZ5xxRn3sDgDQxNRLD8pbb70VevfuHY455phw8cUXh1WrVqX7Fy1aFLZv3x5GjBhRWTae/unbt29YuHBhfewKANAE1XkPypAhQ8KcOXPCCSeckE7vTJs2LXz+858PS5YsCWvXrg3t2rULXbt2LfuZHj16pG17s23btnQp2bRpU13vNgDQnAPKqFGjKv9/8sknp8DSr1+/8POf/zx07NixVo85Y8aMFHQAgJah3qcZx96S448/PixfvjyNS/n444/Dhg0bysrEWTzVjVkpmTJlShq/UrqsXr26vncbAGjOAWXLli1hxYoVoVevXmHw4MGhbdu2YcGCBZXbly1blsaoDB06dK+P0b59+9C5c+eyCwDQfNX5KZ5vfetb4bzzzkundeIU4ptvvjm0bt06fPWrXw1dunQJ48aNC5MnTw7dunVLQePKK69M4cQMHgCg3gLKO++8k8LI+++/H4488shw5plnpinE8f/RHXfcEVq1apUWaIsDX0eOHBnuvvvuut4NAKAJqyiKoghNTJzFE3tj4niU+jjdc/QNT9T5YwJAU7Jy5uhG/fz2XTwAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsCCgAQHYEFAAgOwIKAJAdAQUAyI6AAgBkR0ABALIjoAAA2RFQAIDsNGpAmTVrVjj66KNDhw4dwpAhQ8If/vCHxtwdAKClB5Sf/exnYfLkyeHmm28Of/zjH8Mpp5wSRo4cGdavX99YuwQAtPSA8v3vfz+MHz8+XH755eHEE08Ms2fPDocccki49957G2uXAIBMtGmMX/rxxx+HRYsWhSlTplTe16pVqzBixIiwcOHCPcpv27YtXUo2btyYrjdt2lQv+7dr24f18rgA0FRsqofP2NJjFkWRZ0B57733ws6dO0OPHj3K7o+333zzzT3Kz5gxI0ybNm2P+/v06VOv+wkALVWXO+vvsTdv3hy6dOmSX0CpqdjTEserlOzatSt88MEH4fDDDw8VFRV1nu5i8Fm9enXo3LlzaG7Ur+nThk1bc2+/llDH5l6/+qxj7DmJ4aR37977LdsoAeWII44IrVu3DuvWrSu7P97u2bPnHuXbt2+fLlV17dq1XvcxNkhzPfAi9Wv6tGHT1tzbryXUsbnXr77quL+ek0YdJNuuXbswePDgsGDBgrJekXh76NChjbFLAEBGGu0UTzxlc+mll4bTTz89fPaznw133nln2Lp1a5rVAwC0bI0WUL7yla+Ev/zlL2Hq1Klh7dq14dRTTw3z58/fY+BsQ4unkuLaLLufUmou1K/p04ZNW3Nvv5ZQx+Zev1zqWFEcyFwfAIAG5Lt4AIDsCCgAQHYEFAAgOwIKAJCdFhdQ/v3f/z387d/+bfpiwgNd7C2OI46zjXr16hU6duyYvjPorbfeKisTV7a9+OKL04I28XHHjRsXtmzZEhpaTfdj5cqVaTXe6i4PPvhgZbnqtv/0pz8NjaE2z/WwYcP22P9vfvObZWVWrVoVRo8enY6N7t27h2uvvTbs2LEj5F6/WP7KK68MJ5xwQjo++/btG/7lX/6l8jurcmjDWbNmhaOPPjp06NAhDBkyJPzhD3/YZ/l47A0YMCCVHzRoUJg3b16NX5MNqSb1++EPfxg+//nPh0984hPpEvd99/KXXXbZHm11zjnnhKZQvzlz5uyx7/Hncm6/mtaxuveTeInvHzm24fPPPx/OO++8tHpr3I9HHnlkvz/zm9/8Jnz6059Os3iOO+641K4H+7qusaKFmTp1avH973+/mDx5ctGlS5cD+pmZM2emso888kjx5z//ufjSl75U9O/fv/jrX/9aWeacc84pTjnllOLFF18sfvvb3xbHHXdc8dWvfrVoaDXdjx07dhRr1qwpu0ybNq047LDDis2bN1eWi4fKfffdV1auav0bUm2e67/7u78rxo8fX7b/GzduLHseTjrppGLEiBHFn/70p2LevHnFEUccUUyZMqXIvX6vvfZaMWbMmOKXv/xlsXz58mLBggXFJz/5yWLs2LFl5RqrDX/6058W7dq1K+69995i6dKlqR26du1arFu3rtryv/vd74rWrVsXt912W/H6668XN954Y9G2bdtUz5q8JhtKTet30UUXFbNmzUrH2RtvvFFcdtllqS7vvPNOZZlLL700HQdV2+qDDz4oGkNN6xePsc6dO5ft+9q1a8vK5NR+tanj+++/X1a/JUuWpGM21j3HNpw3b17xb//2b8VDDz2U3gcefvjhfZb/7//+7+KQQw5Jn5PxNXjXXXel+s2fP7/Wz1lttLiAUhIPpAMJKLt27Sp69uxZfO9736u8b8OGDUX79u2Ln/zkJ+l2bMDY6C+//HJlmV/96ldFRUVF8b//+79FQ6mr/Tj11FOLb3zjG2X3HchBnXMdY0D513/9132+gFu1alX2RnrPPfekN9pt27YVTa0Nf/7zn6c3j+3btzd6G372s58tJk6cWHl7586dRe/evYsZM2ZUW/4f//Efi9GjR5fdN2TIkOKf/umfDvg1mXP9dhfDcadOnYr777+/7MPt/PPPL3JQ0/rt7701t/ariza84447Uhtu2bIlyzas6kDeB6677rriU5/6VNl9X/nKV4qRI0fW2XN2IFrcKZ6aevvtt9NCcrELsur3CMTurIULF6bb8Tp2xcdVcUti+VatWoWXXnqpwfa1LvZj0aJFYfHixem0wu4mTpyYvkcprvx77733HtDXZedUxwceeCDt/0knnZS+gPLDDz8se9x4KqHqQoEjR45MX5i1dOnS0FDq6liKp3fiKaI2bdo0aht+/PHH6Ziq+vqJdYm3S6+f3cX7q5YvtUWp/IG8JhtKbeq3u3gcbt++PXTr1m2PLvZ4qjGeurviiivC+++/HxpabesXT0n269cvfdnc+eefX/Yayqn96qoNf/SjH4ULL7wwHHroodm1YW3s7zVYF8/ZgWgS32bcmOILKdp9hdt4u7QtXseDsKr4wRDfcEplGmpfD3Y/4gtt4MCBaZxOVdOnTw9f+MIX0viMJ598MvzzP/9zehOKYx0aUm3reNFFF6U3zHgO9tVXXw3XX399WLZsWXjooYcqH7e6Ni5ta0pt+N5774XvfOc7YcKECY3ehnFfdu7cWe1z++abb1b7M3tri6qvt9J9eyvTUGpTv93FYzEel1Xf7ONYhTFjxoT+/fuHFStWhG9/+9th1KhR6c0/ftFqzvWLH8Yx/J588skpKN9+++3p/SSGlKOOOiqr9quLNozjLpYsWZLeO6vKpQ1rY2+vwfgH21//+tfwf//3fwd93LeYgHLDDTeEW2+9dZ9l3njjjTTorjnX72DFA2/u3Lnhpptu2mNb1ftOO+209L1J3/ve9+rsw62+61j1wzr2lMTBecOHD09vHMcee2xoLm0Y30DiQL0TTzwx3HLLLQ3ahtTczJkz00Dl+Jd21YGk8a/xqsdr/LCPx2ksF4/bnMUvfK36pa8xnMQ/en7wgx+k4NzcxGAS2yj2SlbVlNswF80ioFxzzTVpxPS+HHPMMbV67J49e6brdevWpQ+1kng7fn9Qqcz69evLfi7O/oizK0o/3xD1O9j9+MUvfpG6m7/+9a/vt2zsjo1vNtu2bauT72poqDpW3f9o+fLl6U0j/uzuI9BjG0dNpQ03b96c/mrr1KlTePjhh0Pbtm0btA2rE08nxb8WS89lSby9t/rE+/dV/kBekw2lNvUriT0LMaA8/fTT6cNrf8dG/F3xeG3ID7eDqV9JPA5jII77nlv7HWwdY8iPATP2Tu5PY7VhbeztNRhPG8dZV/H5Otjj4oAULVRNB8nefvvtlffF2R/VDZJ95ZVXKsv8+te/brRBsrXdjziQdPeZH3vz3e9+t/jEJz5RNLS6eq5feOGF9DhxBkHVQbJVR6D/4Ac/SINkP/rooyL3+sVj8owzzkhtuHXr1qzaMA6mmzRpUtlgur/5m7/Z5yDZL37xi2X3DR06dI9Bsvt6TTakmtYvuvXWW9OxtXDhwgP6HatXr07HwKOPPlo0hfrtPgj4hBNOKK6++uos2+9g6hg/R+J+v/fee1m3YW0GycZZjVXFmYS7D5I9mOPiQLS4gPI///M/aXpfaSpt/H+8VJ1SG19McTpW1SlxcfpUPLBeffXVNDK7umnGp512WvHSSy+lD784zbOxphnvaz/iVMZYv7i9qrfeeiu9eOKMkd3F6as//OEP0zTPWO7uu+9OU9DilO3GUNM6xqm306dPTx/6b7/9dmrHY445pjjrrLP2mGZ89tlnF4sXL07T6Y488shGm2Zck/rFN/c4y2XQoEGprlWnNcZ6NXYbxumI8U18zpw5KYBNmDAhvZ5KM6YuueSS4oYbbiibZtymTZv0ARan4d58883VTjPe32uyodS0fnHf4wyrX/ziF2VtVXoPitff+ta3UniJx+vTTz9dfPrTn07HQUOG5drWL763xlC9YsWKYtGiRcWFF15YdOjQIU1FzbH9alPHkjPPPDPNbtldbm24efPmys+6GFDiUhvx//HzMIp1i3XcfZrxtddem16DcVp8ddOM9/Wc1YUWF1Di1K/YQLtfnn322T3WiyiJif+mm24qevTokRpk+PDhxbJly/aYFx8/RGLoiX8ZXX755WWhp6Hsbz/ii2X3+kbxg7hPnz4pBe8uhpY49Tg+5qGHHprW6Jg9e3a1ZXOs46pVq1IY6datW2q/uK5IfOFVXQclWrlyZTFq1KiiY8eOaQ2Ua665pmyabq71i9fVHdPxEsvm0IZxHYW+ffumD+b4l1dc46Uk9vrE1+Xu06SPP/74VD5Od3ziiSfKth/Ia7Ih1aR+/fr1q7atYhCLPvzwwxSUY0COwSyWj2tM1OUbf33W76qrrqosG9vn3HPPLf74xz9m3X61OUbffPPN1G5PPvnkHo+VWxs+u5f3iFKd4nWs4+4/E98z4vMR/6Cr+pl4IM9ZXaiI/9TdCSMAgINnHRQAIDsCCgCQHQEFAMiOgAIAZEdAAQCyI6AAANkRUACA7AgoAEB2BBQAIDsCCgCQHQEFAMiOgAIAhNz8PzqyDlBY37ZJAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.hist(nn.h.view(-1).tolist(), 50);" ] }, { "cell_type": "code", "execution_count": 84, "id": "0a1f9e0e", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAGTxJREFUeJzt3XuMVOX9+PEPF0HlWqhACCC0NUXrLcULxKb1QkGLiVZsbWIsGmKjQSLSqNBQjLYJBE3xUgRjGqFVg7EJGvDSEqz4h6iIJVGsJCalkCKgNbDIL1yE/eU5ye6X9crCwnx29/VKJrMz5+zs2Rncefuc55zpUF9fXx8AAIl0rPUGAAB8lkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0OkcrdODAgdi8eXP06NEjOnToUOvNAQAOQTk37M6dO2PgwIHRsWPHthcoJU4GDx5c680AAA7Dpk2bYtCgQW0vUMrIScMv2LNnz1pvDgBwCOrq6qoBhob38TYXKA27dUqcCBQAaF0OZXqGSbIAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQ613oDgNZh6LTnvnadDbPHHZNtAdo+IygAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQECgCQjkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0BAoAkI5AAQDSESgAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIJ3Otd4AoH0ZOu25r11nw+xxx2RbgLyMoAAA6QgUACAdgQIApCNQAIB0BAoAkI5AAQDScZgxcEiH/h5LDkUGjKAAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEDbCpTZs2dHhw4dYsqUKY337d69OyZNmhR9+/aN7t27x/jx42Pr1q1Nvm/jxo0xbty4OPHEE6Nfv35x++23x6effnokmwIAtCGHHSirV6+ORx55JM4888wm9992222xdOnSePrpp2PlypWxefPmuOqqqxqX79+/v4qTvXv3xquvvhqLFi2KhQsXxsyZM4/sNwEA2negfPLJJ3HttdfGo48+Gt/4xjca79+xY0f86U9/ij/84Q9x8cUXx4gRI+Kxxx6rQuS1116r1vn73/8e7777bjz++ONx9tlnx2WXXRa/+93vYt68eVW0AAAcVqCUXThlFGT06NFN7l+zZk3s27evyf3Dhw+PIUOGxKpVq6rb5fqMM86I/v37N64zduzYqKuri3Xr1n3hz9uzZ0+1/OALANB2NfuzeBYvXhxvvfVWtYvns7Zs2RJdunSJ3r17N7m/xEhZ1rDOwXHSsLxh2ReZNWtW3H333c3dVACgPYygbNq0KW699dZ44okn4vjjj49jZfr06dXuo4ZL2Q4AoO1qVqCUXTjbtm2L73//+9G5c+fqUibCPvjgg9XXZSSkzCPZvn17k+8rR/EMGDCg+rpcf/aonobbDet8VteuXaNnz55NLgBA29WsQLnkkkvi7bffjrVr1zZezjnnnGrCbMPXxx13XKxYsaLxe9avX18dVjxq1Kjqdrkuj1FCp8Hy5cur6DjttNNa8ncDANrDHJQePXrE6aef3uS+bt26Vec8abh/4sSJMXXq1OjTp08VHZMnT66iZOTIkdXyMWPGVCFy3XXXxZw5c6p5JzNmzKgm3paREqBlDZ32nKcUaPuTZL/O3Llzo2PHjtUJ2srRN+UInYcffrhxeadOnWLZsmVx8803V+FSAmfChAlxzz33tPSmAADtNVBefvnlJrfL5NlyTpNy+TInn3xyPP/880f6owGANspn8QAA6QgUAKDtz0EByDL5d8PsccdkW4CWZwQFAEhHoAAA6djFA62Yc5wAbZURFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQECgCQjkABANIRKABAOp1rvQFA2zF02nO13gSgjTCCAgCkI1AAgHQECgCQjkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0BAoAkI5AAQDSESgAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQECgCQjkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0BAoAkI5AAQDSESgAQDqda70BwBcbOu05Tw3QbhlBAQDSESgAQDoCBQBIR6AAAK17kuz8+fOry4YNG6rb3/ve92LmzJlx2WWXVbd3794dv/71r2Px4sWxZ8+eGDt2bDz88MPRv3//xsfYuHFj3HzzzfGPf/wjunfvHhMmTIhZs2ZF587m6wI5JxpvmD2uRR4HOEojKIMGDYrZs2fHmjVr4s0334yLL744rrjiili3bl21/LbbboulS5fG008/HStXrozNmzfHVVdd1fj9+/fvj3HjxsXevXvj1VdfjUWLFsXChQuryAEAaNChvr6+Po5Anz594t57742rr746TjrppHjyySerr4v33nsvTj311Fi1alWMHDkyXnjhhbj88surcGkYVVmwYEHceeed8eGHH0aXLl0O6WfW1dVFr169YseOHdGzZ88j2XxIy2HGeRhBgZbRnPfvw56DUkZDyq6cXbt2xahRo6pRlX379sXo0aMb1xk+fHgMGTKkCpSiXJ9xxhlNdvmU3UBlgxtGYb5I2V1U1jn4AgC0Xc0OlLfffruaO9K1a9e46aabYsmSJXHaaafFli1bqhGQ3r17N1m/xEhZVpTrg+OkYXnDsi9T5qiU4mq4DB48uLmbDQC05UD57ne/G2vXro3XX3+9muxaJrm+++67cTRNnz69Gg5quGzatOmo/jwAoLaafehMGSX5zne+U309YsSIWL16dTzwwANxzTXXVJNft2/f3mQUZevWrTFgwIDq63L9xhtvNHm8srxh2ZcpozXlAgC0D0d8HpQDBw5Uc0RKrBx33HGxYsWKxmXr16+vDisuc1SKcl12EW3btq1xneXLl1cTZcpuIgCAZo+glF0t5ZwnZeLrzp07qyN2Xn755fjb3/5WzQ2ZOHFiTJ06tTqyp0TH5MmTqygpR/AUY8aMqULkuuuuizlz5lTzTmbMmBGTJk0yQgIAHF6glJGPX/7yl/HBBx9UQXLmmWdWcfLjH/+4Wj537tzo2LFjjB8/vsmJ2hp06tQpli1bVs1dKeHSrVu3ag7LPffc05zNAADauCM+D0otOA8K7YHzoOThPCjQis6DAgBwtAgUACAdgQIApCNQAIB0BAoAkI5AAQDSESgAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCk07nWGwDt0dBpz9V6EwBSM4ICAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASMdhxgAtcFj4htnjPI/QgoygAADpCBQAIB2BAgCkI1AAgHQECgCQjkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0BAoAkI5AAQDSESgAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdDrXegOgrRk67blabwJAq2cEBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQECgCQjkABAFp3oMyaNSvOPffc6NGjR/Tr1y+uvPLKWL9+fZN1du/eHZMmTYq+fftG9+7dY/z48bF169Ym62zcuDHGjRsXJ554YvU4t99+e3z66act8xsBAO0rUFauXFnFx2uvvRbLly+Pffv2xZgxY2LXrl2N69x2222xdOnSePrpp6v1N2/eHFdddVXj8v3791dxsnfv3nj11Vdj0aJFsXDhwpg5c2bL/mYAQKvVob6+vv5wv/nDDz+sRkBKiPzwhz+MHTt2xEknnRRPPvlkXH311dU67733Xpx66qmxatWqGDlyZLzwwgtx+eWXV+HSv3//ap0FCxbEnXfeWT1ely5dvvbn1tXVRa9evaqf17Nnz8PdfDgqhk57zjPbDm2YPa7WmwDpNef9+4jmoJQfUPTp06e6XrNmTTWqMnr06MZ1hg8fHkOGDKkCpSjXZ5xxRmOcFGPHjq02et26dV/4c/bs2VMtP/gCALRdhx0oBw4ciClTpsQFF1wQp59+enXfli1bqhGQ3r17N1m3xEhZ1rDOwXHSsLxh2ZfNfSnF1XAZPHjw4W42ANCWA6XMRXnnnXdi8eLFcbRNnz69Gq1puGzatOmo/0wAoHY6H8433XLLLbFs2bJ45ZVXYtCgQY33DxgwoJr8un379iajKOUonrKsYZ033nijyeM1HOXTsM5nde3atboAAO1Ds0ZQynzaEidLliyJl156KYYNG9Zk+YgRI+K4446LFStWNN5XDkMuhxWPGjWqul2u33777di2bVvjOuWIoDJZ5rTTTjvy3wgAaF8jKGW3TjlC59lnn63OhdIwZ6TMCznhhBOq64kTJ8bUqVOribMlOiZPnlxFSTmCpyiHJZcQue6662LOnDnVY8yYMaN6bKMkAECzA2X+/PnV9YUXXtjk/sceeyyuv/766uu5c+dGx44dqxO0laNvyhE6Dz/8cOO6nTp1qnYP3XzzzVW4dOvWLSZMmBD33HOPVwQAOPLzoNSK86CQmfOg8GWcK4X2rq4Z50E5rEmy0F6JD4Bjw4cFAgDpCBQAIB2BAgCkI1AAgHQECgCQjkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0BAoAkI5AAQDSESgAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQECgCQjkABANIRKABAOp1rvQGQxdBpz9V6E2jjDuXf2IbZ447JtkB2RlAAgHQECgCQjkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0BAoAkI5AAQDSESgAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQECgCQjkABANIRKABAOgIFAEinc603AID/M3Tac1/7dGyYPc5TRpvX7EB55ZVX4t577401a9bEBx98EEuWLIkrr7yycXl9fX3cdddd8eijj8b27dvjggsuiPnz58cpp5zSuM7HH38ckydPjqVLl0bHjh1j/Pjx8cADD0T37t1b7jeDZv7RB6AV7+LZtWtXnHXWWTFv3rwvXD5nzpx48MEHY8GCBfH6669Ht27dYuzYsbF79+7Gda699tpYt25dLF++PJYtW1ZFz69+9asj+00AgPY7gnLZZZdVly9SRk/uv//+mDFjRlxxxRXVfX/+85+jf//+8cwzz8QvfvGL+Ne//hUvvvhirF69Os4555xqnYceeih+8pOfxH333RcDBw480t8JAGjlWnSS7L///e/YsmVLjB49uvG+Xr16xfnnnx+rVq2qbpfr3r17N8ZJUdYvu3rKiMsX2bNnT9TV1TW5AABtV4sGSomTooyYHKzcblhWrvv169dkeefOnaNPnz6N63zWrFmzqtBpuAwePLglNxsASKZVHGY8ffr02LFjR+Nl06ZNtd4kAKC1BMqAAQOq661btza5v9xuWFaut23b1mT5p59+Wh3Z07DOZ3Xt2jV69uzZ5AIAtF0tGijDhg2rImPFihWN95X5ImVuyahRo6rb5bocflwOU27w0ksvxYEDB6q5KgAAzT6K55NPPon333+/ycTYtWvXVnNIhgwZElOmTInf//731XlPSrD89re/rY7MaThXyqmnnhqXXnpp3HjjjdWhyPv27YtbbrmlOsLHETwAwGEFyptvvhkXXXRR4+2pU6dW1xMmTIiFCxfGHXfcUZ0rpZzXpIyU/OAHP6gOKz7++OMbv+eJJ56oouSSSy5pPFFbOXcKAEDRob6cvKSVKbuNytE8ZcKs+SgcCmeSpS1xqntaq+a8f7eKo3gAgPZFoAAA6QgUACAdgQIApCNQAIB0BAoA0PrPgwJA2zi03uHKZCZQaPWc44T2xr952gO7eACAdAQKAJCOQAEA0hEoAEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQECgCQjkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0BAoAkE7nWm8AALUxdNpzX7vOhtnjjsm2wGcZQQEA0jGCQqv/PzwA2h6BQs2IDwC+jF08AEA6AgUASEegAADpCBQAIB2BAgCkI1AAgHQECgCQjkABANJxojYAvpTP66FWjKAAAOkIFAAgHYECAKQjUACAdAQKAJCOo3io2cx/APgyRlAAgHQECgCQjl08ABwRJ3PjaDCCAgCkI1AAgHQECgCQjkABANIRKABAOgIFAEhHoAAA6QgUACAdgQIApCNQAIB0nOqeJnwKMXA0OB0+zWUEBQBIxwgKACkYZeFgRlAAgHQECgCQTk138cybNy/uvffe2LJlS5x11lnx0EMPxXnnnVfLTWq1DI0C0JbULFCeeuqpmDp1aixYsCDOP//8uP/++2Ps2LGxfv366NevX602C4B2cKThhtnjWuRxOHo61NfX10cNlCg599xz449//GN1+8CBAzF48OCYPHlyTJs27Su/t66uLnr16hU7duyInj17Rlvn0F+AY0/EtLzmvH/XZARl7969sWbNmpg+fXrjfR07dozRo0fHqlWrPrf+nj17qkuD8os1/KLtwYE9/6/WmwDQ7rSX95haPKeHMjZSk0D56KOPYv/+/dG/f/8m95fb77333ufWnzVrVtx9992fu7+MuADA0dDrfs/r0bJz585qJKXVnweljLSU+SoNyu6gjz/+OPr27RsdOnSoaQmWSNq0aVO72NWUmdciB69DHl6LHLwOTZWRkxInAwcOjK9Tk0D55je/GZ06dYqtW7c2ub/cHjBgwOfW79q1a3U5WO/evSOLEicCJQevRQ5ehzy8Fjl4Hf7P142c1PQ8KF26dIkRI0bEihUrmoyKlNujRo2qxSYBAInUbBdP2WUzYcKEOOecc6pzn5TDjHft2hU33HBDrTYJAGjvgXLNNdfEhx9+GDNnzqxO1Hb22WfHiy+++LmJs5mV3U533XXX53Y/4bVor/w3kYfXIgevQys8DwoAwJfxWTwAQDoCBQBIR6AAAOkIFAAgHYHSwspnBpUjksoZbteuXdvSD8/X2LBhQ0ycODGGDRsWJ5xwQnz729+ujrQqn//E0Tdv3rwYOnRoHH/88dUHgr7xxhue9mOofCxI+RDWHj16VJ8Kf+WVV1afEE9tzZ49u3pPmDJlipeiGQRKC7vjjjsO6RS+HB3ls5zKSf8eeeSRWLduXcydOzcWLFgQv/nNbzzlR9lTTz1Vnd+oBOFbb70VZ511VowdOza2bdvmuT9GVq5cGZMmTYrXXnstli9fHvv27YsxY8ZU55iiNlavXl39PTrzzDO9BM1VDjOmZTz//PP1w4cPr1+3bl05dLv+n//8p6c2gTlz5tQPGzas1pvR5p133nn1kyZNary9f//++oEDB9bPmjWrptvVnm3btq36W7Ry5cpab0q7tHPnzvpTTjmlfvny5fU/+tGP6m+99dZab1KrYgSlhZTPEbrxxhvjL3/5S5x44okt9bC0gB07dkSfPn08l0dR2YW2Zs2aGD16dON9HTt2rG6vWrXKc1/Df/uFf/+1UUazxo0b1+S/Cw5dq/g04+zKue6uv/76uOmmm6pT95d5EOTw/vvvx0MPPRT33XdfrTelTfvoo49i//79nzsTdLlddrtx7JVdnWXOwwUXXBCnn366l+AYW7x4cbWrs+zi4fAYQfkK06ZNqyY2fdWl/PEtb4Dl46OnT59+mC8DLfVaHOy///1vXHrppfGzn/2sGt2C9vZ/7++88071RsmxtWnTprj11lvjiSeeqCaMc3ic6v4rlM8K+t///veVT+C3vvWt+PnPfx5Lly6t3iQblP+b7NSpU1x77bWxaNGiw3x5aO5rUT4pu9i8eXNceOGFMXLkyFi4cGG1u4Gju4un7Nr861//Wh050qB8IOj27dvj2Wef9fQfQ7fcckv1nL/yyivVEW0cW88880z89Kc/rd4DDn5PKO8R5W9ROdrz4GV8MYHSAjZu3Bh1dXWNt8ubYzl6ofyxLodaDho0qCV+DIeojJxcdNFFMWLEiHj88cf9IThGyr/18snkZUSxYRfDkCFDqjfLMgLGsdndPHny5FiyZEm8/PLLccopp3jaa6CMqP/nP/9pct8NN9wQw4cPjzvvvNMut0NkDkoLKH+ED9a9e/fqupyDQ5wc+zgpIycnn3xyNe+kjLw0GDBgwDHemvalHGJcRkzKPKwSKvfff391eGv5w8yx263z5JNPVqMn5Vwo5ZPii169elXnBeLYKM/9Z+f9dOvWLfr27StOmkGg0KaUcz+UibHl8tk49MHdR9c111xTBeHMmTOrN8ZywsIXX3zxcxNnOXrmz59fXZdIP9hjjz1WTeSH1sQuHgAgHTMHAYB0BAoAkI5AAQDSESgAQDoCBQBIR6AAAOkIFAAgHYECAKQjUACAdAQKAJCOQAEA0hEoAEBk8/8Bcy1IamXYVBIAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.hist(nn.hpreact.view(-1).tolist(), 50);\n" ] }, { "cell_type": "code", "execution_count": 85, "id": "663faae5", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABj0AAAElCAYAAAC/JSDoAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIwFJREFUeJzt3X+QVeV9P/DPxZUVI7sGBZYtYP2RKI2VtkZ+jAmNkUK0Y4Nim5hkgtaaxiIdIdaETtTYZrKptqmNMdo/WmimwaTOBB3thIxigGYCtiFlUtPKiCUFhx8mZtiFtaCV853nfLvbXX4s7rK79+5zX6+Z4/Xec/beh3PPuc85532e56kURVEEAAAAAADACDeq2gUAAAAAAAAYDEIPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgCw1RYw4fPhy7du2KsWPHRqVSqXZxAAAAAACAKiqKIvbv3x+tra0xatSo6oQeDz30UNx///2xZ8+emD59ejz44IMxY8aME/5dCjymTJkyVMUCAAAAAABGoJ07d8bkyZOHP/T45je/GcuWLYtHHnkkZs6cGQ888EDMnz8/tm7dGhMmTOjzb1MLDwBgYNrb2/uc39zcbNUCAABQE+epzlHpr7eSH1SK1C5kkKWg47LLLouvfOUr3V1WpdYbS5Ysic985jN9/m1HR4eNHQAG6ETVuq4jAQAAqJXzVOeoDCREa2pqGt6BzF9//fXYvHlzzJ079/8+ZNSo8vnGjRuPWv7QoUNl0NFzAgAAAAAA6K9BDz1+9rOfxZtvvhkTJ07s9Xp6nsb3OFJbW1vZsqNrMp4HAAAAAABQE6FHfy1fvrxsktI1pYFIAAAAAAAA+mvQBzI/++yz45RTTom9e/f2ej09b2lpOWr5xsbGcgIAAAAAAKiplh6jR4+OSy+9NNauXdv9WhrIPD2fPXv2YH8cAAAAAADA0LT0SJYtWxaLFi2Kd7/73TFjxox44IEHorOzM2666aah+DgA4H9VKhXrAgAAgJrhPHVkKYpixH+fQxJ6fOhDH4qf/vSncffdd5eDl//Kr/xKrFmz5qjBzQEAAAAAAAZLpThRdDPMOjo6orm5udrFAAAAAACAulLUeEuP9vb2aGpqGt4xPQAAAAAAAKpB6AEAAAAAAGRB6AEAAAAAAGRB6AEAAAAAAGShodoFAADqZ8AxAAAA6ktf56nOUWtPJYPrBlp6AAAAAAAAWRB6AAAAAAAAWRB6AAAAAAAAWRB6AAAAAAAAWRB6AAAAAAAAWRB6AAAAAAAAWRB6AAAAAAAAWWiodgFGmqIojjuvUqlEPeprndTzeoGRsg/6XQMARgrHLQAw8rg2OPyKOr+GraUHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQhYZqF2CkqVQqkaOiKAb87z7ROjmZ94ZcDOV+cLL7kH0wL75PAHJWzXqur+M59S9vdVuxvUDehnL/r+Zvi9+1kacyQre1waKlBwAAAAAAkAWhBwAAAAAAkAWhBwAAAAAAkAWhBwAAAAAAkAWhBwAAAAAAkAWhBwAAAAAAkAWhBwAAAAAAkIVBDz0+97nPRaVS6TVddNFFg/0xWSqKos9pKB35nR05nUy5TvTeUA/sBwC1ccwDMFDOaRiMbcU5MORtKPf/av62+F2rL5UM6rGGoXjTd73rXfHMM8/834c0DMnHAAAAAAAAdBuSNCKFHC0tLUPx1gAAAAAAAMM3pseLL74Yra2tcd5558VHP/rR2LFjx3GXPXToUHR0dPSaAAAAAAAAqh56zJw5M1auXBlr1qyJhx9+OLZv3x7vfe97Y//+/cdcvq2tLZqbm7unKVOmDHaRAAAAAACAOlAphni0yH379sU555wTX/rSl+Lmm28+ZkuPNHVJLT3qNfg40VdRzYFi+irbSBnABgCoDbV8zAMAAEDtam9vj6ampj6XGfIRxs8888x45zvfGdu2bTvm/MbGxnICAAAAAACouTE9ejpw4EC89NJLMWnSpKiHuxb7mkaydMfl8SYAgME6rnBsAQAAQE2FHnfccUesX78+fvKTn8T3v//9uPbaa+OUU06JG264YbA/CgAAAAAAYOi6t3r55ZfLgOPVV1+N8ePHx3ve857YtGlT+f8AAAAAAAAjdiDz/koDmTc3N0c9DsppUE8AAAAAABj4QOZDPqYHAAAAAADAcBB6AAAAAAAAWRB6AAAAAAAAWRB6AAAAAAAAWWiodgFycqKByof67wEAAID6VRRFn/NddwCgHmjpAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZKGh2gUAAACgPhRF0ef8SqUybGXJZb1ZZ9ge7GNQzxxbcCxaegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFloqHYBAACAka0oiuPOq1Qqw1oWapvtwXqDWuS3CUYu+y/HoqUHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAABQn6HHhg0b4pprronW1taoVCrx+OOP95pfFEXcfffdMWnSpBgzZkzMnTs3XnzxxcEsMwAAUEPSecHxJgAAgJoOPTo7O2P69Onx0EMPHXP+fffdF1/+8pfjkUceieeeey7e9ra3xfz58+PgwYODUV4AAAAAAIBjqhSpacYApTu3Vq9eHQsWLCifp7dKLUA+9alPxR133FG+1t7eHhMnToyVK1fGhz/84RO+Z0dHRzQ3Nw+0SAAAAAAAQIZS3tDU1DR8Y3ps37499uzZU3Zp1SUFGDNnzoyNGzcO5kcBAAAAAAD00hCDKAUeSWrZ0VN63jXvSIcOHSqnni09AAAAAAAA+mtQW3oMRFtbW9kapGuaMmVKtYsEAAAAAADUe+jR0tJSPu7du7fX6+l517wjLV++vOyHq2vauXPnYBYJAAAAAACoE4Maepx77rlluLF27dpe3VU999xzMXv27GP+TWNjYznwSM8JAAAAAABgyMf0OHDgQGzbtq3X4OVbtmyJcePGxdSpU+P222+Pz3/+8/GOd7yjDEHuuuuuaG1tjQULFvS7cAC5KIqiz/mVSiXqUV/rpV7XSc583zByqccAvx3UMvUU9bAtVnM7t4+Rfejxgx/8IK644oru58uWLSsfFy1aFCtXrow777wzOjs74xOf+ETs27cv3vOe98SaNWvitNNOG9ySAwAAAAAA9FApThTVDbPUHVYa0BwgJ+6K6P960dIjP75vGLnUY4DfDmqZeopaoaUHDL00LviJhsgY1DE9AAAAAAAAqkXoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZKGh2gUAqAeVSqXaRahJ1kt9Ganfd1EUWf67oD9s58BA+O1guNjWqIdtsZrbuX2MkUZLDwAAAAAAIAtCDwAAAAAAIAtCDwAAAAAAIAtCDwAAAAAAIAtCDwAAAAAAIAtCDwAAAAAAIAtCDwAAAAAAIAsN1S4AADB4iqLoc36lUrG6AQCAmj1vcc4CnCwtPQAAAAAAgCwIPQAAAAAAgCwIPQAAAAAAgCwIPQAAAAAAgCwIPQAAAAAAgCwIPQAAAAAAgCw0VLsAAMDgqVQqVucgs04BAMAxNjByaOkBAAAAAABkQegBAAAAAABkQegBAAAAAABkQegBAAAAAABkQegBAAAAAABkQegBAAAAAABkQegBAAAAAADUZ+ixYcOGuOaaa6K1tTUqlUo8/vjjvebfeOON5es9pw984AODWWYAAAAAAICTDz06Oztj+vTp8dBDDx13mRRy7N69u3t69NFH+/sxAAAAAAAA/dLQv8UjrrrqqnLqS2NjY7S0tPT3rQEAAAAAAGprTI9169bFhAkT4sILL4xbb701Xn311eMue+jQoejo6Og1AQAAAAAAVD30SF1bfe1rX4u1a9fGn/3Zn8X69evLliFvvvnmMZdva2uL5ubm7mnKlCmDXSQAAAAAAKAOVIqiKAb8x5VKrF69OhYsWHDcZf7zP/8zzj///HjmmWfiyiuvPGZLjzR1SS09BB8AAAAAAEBP7e3t0dTUFMPevVVP5513Xpx99tmxbdu2447/kQrZcwIAAAAAAOivIQ89Xn755XJMj0mTJg31RwEAAAAAAHWsob9/cODAgV6tNrZv3x5btmyJcePGldO9994bCxcujJaWlnjppZfizjvvjAsuuCDmz58/2GWvO331RJa6GgMAAAAAgHrW7zE91q1bF1dcccVRry9atCgefvjhcnyPf/3Xf419+/ZFa2trzJs3L/70T/80Jk6c+JbeP43pkQY052hCDwAAAAAA6lX7WxjT46QGMh8KQo/jE3oAAAAAAFCv2mthIHMAAAAAAIDhIPQAAAAAAACyIPQAAAAAAACyIPQAAAAAAACyIPQAAAAAAACy0FDtAvDWVSoVqwsAAAAAAI5DSw8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALDZGZoij6nF+pVIatLACMTOoSAEZyXeWcB2qbY00AGFpaegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFloiMxUKpVqFwGAEU5dAkCtU1fByGX/BYChpaUHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAACQBaEHAAAAAABQf6FHW1tbXHbZZTF27NiYMGFCLFiwILZu3dprmYMHD8bixYvjrLPOijPOOCMWLlwYe/fuHexyQ80riqLPCQAAAADXY2A4FXVwzbJfocf69evLQGPTpk3x9NNPxxtvvBHz5s2Lzs7O7mWWLl0aTz75ZDz22GPl8rt27YrrrrtuKMoOAAAAAADQrVKcRHzz05/+tGzxkcKNOXPmRHt7e4wfPz5WrVoV119/fbnMCy+8ENOmTYuNGzfGrFmzTvieHR0d0dzcPNAiQc040a5VqVSGrSwAAAAA9cD1GMh7H0kZRFNT09CN6ZE+IBk3blz5uHnz5rL1x9y5c7uXueiii2Lq1Kll6HEshw4dKoOOnhMAAAAAAEB/DTj0OHz4cNx+++1x+eWXx8UXX1y+tmfPnhg9enSceeaZvZadOHFiOe9444Sklh1d05QpUwZaJAAAAAAAoI4NOPRIY3s8//zz8Y1vfOOkCrB8+fKyxUjXtHPnzpN6PwAAAAAAoD41DOSPbrvttnjqqadiw4YNMXny5O7XW1pa4vXXX499+/b1au2xd+/ect6xNDY2lhMAAAAAAMCwtfRIg5ykwGP16tXx7LPPxrnnnttr/qWXXhqnnnpqrF27tvu1rVu3xo4dO2L27NknVVAAAAAAAIBBa+mRurRatWpVPPHEEzF27NjucTrSWBxjxowpH2+++eZYtmxZObh5GkV9yZIlZeAxa9as/nwUjHiVSqXaRQAAGPHSjVd9ccwFADg2gLeuUgfXLCvFic4i3sIKWbFiRdx4443l/x88eDA+9alPxaOPPhqHDh2K+fPnx1e/+tXjdm91pI6OjjI8AQAAEHoAAABd0rjgqbHFoIUew0HoAQAAdBF6AAAA/Qk9+jWmBwAAAAAAQK0SegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFloqHYBchpIsVKpRK5OZrz7nNcLANB/BqamPxxL5sX+b53CSDPUv1v1ep0JBotjC45FSw8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALDdUuwEhTqVSiHtXrvxs4OUVR9DnfbwvUJ/s+1C/7v3XK0HL8PfL2Mb+LYB9i8GnpAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZEHoAQAAAAAAZKGh2gUAIF+VSqXaRQAAgLrh+BsAtPQAAAAAAAAyoXsrAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAAAgC0IPAAAAAACg/kKPtra2uOyyy2Ls2LExYcKEWLBgQWzdurXXMu973/uiUqn0mj75yU8OdrkBAOCYiqI47gQAAEDe+hV6rF+/PhYvXhybNm2Kp59+Ot54442YN29edHZ29lrulltuid27d3dP991332CXGwAAAAAAoJeG6Ic1a9b0er5y5cqyxcfmzZtjzpw53a+ffvrp0dLS0p+3BgAAAAAAqN6YHu3t7eXjuHHjer3+9a9/Pc4+++y4+OKLY/ny5fHaa68d9z0OHToUHR0dvSYAAAAAAIAhbenR0+HDh+P222+Pyy+/vAw3unzkIx+Jc845J1pbW+NHP/pRfPrTny7H/fjWt7513HFC7r333oEWAwAAAAAAoFQpBjii46233hrf/va343vf+15Mnjz5uMs9++yzceWVV8a2bdvi/PPPP2ZLjzR1SS09pkyZMpAiAQBAnwOWVyoVawgAAGCESr1PNTU1DX5Lj9tuuy2eeuqp2LBhQ5+BRzJz5szy8XihR2NjYzkBAAAAAACcjIb+3jW3ZMmSWL16daxbty7OPffcE/7Nli1bysdJkyYNvJQAQNWdqHGoO+ipFbZFAABy43zMemGIQo/FixfHqlWr4oknnoixY8fGnj17ytebm5tjzJgx8dJLL5Xzr7766jjrrLPKMT2WLl0ac+bMiUsuuaQ/HwUAAAAAADB0Y3oc7665FStWxI033hg7d+6Mj33sY/H8889HZ2dnOTbHtddeG5/97GdP2M9WzzE9UogCANQWdxYBAABUh/Mx64W3PqbHgAcyHypCDwCoTQ6yAQAAqsP5mPXCWw89RvU5FwAAAAAAYIQQegAAAAAAAFkQegAAAAAAAFkQegAAAAAAAFloqHYBAICRoVKpVLsIDCIDITJS2FaxrfS9H9Ry/Wz/tU6BweP3fuStF6pHSw8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALQg8AAAAAACALDdUuAAAAw69SqVjtjAi2VWwrI3c/GKnlrmXWKVCLqvnbVBRFn/P9btYnLT0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAIAsCD0AAAAAAID6Cz0efvjhuOSSS6KpqamcZs+eHd/+9re75x88eDAWL14cZ511VpxxxhmxcOHC2Lt374AK1t7eHkVRHHMCAAAAAAA4qdBj8uTJ8cUvfjE2b94cP/jBD+L9739/fPCDH4wf//jH5fylS5fGk08+GY899lisX78+du3aFdddd11/PgIAAAAAAGBAKsVJNp0YN25c3H///XH99dfH+PHjY9WqVeX/Jy+88EJMmzYtNm7cGLNmzXpL79fR0RHNzc1lS4/UmuSYha5UTqbIAAAAAACMcCe6tO06cn76yg1OekyPN998M77xjW9EZ2dn2c1Vav3xxhtvxNy5c7uXueiii2Lq1Kll6AEAAAAAADCUGvr7B//2b/9Whhxp/I40bsfq1avjl37pl2LLli0xevToOPPMM3stP3HixNizZ89x3+/QoUPl1LOlBwAAAAAAQH/1u6XHhRdeWAYczz33XNx6662xaNGi+Pd///cYqLa2trI7q65pypQpA34vAAAAAACgfvU79EitOS644IK49NJLy8Bi+vTp8Vd/9VfR0tISr7/+euzbt6/X8nv37i3nHc/y5cvLfri6pp07dw7sXwIAAAAAANS1AY/p0eXw4cNl91QpBDn11FNj7dq13fO2bt0aO3bsKLvDOp7GxsZy4JGeEwAAAAAAwJCO6ZFaZVx11VXl4OT79++PVatWxbp16+I73/lO2TXVzTffHMuWLYtx48aV4cWSJUvKwGPWrFn9Llh6PwAAAAAAOJZKpWLFcHKhxyuvvBIf//jHY/fu3WUocckll5SBx2/8xm+U8//yL/8yRo0aFQsXLixbf8yfPz+++tWv9ucjAAAAAAAABqRSFEURNaSjo0MrDwAAAAAAoJc0LviJhsg46TE9AAAAAAAAaoHQAwAAAAAAyILQAwAAAAAAyILQAwAAAAAAyELNhR41Nq46AAAAAAAwQvKDmgs99u/fX+0iAAAAAAAAIzA/qBQ11rTi8OHDsWvXrhg7dmxUKpXo6OiIKVOmxM6dO6OpqanaxYOqsB+A/QDUBaAuAOcG4BwZ1AXUq6IoysCjtbU1Ro3quy1HQ9SYVODJkycf9XoKPIQe1Dv7AdgPQF0A6gJwbgDOkUFdQD1qbm5+S8vVXPdWAAAAAAAAAyH0AAAAAAAAslDzoUdjY2Pcc8895SPUK/sB2A9AXQDqAnBuAM6RQV0AMfIGMgcAAAAAAMiypQcAAAAAAMBbIfQAAAAAAACyIPQAAAAAAACyIPQAAAAAAACyUPOhx0MPPRS/+Iu/GKeddlrMnDkz/vmf/7naRYIh0dbWFpdddlmMHTs2JkyYEAsWLIitW7f2WuZ973tfVCqVXtMnP/lJ3wjZ+NznPnfUNn7RRRd1zz948GAsXrw4zjrrrDjjjDNi4cKFsXfv3qqWGQZbOu45cj9IU9r2E3UBOdqwYUNcc8010draWm7vjz/+eK/5RVHE3XffHZMmTYoxY8bE3Llz48UXX+y1zM9//vP46Ec/Gk1NTXHmmWfGzTffHAcOHBjmfwkM/j7wxhtvxKc//en45V/+5Xjb295WLvPxj388du3adcL644tf/KKvhGzqghtvvPGobfwDH/hAr2XUBeS+HxzrPCFN999/f/cy6gOo8dDjm9/8Zixbtizuueee+OEPfxjTp0+P+fPnxyuvvFLtosGgW79+fXlBa9OmTfH000+XJzfz5s2Lzs7OXsvdcsstsXv37u7pvvvu822QlXe96129tvHvfe973fOWLl0aTz75ZDz22GPlPpNO9q+77rqqlhcG27/8y7/02gdSnZD89m//dvcy6gJyk4530rF+uuHpWNLxzpe//OV45JFH4rnnnisv/KbzghSGd0mBx49//ONyn3nqqafKiwaf+MQnhvFfAUOzD7z22mvl+fBdd91VPn7rW98qb476rd/6raOW/ZM/+ZNedciSJUt8LWRTFyQp5Oi5jT/66KO95qsLyH0/6Ln9p+lv//Zvy9Aj3RDYk/qAetcQNexLX/pSeVJ/0003lc/TSc4//uM/ljv0Zz7zmWoXDwbVmjVrej1fuXJl2eJj8+bNMWfOnO7XTz/99GhpabH2yVZDQ8Mxt/H29vb4m7/5m1i1alW8//3vL19bsWJFTJs2rQwLZ82aVYXSwuAbP358r+fpLt3zzz8/fv3Xf737NXUBubnqqqvK6VhSK48HHnggPvvZz8YHP/jB8rWvfe1rMXHixPLuxw9/+MPxH//xH+WxVAoN3/3ud5fLPPjgg3H11VfHn//5n5d3S8JI3Qeam5u7A/AuX/nKV2LGjBmxY8eOmDp1avfrqdW4cwVy3A+6NDY2HncbVxdQD/vBkdv/E088EVdccUWcd955vV5XH1Dvaralx+uvv15e7E1N17uMGjWqfL5x48aqlg2GQ7rAm4wbN67X61//+tfj7LPPjosvvjiWL19e3vkFOUndlaSLU+mgLd2plU7mk1QnpBZQPeuF1PVVOtFXL5CrdDz093//9/G7v/u75R1cXdQF1JPt27fHnj17ev3+p4vAqevbrt//9Ji6tOoKPJK0fDp/SC1DIMdzhVQvpO3+yKA8dQP6q7/6q2VXJ//zP/9TtTLCUFi3bl15c+CFF14Yt956a7z66qvd89QF1JvU1XO6OTx16Xkk9QH1rmZbevzsZz+LN998s7yDq6f0/IUXXqhauWA4HD58OG6//fa4/PLLy3Cjy0c+8pE455xzygvCP/rRj8q+fVPT9tTEHXKQLmClVk7pJCY11b333nvjve99bzz//PPlBa/Ro0cfdXKf6oU0D3KU7mLft29f2Yd1F3UB9abrN/5Y5wVd89Jjugh2ZMvBdPOIOoLcpG7d0nnADTfcUI5h0+UP//AP49d+7dfK7f773/9+eYNUOp5KPShADlLXVqlr23PPPTdeeuml+OM//uPyjvgUdpxyyinqAurO3/3d35UtOo7s8ll9ADUcekA9S2N7pIu8PccySHr2S50GMkyDeV555ZXlAV/q+gRGup7NeC+55JIyBElB3z/8wz+UA9dCvUlduqX9omfXPOoCgPqVWr3+zu/8Ttnt28MPP9xrXhoPs+dxVLpZ5Pd///ejra2t7BIIRrrUnWHP8+G0nafz4NT6I50XQ71J3f+n3hFOO+20Xq+rD6CGu7dK3fekpD411eopPddHKTm77bbbysE3v/vd78bkyZP7XDZdEE62bds2TKWD4ZVadbzzne8st/H025+6+kl3vfekXiBX//Vf/xXPPPNM/N7v/V6fy6kLyF3XsX9f5wXp8ZVXXuk1P3Xr8/Of/9y5A9kFHql+SGN89Gzlcbz6Ie0HP/nJT4atjDCcUne46dpR1/mwuoB68k//9E9lzx8nOldI1AfUo5oNPdJdKZdeemmsXbu2V5c/6fns2bOrWjYYCulurRR4rF69Op599tmyye6JbNmypXxMLT4gRwcOHChbMqVtPNUJp556aq96IR3kpTE/1AvkaMWKFWV3Pb/5m7/Z53LqAnKXjonShayev/8dHR3lWB1dv//pMYXiafynLul4Kp0/dAWDkEPgkcY+S4F4GrfjRFL9kMa1ObLrN8jFyy+/XI7p0XU+rC6g3lqEp3Pk6dOnn3BZ9QH1qKa7t0rNsRYtWlQOSDhjxox44IEHorOzM2666aZqFw2GpEurVatWxRNPPFH2ydjV/3QaqDN165Mu/Kb5V199dXmSk8b0WLp0acyZM6ds1gs5uOOOO+Kaa64pu7TatWtX3HPPPWWrv9RnddoX0gBtqW5IfVWnuxuXLFlSntzMmjWr2kWHQZUu1KbQIx0HpXEJuqgLyDnk7tlyNQ1enk7Q0+/91KlTy7HOPv/5z8c73vGOMgS56667ym7fFixYUC4/bdq0sq/3W265JR555JHyAnG6mSR1hdKzezgYiftAuqB7/fXXxw9/+MOyRXga+7LrXCHNTzcMpjENUhB4xRVXlOcS6Xk6V/jYxz4Wb3/726v4L4PB2Q/SlMb7W7hwYRmEp2OiO++8My644IKYP39+uby6gHo4Juq6+eOxxx6Lv/iLvzjq79UH8L+KGvfggw8WU6dOLUaPHl3MmDGj2LRpU7WLBEMi7Y7HmlasWFHO37FjRzFnzpxi3LhxRWNjY3HBBRcUf/RHf1S0t7f7RsjGhz70oWLSpEnlb/4v/MIvlM+3bdvWPf+///u/iz/4gz8o3v72txenn356ce211xa7d++uaplhKHznO98p64CtW7f2el1dQK6++93vHvM4aNGiReX8w4cPF3fddVcxceLE8jjoyiuvPGr/ePXVV4sbbrihOOOMM4qmpqbipptuKvbv31+lfxEM3j6wffv2454rpL9LNm/eXMycObNobm4uTjvttGLatGnFF77wheLgwYO+CrLYD1577bVi3rx5xfjx44tTTz21OOecc4pbbrml2LNnT6/3UBeQ+zFR8td//dfFmDFjin379h319+oD+P8q6T9dAQgAAAAAAMBIVbNjegAAAAAAAPSH0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAMiC0AMAAAAAAIgc/D/qX8KLSZ1CGQAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.figure(figsize=(20,10));\n", "plt.imshow(nn.h.abs() > 0.99, cmap='gray', interpolation='nearest');" ] }, { "cell_type": "code", "execution_count": 86, "id": "4fcb50fd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 200000: 3.7371\n", " 10000/ 200000: 1.7613\n", " 20000/ 200000: 1.4222\n", " 30000/ 200000: 1.8768\n", " 40000/ 200000: 1.4871\n", " 50000/ 200000: 2.2807\n", " 60000/ 200000: 1.7422\n", " 70000/ 200000: 1.5854\n", " 80000/ 200000: 1.9995\n", " 90000/ 200000: 1.4310\n", " 100000/ 200000: 1.7311\n", " 110000/ 200000: 1.4871\n", " 120000/ 200000: 1.3908\n", " 130000/ 200000: 1.3281\n", " 140000/ 200000: 1.5639\n", " 150000/ 200000: 1.7007\n", " 160000/ 200000: 1.6561\n", " 170000/ 200000: 1.1525\n", " 180000/ 200000: 1.7005\n", " 190000/ 200000: 1.4037\n", "train_loss=1.4660104513168335\n", "val_loss=1.657399296760559\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGeCAYAAABhOIBvAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUZpJREFUeJzt3QlcVGX3B/ADKOAGLii4oLjlhoGiIJqYaS75VpZvqS2alW1qi2VJi7ZraWalr5al9r5mWf3VFg1T1NxQDEXNfQcXUFxAQUFh/p/z4Iwzwyx3Zu6duXfm9/185gMz3Jm5d2bgHp7nnPP46XQ6HQEAAABolL+ndwAAAADAFQhmAAAAQNMQzAAAAICmIZgBAAAATUMwAwAAAJqGYAYAAAA0DcEMAAAAaBqCGQAAANA0BDMAAACgaZXIy5SVldGpU6eoRo0a5Ofn5+ndAQAAAAl4QYJLly5RgwYNyN/fwbEWnRvMmDFD16RJE11QUJAuPj5et2XLFpvbX7hwQffcc8/pIiIidIGBgbqWLVvqli1bJum5srOzeXkGXPAa4DOAzwA+A/gM4DNA2nsN+DzuKMVHZhYtWkRjx46l2bNnU0JCAk2fPp369u1L+/fvp3r16lXYvqSkhO68807xs59//pkaNmxIx48fp5o1a0p6Ph6RYdnZ2RQSEiL78QAAAID8CgoKKDIy0nAed4QfRzSkIA5gOnfuTDNmzDBMA/HOjhkzhsaPH19hew56pkyZQvv27aPKlSs79WKEhoZSfn4+ghkAAACNcOX8rWgCMI+yZGRkUO/evW8+ob+/uJ6WlmbxPr/++islJibSqFGjKDw8nKKjo+nDDz+k0tJSi9sXFxeLF8D4AgAAAL5D0WAmLy9PBCEclBjj6zk5ORbvc+TIETG9xPdbvnw5vfXWW/TJJ5/Q+++/b3H7SZMmiUhOf+FRHwAAAPAdqivN5mkozpf56quvKC4ujgYPHkxvvPGGmH6yJDk5WQxJ6S+cKwMAAAC+Q9EE4LCwMAoICKDc3FyT2/l6RESExfvUr19f5Mrw/fTatGkjRnJ42iowMNBk+6CgIHEBAAAA36ToyAwHHjy6kpqaajLywtc5L8aSbt260aFDh8R2egcOHBBBjnkgAwAAAKD4NBOXZc+ZM4e+/fZb2rt3Lz377LNUWFhII0aMED8fNmyYmCrS45+fP3+eXnjhBRHELFu2TCQAc0IwAAAAgDnF+8xwzsvZs2dpwoQJYqooNjaWUlJSDEnBWVlZJp3+OIF3xYoV9NJLL9Gtt94q+sxwYPPaa68pvasAAACgQYr3mXE39JkBAADQHtX2mQEAAABQGoIZAAAA0DQEMwAAAKBpCGYAAABA0xSvZvIWZy8V08w1hyi4cgCN79/a07sDAAAAN2BkRqKCq9do/qZjtHDLcal3AQAAADdAMOMgr6pjBwAA8AIIZiTyU/Z9AAAAACchmAEAAABNQzDjKMwzAQAAqAqCGYn8/DDRBAAAoEYIZgAAAEDTEMxIpB+XwSwTAACAuiCYAQAAAE1DMOMgnQ5jMwAAAGqCYEYi5P8CAACoE4IZAAAA0DQEMxL53UgBxiQTAACAuiCYAQAAAE1DMOMg5P8CAACoC4IZiZAADAAAoE4IZgAAAEDTEMw4SIcUYAAAAFVBMAMAAACahmDGQUgABgAAUBcEMxIhARgAAECdEMwAAACApiGYkcjvxtAMOgADAACoC4IZAAAA0DQEMxKVj8tgaAYAAEBtEMwAAACApiGYAQAAAE1DMONgaXZJaZmCbwcAAAA4CsGME05evOLM3QAAAEABCGYk8ruZAkwXCkuUeC8AAADACQhmAAAAQNPcEszMnDmToqKiKDg4mBISEig9Pd3qtvPnzxcN6owvfD+1KSq5Tmv2naHi66We3hUAAACfpngws2jRIho7dixNnDiRtm3bRjExMdS3b186c+aM1fuEhITQ6dOnDZfjx4+T2tZmeu67bTRi/lZ6//e9ntolAAAAcEcwM23aNBo5ciSNGDGC2rZtS7Nnz6aqVavS3Llzrd6HR2MiIiIMl/DwcNW9WWv3nxVfF6ZneXpXAAAAfJqiwUxJSQllZGRQ7969bz6hv7+4npaWZvV+ly9fpiZNmlBkZCTde++9tHv3bqvbFhcXU0FBgclFCWYDMwAAAOALwUxeXh6VlpZWGFnh6zk5ORbv06pVKzFq88svv9CCBQuorKyMunbtSidOnLC4/aRJkyg0NNRw4QDInUrLdDR91QE6fPayW58XAAAAVFrNlJiYSMOGDaPY2Fjq0aMHLV68mOrWrUtffvmlxe2Tk5MpPz/fcMnOznb7Pk9fdZB6ffKX258XAAAAiCop+SKEhYVRQEAA5ebmmtzO1zkXRorKlStThw4d6NChQxZ/HhQUJC6KwzwTAACA743MBAYGUlxcHKWmphpu42kjvs4jMFLwNNWuXbuofv36Cu4pAAAAaJWiIzOMy7KHDx9OnTp1ovj4eJo+fToVFhaK6ibGU0oNGzYUuS/s3XffpS5dulCLFi3o4sWLNGXKFFGa/eSTT5JaOgCfuFDk0X0BAAAANwYzgwcPprNnz9KECRNE0i/nwqSkpBiSgrOyskSFk96FCxdEKTdvW6tWLTGys2nTJlHWrRbrD+Z5ehcAAADgBj+dTqcjL8Kl2VzVxMnA3HxPLmcvFVPnD1aJ7x9KaEwLt1TsL3Ns8gDZng8AAMCXFLhw/lZdNZNaGXcALr5W5sldAQAAACMIZpzwf9ss97wBAAAA90MwIxEqswEAANQJwQwAAABoGoIZAAAA0DQEMxLxSt4AAACgPghmAAAAQNMQzEiEcRkAAAB1QjCjMn8dOEvJi3dRUcl1T+8KAACAJii+nIG3cFfKzPC56eJrnWqB9ErfVu55UgAAAA3DyIxKnbp4xdO7AAAAoAkIZgDAI3j1+Xkbj2JKFQBchmkmAPCI/tPX06Xi63T8XBG9fU87vAsA4DSMzMgo73KxnA8H4NU4kGEbDuWJrxnHL9BvO055eK8AQIsQzEjkJ6E4e+Ivu+V8QoOjeYU0a+1hKrzxxx/AGw2atYnGfL+d9pwq8PSuAIDGYJpJRsfPF5ISen2ylsp0RCcvFlFy/zZUNTDAIx2JdTodvb9sLzUNq0aPdGni9ucH35B1vojaNgjx9G4AgIZgZEYDOJBhCzZnUbuJK+jNpf94ZD94GuCbDUc99vygbrkFV0XACwDgbghmNOi7LVkeed5LVzHNpSb/nMynIV+lUWb2RU/vish1Sfgwlcb9vNPTuwIAPgjBDEh25VqpR1+t7VkX6HxhiUf3QU0e/DKNNh85T/f9Z6Ond4Wmrzogvv6cccLTuwIAPgjBjFQSUlT+OVlALV5fTu/8VjER+NtNx2jyH/sceDr1rQb13HfbPPbcGw/l0X3/2URdJ6fK+rgFV69pdmqkqKQ8uNTo7gMAyAbBjMyul+lo3sZjFRqBTfx1N83+67CilRrLdp4WVU/WRjUGf5kmpiaUUqpP7rHj6rVS2nzkHF0vLZP82Gv2nblxX+n3sWfT4Ty69e0/qWnycrFPAACgTQhmFNJ2wgp6fcmuCrcruYDkqIXb6KOUfRZzKHhUY8vR8zT0q82KPPelq9dEzsQLP2y3uy1vM+SrzTRlxX7ypOkrDxq+/+Of09a3W3WAvl5/hDwp+3wRdZu8WiRgAwCAKQQzClq4JavCaIVcMwLT/txPaYfPWfzZ+cJiOn6ukPp8+het2V8+omHeqExuv+04LZoG/pJpv+nZit254uu8TcdILaxN1Zy8eIWmrzooStLLJI48KeHd3/eIfXnv9z0eef7x/7eT3v715vQpf66vOTCyBgCgJAQzKmWvjcznqw/R0DnWR1nu+mw9Hci9TCPmbZV/53zIFaORNKmtfTgHZ/epfFlH4VbuKQ8APYGDqB+2ZtP8Tceo+Hr5dFzf6euoy4epVHK9PKBB2g4AeBKCGYnk6lEnNVlT6tPtyL5Io77bJqYhjBXeSA61h//D5qkpfU4KuI5HngZ8voEGzvR8lZEcLOU2HTpzmc4VltCRvMtu248zBVfFBQDAHIIZD5TT6v+blSOZ9t6ZG2nZrtP09P8ynNqfXzJPiqThEfO1M4LDr42aE3b/b1t5eTKPjGkdV3vty7nk6d0QI0LxH6aKi5TfHwDwLQhmFFZYcl2srWRs4+HyhfVsWbz9JP03TXpOybFzzi2lcDr/qiwlzTorEw2r9uRS0sdraFvWBZJL/8/WUZsJKXQZa1UprvtHa5wOlOVUcOW6SbI5AIAxBDMSOTvL1OPjNdRz6lqT2/7cnUsZx8+LJF1bJsi0cCUnAhvbdcL18mwuaX7i27/tbvfkf/8Wa+0Mn5tOcuERD46jth13LUCyFIBdLHK9KZ839X3Jv+L5wIFHDu80+wwDABhDMKOwC0UVTwbfp2fRoFlp1GPK2gq5Luak9oXRN1CzxHy64+4ZG0gOqx3Isym20R+GR3l+33mKrkjM87GHH49XYE6xUW5tydLtJyn23ZU0ZYX05oZyee67DLHPSldM8fTcZ6sOij4/asK5XzeZvgac03XRwu8RAIAeghkV5FcM+Hw9/brDcknzv75wPPB4fL79ERM1efq/GTR64XarC1hycPK1A/1V/vX5BrEo5jMLHOtY/NYv5c8/c43lxoNScnm4H83OE46vlbR8V47Y590KNlXkLsqt30qhT1cdEH1+5Bh1MlyXGIP9ses09f10HR3IvVQh9wsAwFkIZjyMe5jwCez57+03m/NUlZaUChJXplbSbowS6BNnzRn3N5GCp7XcjUeVOn+wSvSjOXOp2KH75hjlLRlPfSUv3kkv/rC9wnILPGVnacSOc0m4ysgSfgylcl/4sY+Y5YVZ8+x322h/7iWPfd4BwDtV8vQOgDp6iVjDSzDwmlKdo2rJ/th+NqZCuGMxP+e790aLEQulubqA5ay1h5x6DB6h6PPpOovB0ffp2eL7cf1am/zsrwNnDQGgsa6TVoumiL+PuY2iG4aa/GzkfzMUS5h2ZjTJ1rSoN+UcAYB7YGRGIj+5Gs3YMPK/7p8e+jjF9pIC+sUxtx5zLaAoKS0TfVfMy2p/szC9tmJ3Du09XUD/TTtO7rL3tP3yYz7JvrQo0+KCoYfPWh+Z2HQoj5ZstzzqtOKfHMvPZTRC86uFrsrmr+PLP+4wdHf+PPXmMg16q/ZaD5R5vbAeU9YYpjr/PnZeHKPU8ncpbQR4qtAa89GkTDvTdIh1AMAcRmbAbXjNqD/3mJ68x1iYbjCuoLF1oswtuEpjf8ykYYlRFBtZU5yED1qZZjFXWHzdZoD2yk876IVeLU3Gj/acLqAl20+K78f3Nx0tseWhr7eIr9ENQqlleA1yFCfA2mM8RfengyN8z/+wnY6fKxJTP/fENKB/z04Tt1epHEBy+HN3Dj31vwx6OqmZpO2LitXbQwgA1AkjMyALqf8tS/kvPu9yiaTtx/28kxZvO2notMsn4eTFpot7jl2UafG+9krFf844QU98u9XqCALnp3Bwxi39LY0uWXLsnP1cnjMFjuXbWOPIiuTWqsikdPeVMiX03rLy9aS+XHdzsU6s6wQAckIw46O44kZq2bcUy3c6VgYtNbA5eOaSGDWwZ7uVpnzcfNBS75i/zfJw9uUU0KWr122WtL9oFBhNW3lABFE5BVctji45O40odWTJnnYTV9D7Ci1KaZ6QrGSzRk8trAkA2uKWYGbmzJkUFRVFwcHBlJCQQOnp0hqo/fDDDyJXZeDAgeRpymfMuNc9MzZWKPs2X5/pdP4VyY9nKSHVnusSghle40gKThi2Rsq519Ek1nkb5VnxW6lUrOLrZQ6VszvicbOlL/RTb3I5c+lmoPONhWPwtt9FANBAMLNo0SIaO3YsTZw4kbZt20YxMTHUt29fOnPGdsO1Y8eO0SuvvELdu3dXehfhBuP1mbgyh1dJlvJfet5leaZGjB4U74mM9CtdOxOoWHpr1uw/a7h+8Yrtzwl/PrLPSw+K9Qt1rjtw8zkAADyeADxt2jQaOXIkjRgxQlyfPXs2LVu2jObOnUvjx4+3eJ/S0lJ6+OGH6Z133qH169fTxYuONyED57R5K4WSbgkTJxQpHpidVmHKRi/fQtdWe1NGfPK7pkAXXO6sm37sPPn7+VGnJo6UmUvbF17sU06Xzaa8XPHRH7Yr1hzp6hz3/kqT6w/NKU9uNn/fP0s9SE3DqlKb+iHkjGFz0+mde9pZ/BlCXQBwazBTUlJCGRkZlJycbLjN39+fevfuTWlp5RUTlrz77rtUr149euKJJ0QwY0txcbG46BUUKNNB1Q2V2apw5Vqp5ECGWQtkZq45RFNWVDyJcq6JLU2Tl5Pc+OQ366/Dhv2ZeHdb8pQLhSVUq1qg1ZL/Czfye0bL1FSOK8PmbrQ93XTy4hVZ+sPoTfz1H1p6o5y8Yc0qkh+74uPIszYZAHg/RaeZ8vLyxChLeHi4ye18PSfHcn+NDRs20DfffENz5syR9ByTJk2i0NBQwyUyMpKU4IeZesn4P3NLgYyndHxvJU1fdTOImmNUVaO0r9aZLo3wwfK9lHWuiP63+biY/uGKKNPty/ct/eh5WZ4/5p0/nb7vxkPOrd/0j1H+kSOBEgCAV/SZuXTpEj366KMikAkLC5N0Hx714Zwc45EZJQKaKoHy9NzwBdwgT22uld6cnDglsZKGWerO64gPl++rkFSdNGWN+P4tK2tRudPqfdZH4WTPhZLJdaP3ElyffuUWBE3qVKO3rUzrAZCvBzMckAQEBFBurukfTL4eERFRYfvDhw+LxN+7777bcFtZWfmJsVKlSrR//35q3ry5yX2CgoLEBdSDG855C4UXsfaos5eKFVmU1Nr6UHK56/P1tO2tOxV9Dl+RkXXhRkL3WQQzoGmKTjMFBgZSXFwcpaammgQnfD0xMbHC9q1bt6Zdu3ZRZmam4XLPPfdQz549xfdKTSGBvHjtIHCetZ45aluPylO0ut9qhFEu8BaKTzPxFNDw4cOpU6dOFB8fT9OnT6fCwkJDddOwYcOoYcOGIveF+9BER0eb3L9mzZriq/ntAN7KVs8cAADwQDAzePBgOnv2LE2YMEEk/cbGxlJKSoohKTgrK0tUOAGAey3Y7L6FPME9vl5/hHacyKfpg2MpwN9HSjAB3JUAPHr0aHGxZO3atTbvO3/+fIX2CsD9/nZx9XE5cUUVeJf3l+0VXwe0r0/9oivmJQJ4KwyJALgRLzMA2skr0qqiEvmaLgJoAYIZANCcl3/0bMUcd6q+ek36MhFq5Ugz0K3HztPfx+TpfwQgNwQzAAAOGrVwG7V+K4VS9+bSqj3SO2ZreaSHly759+w0rwjiwPsgmAEAcNDyXeUdzJ/49m968r9/0w6zTs7exnitsOJrmCoF9UEwAwCao7ZehvtzLnl6FwB8GoIZAAAfZS1lZlvWBeo6KZX+cGI1+MvF1+mDZXsqrDsGoCQEMwAAPsp41XbOA+JFUNmIeVvFGmbPfrfN4cf85M/9NGf9URo4c6PVbUrLdGLZE7QHULdNh/Ko2+TVmujqjmAGAECiXzJP0uiFFU/wOpVNfDlSpaS3bOdpenpBhvieV3SXquR6mViwUu9grv21uVbuyaWfM06oYrFVsO6hr7eIle+Hz00ntVPVqtkAAFJLoz3hhR8yyZsdzXNskVCubOr8wSpqWLMKpbyYJPl+BVevObF3ANZhZAYANLnY5M4TF+l0/hXy1cCLH/fZBRk04RfPjW7sPlVAl65ep31engDNTRrnbTzqsSAa7EMwAwCaU3D1Ot0zYyMlTlpNalZYfJ2Spqyh8f+3U/bH3p97if74J4f+myb/shTm52wlT+FaKGvnxV/f+W0PLXMiIdpb7Mi+qOpgDsEMAIBCft1xirLPX6EftmZb3ebMpatOnSSul8p/YpG0GzKvX/ndluMiIXjF7hzxWqjZoTOOTcNZwknPM9ccIq25d+ZGajdxBW05co7UCMEMAHgNpf5z3HzknKjqsP68zj0uJ8HGf5BK7/6+h7SSKGzNvpwCp19/XsH96f9lUJ9P15E3Onz2MqUdLg8COOl5yor9lH2+vHJMS4pKSmnwV5tJjRDMAICmcZDxz8l8+nN3DsW+u5LW7j8j+3MM+WqzqOqQG/djYfM2HiOt6zd9PX2z4ahTlV2r9pYvCXGxyL2JwRxQ/Lg1m66V3uxqfL20jC7ZSFDmz8H/ZZwwuY89vT75i4bO2Ux7TxcYbnvPQwGst0I1EwBoGp9cnlmQQSculAcbj83bSscmD6D8K9dEonDTsGpuWQ396/VH6PZW9ahFverkaX4uzgU5O741+6/D1CqiBrkDj3RwQJF0S12nH6PHlDXEVeV5hcX03O0txG3/+mKDSGje8novCg8JrnCfpI/XiGmxc4XF9FRSc4eer/9n6w3f/7knV1R1hQRXdnr/4SaMzACA5ukDGWNx762knlPXWsxz4C6101cdoAO58lThzFhziN5ftpd6T/uL3IVXsXaVn7UgxkY0cyyvkOTm6OwUj57wSMewuel0sajE6efVt8fRTwExfWVW6l7LI3wcyLBNRvdxVqmEvKe8y8WiYm3PqZujOq7SqTiR11kIZgDAK12/cdJJO5xX4Wcf/bGPpq86KFuOxtlLxaSUlH9y6N3f9hhOonpcXSM7nbREULW8t6zgys1FML0RV8Jxxdpdn6+nxdtOiID5yFnbicgp/5ym9KOWg91Jf+ylzh+kqj7Z2lEIZgDA56hl3SAp/x/zFNrcjUfp1x0nHXhcndsTgO3h0bAl209UaJjn6pSYLUfzCsX6Uloeidh7+ubo4dgfd4iRxvGLd1ndPutcET2zYBs9+GWaxZ9/+dcRMdrDX5mWXxtjCGYAwOtwZ1o58In3xR+2kxqcKVBu9EevxEpSqxznu7GLMumlRTtozELT13PR39m04VDF0TM58DQjry9lbcrIUc68Dkokjhebfb5fX7KLPkrZJ7631EiS+x1x6fuVktIKCdCcQM/5XlqHYAYAvM5X6+T54zztzwO0NPOUw4EUV7ss2ppFWu12K+daU7zO0+Av00TCK3PHooU8Jcel4o6OxPHokdxslfQbm7bygAicHR0p4RymhVuyaNbaw1a3eWlRpih9H7/YtHnjh8v3igVFOd9L61DNBABeJ8vJHh5frTtMOfnFNOHutuJ6Tr7jeQXjft5Jv+0oD4Be7deK1IJL17MvXKEnbmtqt9tt5YCbUz+nLl4hfyvzUVJOvL9mnqItVvI3LOHXrmvzOlSnehA5a+Kv/9CCzVkOfV74WMb9tMOti4f+uSeHMrPz6fZWdenz1IPitmFdo6hj41ouj6YZ0weSv5gF5sZ5WPxZf/jrzSKg++7JBGpRzz1VaXLByAwA+Izj5wrpP2sPia+WfLh8n8hP0VeOOHNC0wcy7KrZsL67WMpDeep/GaK3ifHyASv32J9+6Tp5dYWRjTeX7hInvcXbK+bxmMc3lk62trJkxny/ne6ftcnqz/OLrokqJlscCWTY8XNF9OW6I2J5CGdtPJRHt09ZQ5sO59GIeek06ruKq6ube+3/dtH36Vli1ESv+Fr568XBFb9f9qapnF0XS2f2Pr259B86fLaQcguKaeBM66+/WmFkBgC8DnfW1eO/2TNWH6Spfx6wuG3y4l00LLEJtakfYrjtyo2chLX7XZsSsRcK2RvYcCU5s+R6GQVW8rdaecWNBrkvjBRLzYIWDhaCKwWI6Q0lcHBhyZmCq2IpAONKHc4RaVynqs3H4340XM5dKcDf8LqaB1mT/yjPOXHWw19vEV8fmlP+lb1m5Ths4ZLz9a/2pGPnCkUTQkuM2w08/71pDpKfk1ndZy8XKzrdpjSMzACAV1t3IM9qIMP4P2PjZmbmzfCUZC9YMe9lsmb/GVEafdBOfxzun3PLm3/QRBsranOLfev7RXYdtzCVl3e5xKmpOSl4RCn+w1T61mxhTW6vb2/0gkdden6y1iRgaPVmis37fCBDHknvT53rOzT1z/2i4aM1hW4Y8bvioVFFZyGYAQCvtv7gWY9Vnbhq/cGbVT6n86/SiHlbxUn9TrP+OObVKP+5kQxqfuJnpTqdCGQ2H7Gex2Ieyzjyz/4Ro6Z6+uUaLOUVOYKnbvQ5Jc4m2fKCn3q2jt14HSJX8eiYs3aeyCc5FNhYmsGWWWu1tRgmppkAAIjojqk3/3N3R/8NLostuGp7ON94Gmj+JuvrN3E1ypPdm0l6XuP8DKXNWX+UPrgv2uVg9NFvbOfIKIWnW6oH3TxN/vi39dXP5cbl1I42a8w4fsGkM/SJC0ViYUtL7OWDHVag07OSMDIDAF5N6lSR8XZcPhz3/irF9om7r3b/eE2FxFe5+uPIzVJc545eaxxQGi81IMfjOdp92RiPjulxPpWcSwwYyy24Kun15WUOjA2atUmsyK1320draI2LeV9agWAGAMAMT2nYyllwpu2+8SrLuy2cBLkKasHmitNCniDlpK9f6VopnOjb6f1VFnNznMXTb/byjRzBSwwogafBllioFJOyJpmvQjADAKAQ40Zms200NdNzZfFCawsuOjNdZrYMlEt+/PtmZZkjeIThXGEJLdt5WrYFOPkxzfONbOGV1z1FSv8YJS1z8HX3NAQzAABukLrPfk+X1fvOiI65zjBfiFIt/70b97VREpdt8/TgA7Mtr0nkDO7zMtxOTxutmrfxmOiN4+73SSlIAAYAUJGrJWUUVCnA4fvpbAQ5vOCiK66XeXaUQAou21aCO5ZfcLZSytWFQgs1Vn5tC0ZmAADcpKjkut0qlcKS64osuOiKVTIt1OiNeBqPGxC6G+cUwU0YmQEAcNNJr+2EFeL7+qHBNpcPODrpLtGZ2BGZWZanCbYcla8aCCp6+ccd1CrC/esYfb76ILVvGOqRtyT7fBFF1rbdddndMDIDAOAGO4yaoBmX+Fqb2vhhq2M9TTYevpn/YL7+DyiH16ea5OJSCM5wR2m8NY4G2u6AYAYAQGUem7dVUyc38AwXU2bc3lVYSQhmAAC8gK0OwQDezi3BzMyZMykqKoqCg4MpISGB0tOtl7otXryYOnXqRDVr1qRq1apRbGws/e9//3PHbgIAAGiHq+VMXkTxYGbRokU0duxYmjhxIm3bto1iYmKob9++dOaM5ez42rVr0xtvvEFpaWm0c+dOGjFihLisWFGeOAcAAADlvXU8YeeJfJcW0VSCn07h1dR4JKZz5840Y8YMcb2srIwiIyNpzJgxNH78eEmP0bFjRxowYAC99957drctKCig0NBQys/Pp5CQEJJT1Phlsj4eAACAFr0/MJoe6dJE1sd05fyt6MhMSUkJZWRkUO/evW8+ob+/uM4jL/ZwnJWamkr79++npKQki9sUFxeLF8D4AgAAAMq5IMPaZXJSNJjJy8uj0tJSCg8PN7mdr+fkmK5GaoyjsurVq1NgYKAYkfniiy/ozjvvtLjtpEmTRCSnv/CoDwAAAPgOVVYz1ahRgzIzM2nr1q30wQcfiJybtWstd7BMTk4WwY/+kp3tWG8GAAAAcMxnqQfJZzoAh4WFUUBAAOXmmi4Vz9cjIiKs3o+nolq0aCG+52qmvXv3ihGY22+/vcK2QUFB4gIAAADucV3OpdXVPjLD00RxcXEi70WPE4D5emJiouTH4ftwbgwAAACA29dm4imi4cOHi94x8fHxNH36dCosLBTl1mzYsGHUsGFDMfLC+Ctv27x5cxHALF++XPSZmTVrltK7CgAAABqkeDAzePBgOnv2LE2YMEEk/fK0UUpKiiEpOCsrS0wr6XGg89xzz9GJEyeoSpUq1Lp1a1qwYIF4HAAAAAC395lxN/SZAQAAUN6xyQN8o88MAAAAgNIQzAAAAICmIZgBAAAATUMwAwAAAJqGYAYAAAA0DcEMAAAAaBqCGQAAANA0BDMOeKxrlHLvBAAAADgFwYwDqgQGOPcqAwAAgGIQzAAAAICmIZhxgHct/AAAAOAdEMw4QEeIZgAAANQGwQwAAABoGoIZB/iRn3LvBAAAADgFwQwAAABoGoIZAAAA0DQEMw7wwywTAACA6iCYcQBKswEAANQHwYwDUJoNAACgPghmAAAAQNMQzAAAAICmIZgBAAAATUMwAwAAAJqGYAYAAAA0DcGMI7DOJAAAgOogmHFA1cBKyr0TAAAA4BQEMw7ofksYPmYAAAAqg2DGkRcL6xkAAACoDoIZAAAA0DQEMwAAAKBpCGYAAABA0xDMOECHZbMBAABUB8EMAAAAaBqCGQAAANA0BDMAAACgaQhmAAAAQNPcEszMnDmToqKiKDg4mBISEig9Pd3qtnPmzKHu3btTrVq1xKV37942twcAAADfpngws2jRIho7dixNnDiRtm3bRjExMdS3b186c+aMxe3Xrl1LQ4cOpTVr1lBaWhpFRkZSnz596OTJk+RpYdWDPL0LAAAAYMZPp3C9MY/EdO7cmWbMmCGul5WViQBlzJgxNH78eLv3Ly0tFSM0fP9hw4bZ3b6goIBCQ0MpPz+fQkJCSG5Lt5+kFxdlyv64AAAAWnJs8gBZH8+V87eiIzMlJSWUkZEhpooMT+jvL67zqIsURUVFdO3aNapdu7bFnxcXF4sXwPiipIEdGir6+AAAAEDqCWby8vLEyEp4eLjJ7Xw9JydH0mO89tpr1KBBA5OAyNikSZNEJKe/8KgPAAAA+A5VVzNNnjyZfvjhB1qyZIlIHrYkOTlZDEnpL9nZ2W7fTwAAAPCcSko+eFhYGAUEBFBubq7J7Xw9IiLC5n2nTp0qgplVq1bRrbfeanW7oKAgcQEAAADfpOjITGBgIMXFxVFqaqrhNk4A5uuJiYlW7/fxxx/Te++9RykpKdSpUycldxEAAAA0TtGRGcZl2cOHDxdBSXx8PE2fPp0KCwtpxIgR4udcodSwYUOR+8I++ugjmjBhAi1cuFD0ptHn1lSvXl1cAAAAANwazAwePJjOnj0rAhQOTGJjY8WIiz4pOCsrS1Q46c2aNUtUQf373/82eRzuU/P2228rvbsAAACgMYr3mXE3pfvMsKjxyxR5XAAAAK045it9ZgAAAACUhmAGAAAANA3BDAAAAGgaghkAAADQNAQzAAAAoGkIZgAAAEDTEMwAAACApiGYAQAAAE1DMAMAAACahmAGAAAANA3BDAAAAGgaghkAAADQNAQzAAAAoGkIZgAAAEDTEMwAAACApiGYAQAAAE1DMAMAAACahmAGAAAANA3BDAAAAGgaghkAAADQNAQzLvrkgRh53gkAAABwCoIZF0TWrkKD4hq58hAAAADgIgQzLvAjP1dffwAAAHARghkAAADQNAQzAAAAoGkIZgAAAEDTEMy4IDaypnzvBAAAADilknN3822rxvagXzJP0pO3NfP0rgAAAPg8BDNOaFGvOr3cp5XPf3gAAADUANNMAAAAoGkIZgAAAEDTEMwAAACApiGYAQAAAE1DMAMAAACahmAGAAAANA3BDAAAAGiaW4KZmTNnUlRUFAUHB1NCQgKlp6db3Xb37t00aNAgsb2fnx9Nnz7dHbsIAAAAGqV4MLNo0SIaO3YsTZw4kbZt20YxMTHUt29fOnPmjMXti4qKqFmzZjR58mSKiIhQevcAAABA4xQPZqZNm0YjR46kESNGUNu2bWn27NlUtWpVmjt3rsXtO3fuTFOmTKEhQ4ZQUFCQ0rsHAAAAGqdoMFNSUkIZGRnUu3fvm0/o7y+up6WlyfIcxcXFVFBQYHLxlAahwR57bgAAAF+laDCTl5dHpaWlFB4ebnI7X8/JyZHlOSZNmkShoaGGS2RkJHnKE92x8CQAAIC7ab6aKTk5mfLz8w2X7Oxsj+1LYCV/+vbxeI89PwAAgC9SdNXssLAwCggIoNzcXJPb+bpcyb2cV6Om3Joet9T19C4AAAD4FEVHZgIDAykuLo5SU1MNt5WVlYnriYmJSj41AAAA+AhFR2YYl2UPHz6cOnXqRPHx8aJvTGFhoahuYsOGDaOGDRuK3Bd90vCePXsM3588eZIyMzOpevXq1KJFC6V3FwAAADRG8WBm8ODBdPbsWZowYYJI+o2NjaWUlBRDUnBWVpaocNI7deoUdejQwXB96tSp4tKjRw9au3at0rsLAAAAGqN4MMNGjx4tLpaYByjc+Ven07ljtwAAAMALaL6aCQAAAHwbghkFJKGiCQAAwG0QzCjgiyEdaNL97em+Dg2VeHgAAAAwgmBGAaFVK9PQ+MYUWqWyS49TI9gtKU0AAACahmBGxd4fGE3rxvX09G4AAACoGv71l1GbiBqyPdbTSc3o3lhMUwEAANiDYEYGK15MoqN5hdQpqjbJZWyfW2R7LAAAAG+GYEYGrSJqiIsUCU1r07FzhZRbUGxzu6BKAXLsGgAAgNdDMOMmG8ffQRcKS6ht/RAq1emo5Rt/OP1Ysx+Jo2cWZMi6fwAAAFqFBGA3aVizCkU3DCV/fz+qHOBfYZrKEc3rVpN57wAAALQLwYyHPNKlseF7qVNUAAAAUBGCGQ9pFY4ABgAAQA4IZjykZtVATz01AACAV0ECsIfc1b4+bTiYR3FRtTy1CwAAAF4BwYyHBPj70Uf/vtVTTw8AAOA1MM2kEp8OjlH08WMia1Jck1om1VUAAADeAMGMStzXoRG1qFdd0rbmpd1SVamMRnwAAOB9EMxodIoquLLrb114SJAs+wMAAOBJCGY0asfEPrT2ldtdeoxpD8bKtj8AAACegmBGA7q3DLO4dlNUmPOdgP38iLq1qPi44D6vYDFRAABZIJjRgLYNQhy+j78f0aT729+8Qacz+bnZVXCzmEahdEfrcLzuAAAyQDCjIn5Wbk9sVsfhx/L386Oh8TeXTAB1mXB3O9KyBU8kmFw3rpQDAHA3BDMa0OOWujT7kY4uPUa9kGBSk46Na5Kv+m30beLk30zDC4ZGhCJ5HADUA8GMBvj5+VH3lnUduk8Dsz4ytapWJrXgk/i8EfHkqypXKh+DC64cQMuf7+7p3aGHEjCCBwDahmDGS43s3pTUKjayZoXS8j9e8PxJ3ROqBKL3DwCAqxDMeCmudnJFg1D3Tku1qR9CdaoFyr7SeG07j+kJSnZf/ngQlsgAAN+DYEZFbBUY2Ss++vZx02mbAbfWr7BNrzb1JO/Luld70udDO5CaqrZWvJRErSMcC2gSmtYmNRnUsRHVCFZuys+fy9gc5Pg9AADUBcGMBnGPmAq3mV2vFlRxDdFhiVGSn6NSgD8FBvipbtRILSXlW17v5dT96rt5xEtr7olp4OldAAANQjDj5fTl2U/3aC6WQVADa9NJ7RuGklZICaqqWsiH0dkdYwMAAEchmPFy3DjvwPv9qXldaYtYetLzvVqSGoVVd64MuXOUuqa4HBnpAwDQEgQzPiCwkr/LuSS924TTR4OMOgorgEuVleBono25Da/1JLUa1bO5y4/BDRZd1VLiiu8AAEpAMKMi7vwHuW90hEPbfz28Ew3u3FhS4PDGXW1IDpvG3yHL48x9rLNLVVpSg6wgC0Gj0sb1be3yYzg6vdfMwppgyTK9508lNbP585oq6pcEAOqBYMZHKRU4pbyYRCPtnJCcbfzn6cexl/vy1zjHR3DCQ+TtpKtzIkOaK6wc8edLSRVyhkKruB5kPN2jGUXbCazUkgAOAOqCYMbHvX13W/E13mjaqXHtqqRWcibQ8hIRPJKU3N+10Y1bG4XS3nf7UYTZqI+U2ZuqgZVoc3IvsWSFpzhazs2VbkpI7i/P6A4A+B4EM17C2bSHx7o1pe1v3UnP3X4z9yIm0r3rJkXVqWo3z8XVvBdLeBSAR5KSXAwkqlQOcKmTLwdBdaqrr7mfEmY/Ekdv/as8gHYGkpUBwBIEMz7KuHFbrWqBHq12mi9hnabKMo0G8DpXckxZVK1csY8P2NcvOoIe7ya93xEAgGqCmZkzZ1JUVBQFBwdTQkICpaen29z+p59+otatW4vt27dvT8uXLydfV7VygOG/0no1gl0q1X7itqbUpZlpNVOkhaml0CrKjRYEBviL0ZbI2lWoUS3l2vvL5d9xjahJnar0yQMx9NmQWAp1MhHVT2X9dqtbaK6oNOOA0lOjgQDgXRT/S7Zo0SIaO3YszZ49WwQy06dPp759+9L+/fupXr2K7fU3bdpEQ4cOpUmTJtG//vUvWrhwIQ0cOJC2bdtG0dHR5Ks4r2HPO/2oTKezWGrtaBM9W/SnGg54nk5qRi0UKLvlExqvGM2DI5aa+QUpVKbtSGO/c4UlhutTH4gRybXGJ2Je/fvI2UK6281da7u3DKP1B/MouqHt5R/sCQmuRBtkqhhzlZpWdQcA7VE8mJk2bRqNHDmSRowYIa5zULNs2TKaO3cujR8/vsL2n332GfXr14/GjRsnrr/33nu0cuVKmjFjhrivL3P3Cst84par5NbRxFNrUxFKVLNYekxu4Hfm0lXq2aqe1RGFJc91o92n8qlL0zqyJjC/2q8VfZyy3+rP//t4PGVmX6RWFvKILI16WMNBaoiC60QphcvrT+Vf9fRuAICvTDOVlJRQRkYG9e7d++YT+vuL62lpaRbvw7cbb894JMfa9sXFxVRQUGBy0SqlmsapTSUJaz5VC1QmzjY+11tb3qFvu3AxgsU9XDrZ6OLL5chdm4dZDcqUmlDigKVD41qiEsqccSVar9bSFxYFANAyRYOZvLw8Ki0tpfDwcJPb+XpOTo7F+/DtjmzP01GhoaGGS2RkJGnVtAdjqGlYNZo+ONaj+2GrZ4itEmLef1vG9W1FbeqH0IhuTclTuLHdo12aiN4qlvrPcN+XLx/t5NJUnjPimtSSpSMv5/XoyZ2L9GLvlrRqbA/yJB5Fc2ZlcADwbpovyUhOThY5OXo8MqPVgKZleA1a88rtHnv+TwfH0M8ZJ2jsnbdY3ebLR+PE1MqgWRVHyu6303xtVM8W4uJp7w20nntl7xiUMqRzY1Gx1alJLapTPYgWbD5OJy5cMWko9+VfR2R9TkempBgHoXI0x5OjY7HxawMAoOi/n2FhYRQQEEC5ubkmt/P1iAjL7fT5dke2DwoKopCQEJMLOOe+Do3ouye7iFJtW1NhcU2UW0AxIqS8UqtTVPlIRbcW5fkowxKbkDu81Nt6IOcMqQ0IecrrwU6R1KxudREwrH/VtJtwuAsVbHKsu8Qjapww7A6P3xi5eyCuET3WNUqRtaTA9xaMBe+maDATGBhIcXFxlJqaaritrKxMXE9MTLR4H77deHvGCcDWtgd1lvw6a92rPWnX232oZtXygGreY/G04sUkURrNlO5mL/f00pxhnRQfNbFWCWT+Wq18KcmpfeHXf9mY2xweyXGE8SO/MaANLXmuK314v7ILm4L8eGSZm0gCuJviiQE8BTRnzhz69ttvae/evfTss89SYWGhobpp2LBhYqpI74UXXqCUlBT65JNPaN++ffT222/T33//TaNHj1Z6V0EFOJgwbujH17lqR8kTqZy4b47ey31aiX1/2ca0nRz+fKmH5GlMZ3COijvzVHiUihOcrTVKlHNJC3BtGQ9jNYIriZw/ZzzSxX7LCACPBjODBw+mqVOn0oQJEyg2NpYyMzNFsKJP8s3KyqLTp08btu/atavoLfPVV19RTEwM/fzzz7R06VKf7jEjVVU3l26rwb2x7u3xYs+y57vTz88k0v73+xkWTRyj8LB7ZQnVYXrWlhKYeHdb0VuHL+C8Ae3rG77v2lx6yb4WmRcD3G7UxsBR7w9sT4c+6C/DXoGvcsskOI+qWBtZWbt2bYXbHnjgAXEBx2hj7EI+r/VrTc/e3px+yTwl+T7VLHS8tVai7Qzu22KrnFsJjvTe4e7P7/2+x2JyL+enDPlqM507el7eHfQRA26tTzOGdqBOm2qJJOVpKw+Qt+vYuCZty7ooy2PxAqZcbVh8vUyWxwPfgrWZQBYdGru/HT0HMlL95+HyFbKnPVix7P2W8Opiscn7OzYkrXMlLNPCVB53YVYzfg05MHR3QOstuH0DuF/vNvVETp35MjdaovnSbLipcZ1qtPe0e5sG3t+hIQ3uHGmYUlFaJSdHUe5qX19crJ2AuKuukri3zdLMkzTcQnWOUuQ47w9XqIqsc1Qt2nrsgqYCKUf5QhPMV/q2oofmbBHfx9+oQHRUrNG6XLc4mdcFrrerqB9aRdOVghiZ8SKzH+lI/dpF0NJR3dz2nJwYmtCsjsXpGyV8OjhWtLOf8u9bSWt/LDIn9HFpkVB3MI9/3rSSY9OkTjW7AeYbNpbCWPRUoqT8HC3nnbx9dzvydpG1brYeiI10Lpj53xPK/iMB0mk4lkEwo6QQNzcY4xPM7EfjTP7T8Tbc72RTci96oJP2GiPKmZvjLtYqirjKjJOcuYQ6/Q3T5Uf0RiY1sxkEVw2yP3JxZ9tw+lbhUTO5mDebNK5sc8UdKl2Wgj/NxqvHtwx3bkFa4+rFdg3QJ0xOTW1UlxkXT9Ssov3Ef4zMKOippGYi4//jQdoaRQB1GNK5PGDjpRfim6pvLjuoUoAooa6tYAUUTz/ZWkLD3OdDO5AnglM+MTSva/9k7q7mj+7CCe9zH+skAk45ptW4+7U3v17u1s5GcFjJ31/0BUp9uYfbFzFWAoIZBVUPqiR+yR+8cVICcMS790bT9yO70If3R4ucpN9G30ZbrYyCONOPxR3jREoE8k8lWU/8jlbgP/vuLcPomR7NacZDHay+flJH3bq3lB6YacUdrcNtBpycd2Wpk7M1xq8l/w4Y6x9tuRM8VPRKn1tMRr2sjdxICcK1AMGMl/DUXCc3ygJl8FROYvM6YgSEtW8USnVrmP7nqnYcyHMJvSUDY52rHnPnNOqW13vRtyPiaXz/1tSugXuS3LVWwSVldE2uv0+PdMFIjdQpyRrBlRVtRujuxXjtUdfegGZMfSBGJGe+gHVYPM54iJgTsWNudGYd0L6BKgJgawnCSqzhY68ag6uobOXncPm+sfCQYNm7H4/s7rlV4+XEfWHA/cyXi9D/vrOZD3UkYxyAZ064k+TAvx/G4ho7l/CtFHwawSm8VtLCkV0Mayh50mdDYkUV1z0x6uoG7C6cq8Bdh398OlEEM4uf60Y7JvahRrXkSUB1VUiVSg4lF9v6Ax5r5w9okzpV6bYWYU6tYdazVT1KeTHJ4TWwHGVtpEprHuvmvjYDWmYcbMhh73v9JP1jo7sxosd/ozeNv6PCVOCTFoJqa78f80d0pi8fiaOHEtS77ASCGdC8e2MbiiouX1zOQY+btOmThDnngFfeVtNq7Nzm/4P7nF+S5Pfnb6Onk5rRtAdj7E5pLHgyQTQBs7yB07vgcODO+/LLqG70w1NdDNOx3FzSW0Y0OPnX1xbC9TQO1u15KKGxaF9x/43FeVmDmlWoZ6ubwcyed/uKylCpeKkKHqFU87voHb9VAFABt4Z3p6HxkVbn1mc+3JEeTnA+34GTFJPvakNhZtUuVoMWI9VkDHLjHezsGxNZk7o0qyMSt7e9dWeF/XcFT9NxcrItvKyCL6jnYC5Z33amUyZKqu5kXiGvl2ZOSuXgh/e1pw2v3WEz2Kwa6H25jghmALxUvRD3NuibdP+tFOHm57RWrWGcM2vcp8bPwa7Nck4Fyl3Czn1tOFCy5aU7peclhVX3/JSxo+8RV5g1r1tNTHk7gqcU3SXJiQq2MXe0EMtiOPva+FvI89J2Grl9CGZAEa/fxdUfIYZeKeAZcU3UlaTnCW0dLNfe914/+u7JBIsrjDv6WFpt8snTFHKxlZMd4mI15L9ubUCpL99OLeo5Vl7sTPI7T3OuGpskRnU4h8SaCWafG0eXCODP3st9PLNGlZ+a55HsQDADiuBeIMue766q3A1QH/1ohXmlhCck3Fhkj0dRurUIs1h6ajy6YvKH38l/eyff355ub1VXdFL2NOPD6doizGIJ/JO3NRW5F7aWIIiJDJV8kvzYwWVJ5DrXOpP98fhtTalFvRr05aOdRA6JOf4M80KZQ+NdS5J1+G+mhyKQe406CKuB902cgao8mtiEvlx3hAaq7IMPVrh5LJpX6t15Mp96ODgUz0nOpWU6EXQs2X7S5f3gLqhSmodxBc/yXadFQvPnqw86/Xy8inv+lWs0JL6xuCj1lsiVsvl0j2YiB2NUzxZ2t+3aPEx0BX58/t92t42y0W7f0ddiVM/mNHPNYVKKvVdyzrBOJom6x88VSWpeqVUPqmxJGQQzoKhGtaqKNXwC3VzBwWW0F4quqeI/frDdvt6Z/IWNr91BO05cpDvbhNMrP+1w+SWW2gWVT+j68m1XghlLK7jz1M6p/KtWe9PMWX+UlKYPEjmxmF+TzOyLYr+S+1tfNNRaV2B3G9e3taLBjCM03udQ0iCQ3P2XXIVgxkuo62NlSt/B1p1/QFaN7UF7T1/S9KrLSrP4X7uaP0hGIkKDKSI0wmdOLNybpk+7CHpjyS46kHtZ0n04MdaZIHH3qXzRSZaDGm53b6vRoFT9oiPomw3KB2Pgu5AzA177H/9tLcNU998DuJ/OCxIcuTdN56jaDrWQnz8inlqZdTS2NN1l3BeHg8RebcLF9/ycHISYL/7oDN73FS8miZ47SvLVxpmAYAYAfIi6237JK7J2VdEszdHpLrnoE1l5KpBxYMWL79rj6YDTfLkAA41/dHReMEJpC0ZmALyY1NWclWzfrzROvAwPCRIrjNsTFWa/gyrIY924nvTr6G6iMkqJUReeAjOnxvO1VgPo+zuULwRrPFXvSNdgd0PODIAX+/C+aHrk63RR6eEoLq3XAk7ylproPbJ7M/o+PZu8hadHMWwJrVqZbq0q3wrn9Wua9r7544XuigT5anpN61QLpHOFJYbrD7hQQaRzcPsP7mtPd7SpR0lGazoN6dyYrpSU2m3W6AkYmQHwYtwXIy35Dno00fFFAe1NUWgR95B52GixvOiGIfS1UUmt2v/j5twTuQ1PLO90/Gpf5Ru1mQcKlqY+rL2utczWxuL3UoqGFj7H3EzyzQGWK7Q8HcvwSu3WWGxCKuP8kZ/RG8SLVnJTQuNlETgIfLJ7M4pW4TIZGJkB8HLGf6DA1O9jumvqfeKmbPVDg+nOttIquaR4+5529NKdtzi8kKa7fDo4hlbtOUOPdY2ijOMXHL4/T42cvHhFfL85uRflXS4W/W34pLzhUB6t3X/W5WZ2cvTRalmvuhgNqWtjnSn8LluHkRkA8Go6jWc+1jWqJuIFArm7tj5fxNHRoZ+eSbR4glRrIKNfdZ0XKpU6EmMLV2s5OqrQPzqCtrzey2qLCV6P7NPBsSa3cRWYvnme1KZ5vFRG/I2V78FxCGYAwGcoNUhlnFcgNy6RlgP/x6/ENJXsCekeHEhc/XIPiyty25r6qVM9sMKICS8C+tmQWPq/Z5VZpqJxbSSym0Mw4y0wkwAOsjicre1BDI+Ru78JL2roihoSSqA94ZZ6Neg2maqblNBMYidoe3gU6d7YhhQmQ48eS7h6zwDTyAKCGfAa3ryisZzmPdZZdHideE/FVaG9kTviM7mbMybf5djyAeaWjKo4ImA+28aVMu7Gr9OCJxOcvr+987YzVXt6j3dreuM5vPM/Q53Gp1vtUWf4DuAErlIpuV5GXVuor2xQTXq2ricuvohzTOTIvWDcjbektIzUhkcDuIrNnspuXi9NDvFRtcVq3tbW0modESKWX9h6TFqisHHYMuFuy8G9veDGXozQKgL/ZLkDghnwGpxbMNLF4XnwfqN7tqAtR8/RoI6NXHqcbx/vTM9/n2n1JAj2OTpYwL/jS0d1U8VacFJF1XE+v4UX/FyaeYrcxY+0C8EMAPjUCbNWtUBZSrLjmtSmjePvIK3y0tkUydVDWvDuwGi3BjNapr1xRgAAJ3nrCRysC67s2GnudRfzlex59952YoqSV0Kf+dDNhT4t4YZ1lpLL9dPENTWy5Ig7IJgBANnou+uquWIF1E3OeJMT3aUudaH3UEJj0TVbqZFB3p8D7/enZ29vTgNudW6hT16Wg0u/eSVyKIdpJgCQzXM9W1BCszrU3sPtzn1lBMZXjtNZcx/rTMt2nrb6c2vJvfVD1b2UBydvc+k33IRgBgBkbYqGLqbyqiJT9RUow3sydLQN00wA4HW8oaXGewOjRRnymDtayPJ4vKaTMQzqaAMvp2Dp/VOCn4Y/FAhmvIQnVvAF0ALjP9Bq/mP9r5jy/InWEeU9Yh7t0kSUIXP1lSt+fDqReraqSzMe6kBqw2slye2WcPs9drSE13niZRFSrOTH8HILoOA00/nz52nMmDH022+/kb+/Pw0aNIg+++wzql7dervor776ihYuXEjbtm2jS5cu0YULF6hmzZp4nwDcxJvKWo0rQh6Ia0TXy3RUr4by/906i9fziWlUk7o2d63pY0So6cmNp/3im8aTmqwam0RXSsqotgJdiF/u00r0mrmrvf2VxSsHqDi6NcrriWtSy+rP3x8YTaVlOhqW2IR8mWLBzMMPP0ynT5+mlStX0rVr12jEiBH01FNPiWDFmqKiIurXr5+4JCcnK7VrAOBjpjwQQ2pXfgJ2rrqFLXqqC8366zC9c087UjspHYqdVT2oEo3v31rStm8OaEt7ThcYljJwhzrV5B1J4UUw5z7WmXydIsHM3r17KSUlhbZu3UqdOnUSt33xxRd011130dSpU6lBA8uLsr344ovi69q1a5XYLQCwA9OV2sVVZHzROndOBUbWrkrrX3Vf48PMCXeKHjOWtK2PZQ9UlzOTlpYmpof0gQzr3bu3mG7asmWLrM9VXFxMBQUFJhcAALDNWxdUVLOqgRXHD5Y/312MJI1w4+iQN1IkmMnJyaF69UwXsqtUqRLVrl1b/ExOkyZNotDQUMMlMjJS1scHAFArbgrH0AlWOfZiPnurUVcPvhnAWFpcvW2DEHqmR3OrIzZKBlLmtBzeOvTqjR8/XkTzti779u0jd+Lcmvz8fMMlOzvbrc8PAOApXLmzblxP2viadteI8na8ivnUB2JENRkvlOkpg+Iaiqq2Cf/yzoVRHcqZefnll+mxxx6zuU2zZs0oIiKCzpw5Y3L79evXRYUT/0xOQUFB4gIA4Isau7AqM7jHv+NcW6FdrgTzeSPUVdXmsWCmbt264mJPYmIiXbx4kTIyMiguLk7ctnr1aiorK6OEhATn9xasqhaELqEAoH1chrzx0DmqYTQ9A2CPImNebdq0EeXVI0eOpPT0dNq4cSONHj2ahgwZYqhkOnnyJLVu3Vr8XI/zaTIzM+nQoUPi+q5du8R1HtEBy3jIkMsKPb0WDgCAHKYP7kBPJzWjX0ffhhcUJFMs9P3uu+9EANOrVy9D07zPP//c8HPuPbN//37RW0Zv9uzZ9M477xiuJyWVdzycN2+e3ektX/X4bciABwDHqbWYqW6NIEq+q42ndwM0RrFghiuXbDXIi4qKqpAF/vbbb4sLAACAJ3F34vOFJdSnrbx5nqAMrM0EAF69nAGAM9aOu52WPX8bJVpZXuLVfq3E1w/ui/aaF9hPrcN1EiDDCgAAwMKaXu0aWM9FfO72FjSyezOq7MFya7gJ7wIAAIATEMioB4IZAAAA0DQEMwAAPkjD6REAFSCYAQAAAKpbXbvd9JEADAAAAPRa/9Z0rrCYHuikvQWbEcwAAAAAcW+dr4d31uQrgWkmAAAA0DQEMwDgdVqG1/D0LqjWmwPKlwr4eFCMp3cFQDaYZgIAr/N8rxbia79otKI392T3ZjS8axR6pIBXQTADAF6namAlGt+/tad3Q7XQ7A28DaaZAMDg+V4txddBHRvhVQGv0KROVU/vAriBn8586WqNKygooNDQUMrPz6eQkBBP7w6A5lwsKqHQKpU1vegcgLFfd5yiRrWqUMfGtfDCeOn5G9NMAGCiZtVAvCLgVe6JaeDpXQCFYZoJAAAANA3BDAAAAGgaghkAAADQNAQzAAAAoGkIZgAAAEDTEMwAAACApiGYAQAAAE1DMAMAAACahmAGAAAANA3BDAAAAGgaghkAAADQNAQzAAAAoGkIZgAAAEDTvG7VbJ1OZ1hKHAAAALRBf97Wn8d9Opi5dOmS+BoZGenpXQEAAAAnzuOhoaEO3cdP50wIpGJlZWV06tQpqlGjBvn5+ckeNXKQlJ2dTSEhIeRtvP34fOEYcXzah/dQ27z9/VPyGDkc4UCmQYMG5O/v79sjM/wCNGrUSNHn4DfPWz+kvnB8vnCMOD7tw3uobd7+/il1jI6OyOghARgAAAA0DcEMAAAAaBqCGQcEBQXRxIkTxVdv5O3H5wvHiOPTPryH2ubt759aj9HrEoABAADAt2BkBgAAADQNwQwAAABoGoIZAAAA0DQEMwAAAKBpCGYAAABA0xDMSDRz5kyKioqi4OBgSkhIoPT0dPK0SZMmUefOncXSDfXq1aOBAwfS/v37Tba5/fbbxbIOxpdnnnnGZJusrCwaMGAAVa1aVTzOuHHj6Pr16ybbrF27ljp27ChK8Vq0aEHz5893y2v09ttvV9j/1q1bG35+9epVGjVqFNWpU4eqV69OgwYNotzcXM0cHz+e+fHxhY9Jq+/funXr6O677xYtyXl/ly5davJzLqCcMGEC1a9fn6pUqUK9e/emgwcPmmxz/vx5evjhh0V30Zo1a9ITTzxBly9fNtlm586d1L17d7G/3Fr9448/rrAvP/30k/i88Dbt27en5cuXO7wvjhzftWvX6LXXXhPPVa1aNbHNsGHDxBIr9t73yZMnq/742GOPPVZh3/v166eZ90/KMVr6neTLlClTNPEeTpJwblDT304p+2IXl2aDbT/88IMuMDBQN3fuXN3u3bt1I0eO1NWsWVOXm5vr0Zeub9++unnz5un++ecfXWZmpu6uu+7SNW7cWHf58mXDNj169BD7e/r0acMlPz/f8PPr16/roqOjdb1799Zt375dt3z5cl1YWJguOTnZsM2RI0d0VatW1Y0dO1a3Z88e3RdffKELCAjQpaSkKP4aTZw4UdeuXTuT/T979qzh588884wuMjJSl5qaqvv77791Xbp00XXt2lUzx3fmzBmTY1u5ciW3StCtWbNGs+8f78Mbb7yhW7x4sTiWJUuWmPx88uTJutDQUN3SpUt1O3bs0N1zzz26pk2b6q5cuWLYpl+/frqYmBjd5s2bdevXr9e1aNFCN3ToUMPP+TUIDw/XPfzww+Lz//333+uqVKmi+/LLLw3bbNy4URznxx9/LI77zTff1FWuXFm3a9cuh/bFkeO7ePGieC8WLVqk27dvny4tLU0XHx+vi4uLM3mMJk2a6N59912T99X491atx8eGDx8u3h/jfT9//rzJNmp+/6Qco/Gx8YV/L/z8/HSHDx/WxHvYV8K5QU1/O+3tixQIZiTgP0ajRo0yXC8tLdU1aNBAN2nSJJ2a8ImRfzH/+usvw218MnzhhRes3oc/oP7+/rqcnBzDbbNmzdKFhIToiouLxfVXX31VBBTGBg8eLH5hlH6NOJjhP4qW8ImDf/F/+uknw2179+4VrwGfRLRwfOb4vWrevLmurKzMK94/8xMFH1dERIRuypQpJu9jUFCQ+GPP+I8i32/r1q2Gbf744w9xMjl58qS4/p///EdXq1YtwzGy1157TdeqVSvD9QcffFA3YMAAk/1JSEjQPf3005L3xdHjsyQ9PV1sd/z4cZMT4aeffmr1Pmo+Pg5m7r33Xqv30dL7Z+0YzfHx3nHHHSa3aeU9tHRuUNPfTin7IgWmmewoKSmhjIwMMbRnvJglX09LSyM1yc/PF19r165tcvt3331HYWFhFB0dTcnJyVRUVGT4GR8DD22Gh4cbbuvbt69YFXX37t2GbYyPX7+N/viVfo14SJWHg5s1ayaGrnnok/Fz8rC+8fPycG3jxo0Nz6uF49Pj51mwYAE9/vjjJiu+a/39M3b06FHKyckxeS5eWI6Hno3fM56a6NSpk2Eb3p73acuWLYZtkpKSKDAw0OSYeCj9woULko5byr7I9XvJ7ycfkzGekuBh9Q4dOojpC+Phe7UfH08t8LRDq1at6Nlnn6Vz586Z7Ls3vX883bFs2TIxVWZOK+9hvtm5QU1/O6Xsi0+umi23vLw8Ki0tNXlDGV/ft28fqUVZWRm9+OKL1K1bN3HS03vooYeoSZMmIhjg+Vuez+dfpsWLF4uf8y+KpWPT/8zWNvyhvnLlivjFVOo14l9anoPlP5qnT5+md955R8xB//PPP2K/+A+F+UmCn9fevqvl+IzxvP3FixdFToK3vH/m9Ptk6bmM95dPlMYqVaok/hAbb9O0adMKj6H/Wa1atawet/Fj2NsXV3EuAL9nQ4cONVld+Pnnnxd5BnxMmzZtEkEqf76nTZum+uPj/Jj7779f7N/hw4fp9ddfp/79+4sTT0BAgFe9f+zbb78VuSd8zMa08h6WWTg3qOlvp5R9kQLBjJfg5Ck+wW/YsMHk9qeeesrwPUfZnETWq1cv8UeoefPmpHb8R1Lv1ltvFcENn9x//PFHkQznTb755htxvBy4eMv758v4v80HH3xQJHDOmjXL5Gdjx441+VzzH/Onn35aJG6qab0bS4YMGWLymeT9588ij9bwZ9PbzJ07V4wIc/KqFt/DUVbODd4G00x28PA+/7dhnlnN1yMiIkgNRo8eTb///jutWbOGGjVqZHNbDgbYoUOHxFc+BkvHpv+ZrW34P00OKNz5GnH0fsstt4j958fmYUwezbD2vFo5vuPHj9OqVavoySef9Or3T/94tp6Lv545c8bk5zx8zxUycryvxj+3ty+uBjL8vq5cudJkVMba+8rHeOzYMU0cnzGe/uXPkPFnUuvvn9769evFSKi930u1voejrZwb1PS3U8q+SIFgxg6OtuPi4ig1NdVk2I6vJyYmkifxf3z8YV2yZAmtXr26wpCmJZmZmeIr/4fP+Bh27dpl8sdH/8e3bdu2hm2Mj1+/jf743fkacXknj0rw/vNzVq5c2eR5+Q8P59Ton1crxzdv3jwxNM9lkN78/vFnlP9AGT8XD0lzLoXxe8Z/2HguXY8/37xP+mCOt+HyWg4ajI+JpyN5+F7KcUvZF1cCGc714gCVcyrs4feVcwn00zNqPj5zJ06cEDkzxp9JLb9/5qOl/LsRExOjqfdQZ+fcoKa/nVL2RRLJqcI+jEvLOHt8/vz5IlP/qaeeEqVlxlnenvDss8+Kkr21a9ealAcWFRWJnx86dEiUDnKp29GjR3W//PKLrlmzZrqkpKQK5Xd9+vQRJXxcUle3bl2L5Xfjxo0TWeYzZ860WH6nxGv08ssvi+Pj/ecyRi4T5PJAzs7Xl/RxyeHq1avFcSYmJoqLVo5Pn93Px8CVDsa0+v5dunRJlHLyhf/ETJs2TXyvr+bhUlN+bD6enTt3ikoRS6XZHTp00G3ZskW3YcMGXcuWLU1Ke7kCgsteH330UVF+yvvPx2he9lqpUiXd1KlTxXFzZZylsld7++LI8ZWUlIjS2UaNGon3w/j3Ul8BsmnTJlEFwz/nUt8FCxaI92zYsGGqPz7+2SuvvCKqTPgzuWrVKl3Hjh3F+3P16lVNvH/2jtG4tJr3iSt4zKn9PXzWzrlBbX877e2LFAhmJOL6eX6xuV6eS824f4Kn8S+hpQv3F2BZWVnixFe7dm3xYeJeD/yhM+5Two4dO6br37+/6IHAgQIHENeuXTPZhvuexMbGiuPnE6r+OZR+jbjMr379+uIxGzZsKK7zSV6Pf6Gfe+45UQLJv1T33Xef+KXVyvGxFStWiPdt//79Jrdr9f3j57L0ueSSXn256VtvvSX+0PNx9erVq8Kxnzt3Tpz8qlevLkpBR4wYIU5Axrjnxm233SYegz8b/Eff3I8//qi75ZZbxDFxCemyZctMfi5lXxw5Pj7BW/u91PcOysjIEOW3fLIJDg7WtWnTRvfhhx+aBANqPT4+GfLJjU9qfNLl8mTuG2Ie9Kr5/bN3jHocdPDvFAcl5tT+HpKdc4Pa/nZK2Rd7/G4cOAAAAIAmIWcGAAAANA3BDAAAAGgaghkAAADQNAQzAAAAoGkIZgAAAEDTEMwAAACApiGYAQAAAE1DMAMAAACahmAGAAAANA3BDAAAAGgaghkAAAAgLft/iiTFBcJunysAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, max_steps, mini_batch_size)\n", "train_loss = nn.training_loss(datasets)\n", "val_loss = nn.test_loss(datasets)\n", "print(f\"{train_loss=}\")\n", "print(f\"{val_loss=}\")\n", "plt.plot(lossi);" ] }, { "cell_type": "markdown", "id": "34a0058d", "metadata": {}, "source": [ "Sur un réseau aussi simple que celui que nous avons utilisé ici, les erreurs d'initialisation ont finalement peu d'impact sur la qualité finale du réseau. Par contre, sur des réseaux plus complexes, ces erreurs peuvent empêcher le réseau d'apprendre.\n", "\n", "Nous avons pu ici modifier l'initialisation de manière empirique en ajoutant des \"nombres magiques\" (0.01, 0.2, etc.), mais il existe des techniques plus robustes pour trouver ces améliorations." ] }, { "cell_type": "markdown", "id": "6fd249b2", "metadata": {}, "source": [ "### Initialisation \"Kaiming\"" ] }, { "cell_type": "markdown", "id": "b9ead853", "metadata": {}, "source": [ "Plutôt que de choisir \"au doigt mouillé\" les valeurs initiales à utiliser, il existe des méthodes plus robustes.\n", "\n", "- Voir \"Kaiming init\" \n", "- Voir \n", "- Voir " ] }, { "cell_type": "code", "execution_count": 89, "id": "e09f56b3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor(-0.0177) tensor(1.0071)\n", "tensor(0.0058) tensor(3.2131)\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABkoAAAGsCAYAAACSD/sZAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAARNBJREFUeJzt3QuQFtWZMP5nBuSmMl6I3IIOKgkShDHgIK6KiayobCKJyQdUvoDzWaSM0WhIVHC5xNUsiEomKpGYfGg0IbBWItmIS+KyYmKJEkHL9caqJYuC3EwBOq5g4P3X6e8/IxMGZGCGYaZ/v6ouprtP93vmvM07fd6nz3mKCoVCIQAAAAAAAHKouKkrAAAAAAAA0FQESgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC3BEoAAAAAAIDcEigBAAAAAAByq3W0ADt37oy1a9fGkUceGUVFRU1dHQAAaHSFQiHefffd6NatWxQXe/6Jj6ffBABAnhTq0WdqEYGSFCTp0aNHU1cDAAAOujfffDM++clPank+ln4TAAB59OY+9JlaRKAkjSSp/oU7duzY1NUBAIBGt3Xr1uxhoep7Yfg4+k0AAOTJ1nr0mVpEoKR6uq0UJBEoAQAgT0w9S32vFf0mAADypGgf0nWYzBgAAAAAAMgtgRIAAAAAACC3BEoAAAAAAIDcEigBAAAAAAByS6AEAAAAAADILYESAAAAAAAgtwRKAAAAAACA3BIoAQAAAAAAckugBAAAAAAAyC2BEgAAAAAAILcESgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMit/QqUzJo1K0pLS6Ndu3YxaNCgWLZs2T4dN2/evCgqKooRI0bU2l4oFGLKlCnRtWvXaN++fQwdOjReffXV/akaAAAAAABA4wVK5s+fH+PHj4+pU6fGihUron///jFs2LDYsGHDXo9btWpVfO9734uzzz57t30zZsyIO+64I2bPnh1PP/10HH744dk5P/jgg/pWDwAAAAAAoPECJTNnzoxx48ZFRUVF9OnTJwtudOjQIebMmbPHY3bs2BFf+9rX4sYbb4wTTzxxt9EklZWVMWnSpLj44oujX79+cf/998fatWtjwYIF9a0eAAAAAADAPmu970Ujtm/fHsuXL4+JEyfWbCsuLs6mylq6dOkej/unf/qnOO644+Kyyy6LP/3pT7X2vfHGG7Fu3brsHNVKSkqyKb3SOUeNGrXb+bZt25Yt1bZu3VqfXwOABlY6YWG9j1k1fbj3AQAAaHb9GX0ZgJyPKNm0aVM2OqRz5861tqf1FOyoyxNPPBH/9//+3/jpT39a5/7q4+pzzmnTpmXBlOqlR48e9fk1AAAAAAAA9j+Z+75699134+tf/3oWJOnUqVODnTeNaNmyZUvN8uabbzbYuQEAAAAAgPyo19RbKdjRqlWrWL9+fa3tab1Lly67lX/99dezJO5f+MIXarbt3Lnz/71w69axcuXKmuPSObp27VrrnGVlZXXWo23bttkCAAAAAABw0AIlbdq0iQEDBsTixYtjxIgRNYGPtH7llVfuVr53797xn//5n7W2paTtaaTJj370o2zKrMMOOywLlqRzVAdGUs6Rp59+Or75zW8e0C8HwMHJNwIAAJAXcjQC5DxQkowfPz7Gjh0bAwcOjPLy8qisrIyqqqqoqKjI9o8ZMya6d++e5RFp165d9O3bt9bxRx11VPbvrtuvueaauPnmm6NXr17Rs2fPmDx5cnTr1q0mGAMAAAAAAHBIBEpGjhwZGzdujClTpmTJ1tMokEWLFtUkY1+9enUUF9cv9cl1112XBVu+8Y1vxObNm+Oss87KzpkCLQAAAAAAAI2lqFAoFKKZS1N1lZSUZIndO3bs2NTVAThkHMrTaK2aPrypqwDQrLkHxjUD0Hz6Tfo/AId2n6l+Qz8AAAAAAABaEIESAAAAAAAgtwRKAAAAAACA3BIoAQAAAAAAckugBAAAAAAAyC2BEgAAAAAAILdaN3UFAMin0gkL61V+1fThjVYXAAAAAPLLiBIAAAAAACC3jCgBAAAAIJcj1wEgMaIEAAAAAADILYESAAAAAAAgtwRKAAAAAACA3JKjBIAWO9fwqunDG6UuAAAAALQcRpQAAAAAAAC5JVACAAAAAADklkAJAAAAAACQWwIlAAAAAABAbgmUAAAAAAAAuSVQAgAAAAAA5JZACQAAAAAAkFutm7oCAOyb0gkLNRUAAAAANDAjSgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAIAGNmvWrCgtLY127drFoEGDYtmyZXss++KLL8Yll1ySlS8qKorKysrdykybNi1OP/30OPLII+O4446LESNGxMqVK71vAADQAARKAAAAGtD8+fNj/PjxMXXq1FixYkX0798/hg0bFhs2bKiz/Pvvvx8nnnhiTJ8+Pbp06VJnmccffzy+9a1vxVNPPRWPPvpofPjhh3H++edHVVWV9w4AAA5Q6wM9AQAAAB+ZOXNmjBs3LioqKrL12bNnx8KFC2POnDkxYcKE3ZoqjRRJS1LX/mTRokW11u+7775sZMny5cvjnHPO0fwAAHAAjCgBAABoINu3b8+CF0OHDv2o01VcnK0vXbq0wdp5y5Yt2b/HHHPMHsts27Yttm7dWmsBAAB2J1ACAADQQDZt2hQ7duyIzp0719qe1tetW9cgr7Fz58645ppr4u/+7u+ib9++eyyX8pqUlJTULD169GiQ1wcAgJbG1FsAAADNSMpV8sILL8QTTzyx13ITJ07McqVUSyNKBEsAmkbphIX1PmbV9OGNUhcAdidQAgAA0EA6deoUrVq1ivXr19fantb3lKi9Pq688sp4+OGH449//GN88pOf3GvZtm3bZgsAALB3pt4CAABoIG3atIkBAwbE4sWLa02VldYHDx683+ctFApZkOShhx6K//iP/4iePXs2UI0BAAAjSgAAABpQmu5q7NixMXDgwCgvL4/KysqoqqqKioqKbP+YMWOie/fuWQ6R6gTwL730Us3Pa9asieeeey6OOOKIOPnkk2um25o7d2789re/jSOPPLIm30nKPdK+fXvvHwAAHACBEgAAgAY0cuTI2LhxY0yZMiULaJSVlcWiRYtqEryvXr06ios/Gty/du3aOO2002rWb7vttmwZMmRILFmyJNt29913Z/+ee+65tV7r3nvvjUsvvdT7BwAAB0CgBAAAoIGlabLSUpfq4Ee10tLSbGqtvfm4/QAAwEHOUTJr1qzsZr5du3YxaNCgWLZs2R7L/uY3v8mGnB911FFx+OGHZ09TPfDAA7XKpCegioqKai0XXHDB/lQNAAAAAACg8UaUzJ8/P5tzd/bs2VmQJM23O2zYsFi5cmUcd9xxu5U/5phj4h//8R+jd+/eWWLDhx9+OJubN5VNx1VLgZE0bLxa27Zt61s1gGaldMLCpq4CAAAAAORevUeUzJw5M8aNG5cFO/r06ZMFTDp06BBz5syps3yaQ/dLX/pSnHLKKXHSSSfF1VdfHf369YsnnniiVrkUGOnSpUvNcvTRR+f+zQEAAAAAAA6hQMn27dtj+fLlMXTo0I9OUFycrS9duvRjj0/z6i5evDgbfXLOOefsNk9vGmXy6U9/Or75zW/GO++8s8fzbNu2LbZu3VprAQAAAAAAaNSptzZt2hQ7duyIzp0719qe1l955ZU9Hrdly5bo3r17FuBo1apV/PjHP46///u/rzXt1pe//OXo2bNnvP7663HDDTfEhRdemAVfUvm/NW3atLjxxhvrU3UAAAAAAIADz1GyP4488sh47rnn4r333stGlKQcJyeeeGI2LVcyatSomrKnnnpqNjVXmqYrjTI577zzdjvfxIkTs3NUSyNKevTocTB+FQAAAAAAIK+Bkk6dOmUjPNavX19re1pPeUX2JE3PdfLJJ2c/l5WVxcsvv5yNCqkOlPytFERJr/Xaa6/VGShJ+UwkewcAAAAAAA5qjpI2bdrEgAEDslEh1Xbu3JmtDx48eJ/Pk45J03DtyVtvvZXlKOnatWt9qgcAAAAAANC4U2+lKa/Gjh0bAwcOjPLy8qisrIyqqqqoqKjI9o8ZMybLR5JGjCTp31Q2TaWVgiOPPPJIPPDAA3H33Xdn+9N0XCnfyCWXXJKNSkk5Sq677rpsBMqwYcPqWz0AAAAAAIDGC5SMHDkyNm7cGFOmTIl169ZlU2ktWrSoJsH76tWrs6m2qqUgyhVXXJGNEmnfvn307t07fvGLX2TnSdJUXs8//3z8/Oc/j82bN0e3bt3i/PPPj5tuusn0WgAAAAAAQKMqKhQKhWjmUjL3kpKS2LJlS3Ts2LGpqwOwT0onLNRSh6BV04c3dRUA9ol7YOrLNQPkQUvqZ+mbABy8+996jygBAAAAgMbWkoIeALSgZO4AAAAAAAAtiUAJAAAAAACQWwIlAAAAAABAbgmUAAAAAAAAuSVQAgAAAAAA5JZACQAAAAAAkFsCJQAAAAAAQG61buoKALQEpRMWNnUVAAAAAID9YEQJAAAAAACQWwIlAAAAAABAbgmUAAAAAAAAuSVQAgAAAAAA5JZACQAAAAAAkFsCJQAAAAAAQG4JlAAAAAAAALklUAIAAAAAAOSWQAkAAAAAAJBbAiUAAAAAAEBuCZQAAAAAAAC5JVACAAAAAADklkAJAAAAAACQWwIlAAAAAABAbrVu6goAwKGkdMLCeh+zavrwRqkLAAAAAI3PiBIAAAAAACC3BEoAAAAAAIDcEigBAAAAAAByS6AEAAAAAADILYESAAAAAAAgtwRKAAAAAACA3BIoAQAAAAAAcqt1U1cAAAAAAKitdMLCejfJqunDNSPAfhAoAWiAm1EAAAAAoHky9RYAAEADmzVrVpSWlka7du1i0KBBsWzZsj2WffHFF+OSSy7JyhcVFUVlZeUBnxMAANh3AiUAAAANaP78+TF+/PiYOnVqrFixIvr37x/Dhg2LDRs21Fn+/fffjxNPPDGmT58eXbp0aZBzAgAAjRwoqc+TTL/5zW9i4MCBcdRRR8Xhhx8eZWVl8cADD9QqUygUYsqUKdG1a9do3759DB06NF599VXvIwAA0OzMnDkzxo0bFxUVFdGnT5+YPXt2dOjQIebMmVNn+dNPPz1uvfXWGDVqVLRt27ZBzgkAADRioKS+TzIdc8wx8Y//+I+xdOnSeP7557Mb+7T8/ve/rykzY8aMuOOOO7Kb/aeffjoLqKRzfvDBB/WtHgAAQJPZvn17LF++PHv4q1pxcXG2nvpEB/Oc27Zti61bt9ZaAACABgiU1PdJpnPPPTe+9KUvxSmnnBInnXRSXH311dGvX7944oknakaTpDl4J02aFBdffHG27/7774+1a9fGggUL6jynG34AAOBQtGnTptixY0d07ty51va0vm7duoN6zmnTpkVJSUnN0qNHj/16fQAAaOmKD+bTUSkosnjx4li5cmWcc8452bY33ngju7nf9ZzpJj5N6bWnc7rhBwAA2LuJEyfGli1bapY333xTkwEAQB1aRwM9yfTKK6/s8bh0U969e/dsJEirVq3ixz/+cfz93/99tq/6Caj6PB2VbvjT9F/V0hByT0cBAABNrVOnTlmfZ/369bW2p/U9JWpvrHOmfCd7ynkCAAAcYDL3+jryyCPjueeeiz//+c/xgx/8IAtyLFmyZL/Pl272O3bsWGsBAABoam3atIkBAwZkI+mr7dy5M1sfPHjwIXNOAABgP0eU7O+TTGl6rpNPPjn7uaysLF5++eVs+qyUv6T6uHSOrl271jpnKgsAANCcpAfDxo4dGwMHDozy8vIsJ2NVVVWW5zEZM2ZMNuI+9Ymqpzh+6aWXan5es2ZN9qDZEUccUdOP+rhzAgAABylQsuuTTCNGjKj1JNOVV165z+dJx6RpuJKePXtmwZJ0jurASJpK6+mnn45vfvOb9fttAAAAmtjIkSNj48aNMWXKlGw64dTPWbRoUc10w6tXr84eJqu2du3aOO2002rWb7vttmwZMmRIzUj8jzsnAABwkAIl+/N0VPo3lT3ppJOy4MgjjzwSDzzwQNx9993Z/qKiorjmmmvi5ptvjl69emWBk8mTJ0e3bt1qgjEAcCgrnbCw3sesmj68UeoCwKEhPUi2p4fJ/nYa4tLS0igUCgd0TgAA4CAGSur7dFQKolxxxRXx1ltvRfv27aN3797xi1/8IjtPteuuuy4r941vfCM2b94cZ511VnbOdu3aHcCvBgAAAAAAsHdFhX15dOkQl6bqKikpiS1btkjsDjTJ6ACoLyNKgAPlHhjXDNDS6ZvVn34GwP71mT4a+gEAAAAAAJAzAiUAAAAAAEBuCZQAAAAAAAC5JVACAAAAAADklkAJAAAAAACQW62bugIAAAAAtGylExY2dRUAYI+MKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC3BEoAAAAAAIDcEigBAAAAAAByS6AEAAAAAADILYESAAAAAAAgtwRKAAAAAACA3BIoAQAAAAAAckugBAAAAAAAyC2BEgAAAAAAILdaN3UFABpT6YSFGhgAAAAA2CMjSgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC3BEoAAAAAAIDcEigBAAAAAAByS6AEAAAAAADILYESAAAAAAAgtwRKAAAAAACA3BIoAQAAAAAAckugBAAAAAAAyC2BEgAAAAAAILcESgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC39itQMmvWrCgtLY127drFoEGDYtmyZXss+9Of/jTOPvvsOProo7Nl6NChu5W/9NJLo6ioqNZywQUX7E/VAAAAAAAAGi9QMn/+/Bg/fnxMnTo1VqxYEf37949hw4bFhg0b6iy/ZMmSGD16dDz22GOxdOnS6NGjR5x//vmxZs2aWuVSYOTtt9+uWX71q1/Vt2oAAAAAAACNGyiZOXNmjBs3LioqKqJPnz4xe/bs6NChQ8yZM6fO8r/85S/jiiuuiLKysujdu3f87Gc/i507d8bixYtrlWvbtm106dKlZkmjTwAAAAAAABpT6/oU3r59eyxfvjwmTpxYs624uDibTiuNFtkX77//fnz44YdxzDHH7Dby5LjjjssCJJ///Ofj5ptvjmOPPbbOc2zbti1bqm3durU+vwYANLnSCQvrVX7V9OGNVhcAAACAPKvXiJJNmzbFjh07onPnzrW2p/V169bt0zmuv/766NatWxZc2XXarfvvvz8bZXLLLbfE448/HhdeeGH2WnWZNm1alJSU1CxpOi8AAAAAAIBGHVFyoKZPnx7z5s3LRo+kRPDVRo0aVfPzqaeeGv369YuTTjopK3feeeftdp40oiXlSdl1RIlgCQAAAAAA0KgjSjp16hStWrWK9evX19qe1lNekb257bbbskDJH/7whywQsjcnnnhi9lqvvfZanftTPpOOHTvWWgAAAAAAABo1UNKmTZsYMGBArUTs1YnZBw8evMfjZsyYETfddFMsWrQoBg4c+LGv89Zbb8U777wTXbt2rU/1AAAAAAAAGnfqrTTl1dixY7OAR3l5eVRWVkZVVVVUVFRk+8eMGRPdu3fP8ogkKefIlClTYu7cuVFaWlqTy+SII47Ilvfeey9uvPHGuOSSS7JRKa+//npcd911cfLJJ8ewYcPqWz2gBatv8msAgKYya9asuPXWW7P+T//+/ePOO+/M+k978uCDD8bkyZNj1apV0atXr6wfddFFF9XsT/2mCRMmxIIFC7KHynr27Bnf/va34/LLLz9IvxEAALRc9RpRkowcOTKbRisFP8rKyuK5557LRopUJ3hfvXp1vP322zXl77777ti+fXt85StfyUaIVC/pHEmayuv555+PL37xi/GpT30qLrvssmzUyp/+9Kdsii0AAIDmZP78+dkDZlOnTo0VK1ZkgZL0ENiGDRvqLP/kk0/G6NGjs77Qs88+GyNGjMiWF154oaZMOl/qd/3iF7+Il19+Oa655pq48sor41//9V8P4m8GAAAtU1GhUChEM5eSuZeUlMSWLVvkK4EWzIgS8mzV9OFNXQXgEOMe+NA1aNCgOP300+Ouu+6qma64R48ecdVVV2WjQup6GC2N0n/44Ydrtp1xxhnZg2mzZ8/O1vv27ZuVS6NOqqUHzC688MK4+eab96lerhmgKenPHRz6DQD7d/9b7xElAAAA1C2Npl++fHkMHTr0o05XcXG2vnTp0jqPSdt3LZ+kESi7lj/zzDOz0SNr1qyJ9KzbY489Fv/1X/8V559//h7fim3btmWdw10XAACgAXKUAAAAULdNmzbFjh07aqYmrpbWX3nllTqPSXlM6ipfnd8xSTlOvvGNb8QnP/nJaN26dRZ8+elPfxrnnHPOHt+KlDcy5YMEID/2Z+SOUSgARpQAAAAc8lKg5KmnnspGlaQRK7fffnt861vfin//93/f4zETJ07MphmoXt58882DWmcAAGgujCgBAABoIJ06dYpWrVrF+vXra21P6126dKnzmLR9b+X/53/+J2644YZ46KGHYvjw/5ezql+/fvHcc8/Fbbfdttu0XdXatm2bLQAAwN7JUQIAANBA2rRpkyVZX7x4cc22lMw9rQ8ePLjOY9L2Xcsnjz76aE35Dz/8MFvSdFu7SgGZdG4AAODAGFECAADQgMaPHx9jx46NgQMHRnl5eVRWVkZVVVVUVFRk+8eMGRPdu3fPcogkV199dQwZMiSbTiuNGJk3b14888wzcc8992T7O3bsmO2/9tpro3379nHCCSfE448/Hvfff3/MnDnTewcAAAdIoAQAAKABjRw5MjZu3BhTpkzJErKXlZXFokWLahK2r169utbokDPPPDPmzp0bkyZNyqbY6tWrVyxYsCD69u1bUyYFT1LOka997Wvxl7/8JQuW/OAHP4jLL7/cewcAAAeoqFAoFKKZ27p1a5SUlGQJCtPTVkDLVDphYVNXAZrMqun/b056gGrugakv1wzQlPTnDl36GkBLVZ/7XzlKAAAAAACA3BIoAQAAAAAAckugBAAAAAAAyC2BEgAAAAAAILcESgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC3Wjd1BQAAAABoPkonLGzqKgBAgzKiBAAAAAAAyC2BEgAAAAAAILcESgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC3BEoAAAAAAIDcEigBAAAAAAByq3VTVwAA+HilExbWu5lWTR+uaQEAAAA+hhElAAAAAABAbgmUAAAAAAAAuSVQAgAAAAAA5JZACQAAAAAAkFsCJQAAAAAAQG4JlAAAAAAAALklUAIAAAAAAOSWQAkAAAAAAJBbAiUAAAAAAEBu7VegZNasWVFaWhrt2rWLQYMGxbJly/ZY9qc//WmcffbZcfTRR2fL0KFDdytfKBRiypQp0bVr12jfvn1W5tVXX92fqgEAAAAAADReoGT+/Pkxfvz4mDp1aqxYsSL69+8fw4YNiw0bNtRZfsmSJTF69Oh47LHHYunSpdGjR484//zzY82aNTVlZsyYEXfccUfMnj07nn766Tj88MOzc37wwQf1rR4AAAAAAEDjBUpmzpwZ48aNi4qKiujTp08W3OjQoUPMmTOnzvK//OUv44orroiysrLo3bt3/OxnP4udO3fG4sWLa0aTVFZWxqRJk+Liiy+Ofv36xf333x9r166NBQsW1Ld6AAAAAAAAjRMo2b59eyxfvjybGqvmBMXF2XoaLbIv3n///fjwww/jmGOOydbfeOONWLduXa1zlpSUZFN67emc27Zti61bt9ZaAAAAAAAAGjVQsmnTptixY0d07ty51va0noId++L666+Pbt261QRGqo+rzzmnTZuWBVOqlzSdFwAAAAAAwEFJ5r6/pk+fHvPmzYuHHnooSwS/vyZOnBhbtmypWd58880GrScAAAAAAJAPretTuFOnTtGqVatYv359re1pvUuXLns99rbbbssCJf/+7/+e5SGpVn1cOkfXrl1rnTPlNalL27ZtswUAAAAAAOCgjShp06ZNDBgwoCYRe1KdmH3w4MF7PG7GjBlx0003xaJFi2LgwIG19vXs2TMLlux6zpRz5Omnn97rOQEAAAAAAA7qiJJk/PjxMXbs2CzgUV5eHpWVlVFVVRUVFRXZ/jFjxkT37t2zPCLJLbfcElOmTIm5c+dGaWlpTd6RI444IluKiorimmuuiZtvvjl69eqVBU4mT56c5TEZMWLEAf+CAAAAAAAADRYoGTlyZGzcuDELfqSgR5oeK40UqU7Gvnr16igu/migyt133x3bt2+Pr3zlK7XOM3Xq1Pj+97+f/XzddddlwZZvfOMbsXnz5jjrrLOycx5IHhMAAAAAAICPU1QoFArRzKWpukpKSrLE7h07dmzq6gCNpHTCQm0L9bBq+nDtBS2Ye2BcM0BT0TdrWfQbgJaqPn2meuUoAQAAAAAAaEkESgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC3BEoAAAAAAIDcat3UFQAAAGhpZs2aFbfeemusW7cu+vfvH3feeWeUl5fvsfyDDz4YkydPjlWrVkWvXr3illtuiYsuuqhWmZdffjmuv/76ePzxx+Ovf/1r9OnTJ37961/H8ccffxB+IwBaqtIJC+tVftX04Y1WF4CmIlACNIsbMQCA5mL+/Pkxfvz4mD17dgwaNCgqKytj2LBhsXLlyjjuuON2K//kk0/G6NGjY9q0afEP//APMXfu3BgxYkSsWLEi+vbtm5V5/fXX46yzzorLLrssbrzxxujYsWO8+OKL0a5duyb4DQEAoGUpKhQKhWjmtm7dGiUlJbFly5aswwAc+gRKoPF50gtaNvfAh64UHDn99NPjrrvuytZ37twZPXr0iKuuuiomTJiwW/mRI0dGVVVVPPzwwzXbzjjjjCgrK8uCLcmoUaPisMMOiwceeGC/6+WaARqK/ly+6WcAzUV97n+NKAGAFmp/OrA6PQAHZvv27bF8+fKYOHFizbbi4uIYOnRoLF26tM5j0vY0AmVXaQTKggULagItCxcujOuuuy7b/uyzz0bPnj2z10gjT/Zk27Zt2bJrRxEAANidZO4AAAANZNOmTbFjx47o3Llzre1pPeUrqUvavrfyGzZsiPfeey+mT58eF1xwQfzhD3+IL33pS/HlL385y1eyJ2kqr/QEXfWSRrUAAAC7EygBAAA4hKURJcnFF18c3/nOd7IpudIUXimfSfXUXHVJI07SNAPVy5tvvnkQaw0AAM2HqbcAAAAaSKdOnaJVq1axfv36WtvTepcuXeo8Jm3fW/l0ztatW0efPn1qlTnllFPiiSee2GNd2rZtmy0AAMDeGVECAADQQNq0aRMDBgyIxYsX1xoRktYHDx5c5zFp+67lk0cffbSmfDpnSg6/cuXKWmX+67/+K0444QTvHQAAHCAjSgAAABpQSsw+duzYGDhwYJSXl0dlZWVUVVVFRUVFtn/MmDHRvXv3LIdIcvXVV8eQIUPi9ttvj+HDh8e8efPimWeeiXvuuafmnNdee22MHDkyzjnnnPjc5z4XixYtit/97nexZMkS7x0AABwggRIAAIAGlAIaGzdujClTpmQJ2VNOkRTYqE7Yvnr16igu/mhw/5lnnhlz586NSZMmxQ033BC9evWKBQsWRN++fWvKpOTtKR9JCq58+9vfjk9/+tPx61//Os466yzvHQAAHKCiQqFQiGZu69atUVJSkiUo7NixY1NXB9gHpRMWaic4BK2aPrypqwDsI/fA1JdrBmgo+nP5ps8AtMT7XzlKAAAAAACA3BIoAQAAAAAAckuOEqBBGHoNAAAAADRHAiUAAAAAOebBNwDyztRbAAAAAABAbgmUAAAAAAAAuSVQAgAAAAAA5JZACQAAAAAAkFsCJQAAAAAAQG4JlAAAAAAAALklUAIAAAAAAOSWQAkAAAAAAJBbrZu6AgDAoaN0wsJ6H7Nq+vBGqQsAAADAwWBECQAAAAAAkFsCJQAAAAAAQG4JlAAAAAAAALklUAIAAAAAAOSWQAkAAAAAAJBb+xUomTVrVpSWlka7du1i0KBBsWzZsj2WffHFF+OSSy7JyhcVFUVlZeVuZb7//e9n+3ZdevfuvT9VAwAAAAAAaLxAyfz582P8+PExderUWLFiRfTv3z+GDRsWGzZsqLP8+++/HyeeeGJMnz49unTpssfzfuYzn4m33367ZnniiSfqWzUAAAAAAIDGDZTMnDkzxo0bFxUVFdGnT5+YPXt2dOjQIebMmVNn+dNPPz1uvfXWGDVqVLRt23aP523dunUWSKleOnXqtMey27Zti61bt9ZaAAAAAAAAGjVQsn379li+fHkMHTr0oxMUF2frS5cujQPx6quvRrdu3bLRJ1/72tdi9erVeyw7bdq0KCkpqVl69OhxQK8NAAAAAADkU70CJZs2bYodO3ZE586da21P6+vWrdvvSqQ8J/fdd18sWrQo7r777njjjTfi7LPPjnfffbfO8hMnTowtW7bULG+++eZ+vzYAAAAAAJBfreMQcOGFF9b83K9fvyxwcsIJJ8S//Mu/xGWXXbZb+TSF196m8QIAAAAAAGjwESUpb0irVq1i/fr1tban9b0laq+vo446Kj71qU/Fa6+91mDnBAAAAAAAOKBASZs2bWLAgAGxePHimm07d+7M1gcPHhwN5b333ovXX389unbt2mDnBAAAAAAAOOCpt8aPHx9jx46NgQMHRnl5eVRWVkZVVVVUVFRk+8eMGRPdu3fPEq5XJ4B/6aWXan5es2ZNPPfcc3HEEUfEySefnG3/3ve+F1/4whey6bbWrl0bU6dOzUaujB49ur7VAwAAAAAAaLxAyciRI2Pjxo0xZcqULIF7WVlZloS9OsH76tWro7j4o4EqKfBx2mmn1azfdttt2TJkyJBYsmRJtu2tt97KgiLvvPNOfOITn4izzjornnrqqexnAAAAAACAQyqZ+5VXXpktdakOflQrLS2NQqGw1/PNmzdvf6oBAAAAAABw8HKUAAAAAAAAtCQCJQAAAAAAQG4JlAAAAAAAALklUAIAAAAAAOSWQAkAAAAAAJBbAiUAAAAAAEBuCZQAAAAAAAC51bqpKwAAAAAANA+lExbW+5hV04c3Sl0AGopACdAgNz0AAAAAAM2RqbcAAAAAAIDcEigBAAAAAAByS6AEAAAAAADILYESAAAAAAAgtwRKAAAAAACA3BIoAQAAAAAAcqt1U1cAaFylExZqYgCAg2zWrFlx6623xrp166J///5x5513Rnl5+R7LP/jggzF58uRYtWpV9OrVK2655Za46KKL6ix7+eWXx09+8pP44Q9/GNdcc00j/hYAAJAPRpQAAAA0oPnz58f48eNj6tSpsWLFiixQMmzYsNiwYUOd5Z988skYPXp0XHbZZfHss8/GiBEjsuWFF17YrexDDz0UTz31VHTr1s17BgAADUSgBAAAoAHNnDkzxo0bFxUVFdGnT5+YPXt2dOjQIebMmVNn+R/96EdxwQUXxLXXXhunnHJK3HTTTfHZz3427rrrrlrl1qxZE1dddVX88pe/jMMOO8x7BgAADUSgBAAAoIFs3749li9fHkOHDv2o01VcnK0vXbq0zmPS9l3LJ2kEyq7ld+7cGV//+tezYMpnPvOZfarLtm3bYuvWrbUWAABgdwIlAAAADWTTpk2xY8eO6Ny5c63taT3lK6lL2v5x5VPOktatW8e3v/3tfa7LtGnToqSkpGbp0aNHvX8fAADIA8ncAQAADmFphEqanivlOykqKtrn4yZOnJjlSqmWRpQIlkDLVzphYVNXAQCaHSNKAAAAGkinTp2iVatWsX79+lrb03qXLl3qPCZt31v5P/3pT1ki+OOPPz4bVZKW//7v/47vfve7UVpause6tG3bNjp27FhrAQAAdidQAgAA0EDatGkTAwYMiMWLF9fKL5LWBw8eXOcxafuu5ZNHH320pnzKTfL888/Hc889V7N069Yty1fy+9//3nsHAAAHyNRbAAAADShNdzV27NgYOHBglJeXR2VlZVRVVUVFRUW2f8yYMdG9e/csh0hy9dVXx5AhQ+L222+P4cOHx7x58+KZZ56Je+65J9t/7LHHZsuuDjvssGzEyac//WnvHQAAHCCBEgAAgAY0cuTI2LhxY0yZMiVLyF5WVhaLFi2qSdi+evXqKC7+aHD/mWeeGXPnzo1JkybFDTfcEL169YoFCxZE3759vS8AAHAQCJQAAAA0sCuvvDJb6rJkyZLdtn31q1/Nln21atWqA6ofAADwETlKAAAAAACA3BIoAQAAAAAAckugBAAAAAAAyC2BEgAAAAAAILcESgAAAAAAgNxq3dQVAACat9IJC+t9zKrpwxulLgAAAAD1ZUQJAAAAAACQWwIlAAAAAABAbu1XoGTWrFlRWloa7dq1i0GDBsWyZcv2WPbFF1+MSy65JCtfVFQUlZWVB3xOAAAAAACAJgmUzJ8/P8aPHx9Tp06NFStWRP/+/WPYsGGxYcOGOsu///77ceKJJ8b06dOjS5cuDXJOAAAAAACAJgmUzJw5M8aNGxcVFRXRp0+fmD17dnTo0CHmzJlTZ/nTTz89br311hg1alS0bdu2Qc4JAAAAAABw0AMl27dvj+XLl8fQoUM/OkFxcba+dOnS/arA/pxz27ZtsXXr1loLAAAAAABAowZKNm3aFDt27IjOnTvX2p7W161bV+8X399zTps2LUpKSmqWHj167NdrAwAAAAAA+dY6mqGJEydmOU2qpRElgiUA0HyUTlhYr/Krpg9vtLoAAAAA+VavQEmnTp2iVatWsX79+lrb0/qeErU3xjlTrpM95TsBAAAAAJrvg1KJh6WAQ3bqrTZt2sSAAQNi8eLFNdt27tyZrQ8ePHi/KtAY5wQAAAAAAGiUqbfSlFdjx46NgQMHRnl5eVRWVkZVVVVUVFRk+8eMGRPdu3fP8ohUJ2t/6aWXan5es2ZNPPfcc3HEEUfEySefvE/nBAAAAAAAOCQCJSNHjoyNGzfGlClTsmTrZWVlsWjRoppk7KtXr47i4o8GqqxduzZOO+20mvXbbrstW4YMGRJLlizZp3MCAAAAAAA0hqJCoVCIZi4lcy8pKYktW7ZEx44dm7o6cEjN6QnQEpifGHbnHpj6cs1APug30lLoAwAH8/63XjlKAAAAAAAAWhKBEgAAAAAAILcESgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC3BEoAAAAAAIDcEigBAAAAAAByq3VTVwAAAACA3ZVOWKhZAOAgMKIEAAAAAADILYESAAAAAAAgtwRKAAAAAACA3BIoAQAAAAAAckugBAAAAAAAyC2BEgAAAAAAILcESgAAAAAAgNwSKAEAAAAAAHJLoAQAAAAAAMgtgRIAAAAAACC3BEoAAAAAAIDcEigBAAAAAAByq3VTVwAA4OOUTlhY70ZaNX24hgUAAAA+lkAJNKMv/QBo3M9ZwRUAAADIH1NvAQAAAAAAuWVECQAAAABwSDFCHDiYjCgBAABoYLNmzYrS0tJo165dDBo0KJYtW7bX8g8++GD07t07K3/qqafGI488UrPvww8/jOuvvz7bfvjhh0e3bt1izJgxsXbtWu8bAAA0AIESAACABjR//vwYP358TJ06NVasWBH9+/ePYcOGxYYNG+os/+STT8bo0aPjsssui2effTZGjBiRLS+88EK2//3338/OM3ny5Ozf3/zmN7Fy5cr44he/6H0DAIAGUFQoFArRzG3dujVKSkpiy5Yt0bFjx6auDuwTydwBDj2SudOcuAc+dKURJKeffnrcdddd2frOnTujR48ecdVVV8WECRN2Kz9y5MioqqqKhx9+uGbbGWecEWVlZTF79uw6X+PPf/5zlJeXx3//93/H8ccfv0/1cs1A86PfCPXjfh7Y3/tfI0oAAAAayPbt22P58uUxdOjQjzpdxcXZ+tKlS+s8Jm3ftXySRqDsqXySOntFRUVx1FFH7bHMtm3bss7hrgsAALA7gRIAAIAGsmnTptixY0d07ty51va0vm7dujqPSdvrU/6DDz7Icpak6br29mTctGnTsifoqpc0qgUAANidQAkAAEAzkRK7/6//9b8izaB8991377XsxIkTs5En1cubb7550OoJAADNSeumrgAAAEBL0alTp2jVqlWsX7++1va03qVLlzqPSdv3pXx1kCTlJfmP//iPj51nuW3bttkCAADsnUAJAABAA2nTpk0MGDAgFi9eHCNGjKhJ5p7Wr7zyyjqPGTx4cLb/mmuuqdn26KOPZtv/Nkjy6quvxmOPPRbHHnus9wyaGYnZAeDQJVACAADQgMaPHx9jx46NgQMHRnl5eVRWVkZVVVVUVFRk+8eMGRPdu3fPcogkV199dQwZMiRuv/32GD58eMybNy+eeeaZuOeee2qCJF/5yldixYoV8fDDD2c5UKrzlxxzzDFZcAYAANh/AiUAAAfwpOeq6cO1H1DLyJEjY+PGjTFlypQsoFFWVhaLFi2qSdi+evXqKC7+KF3kmWeeGXPnzo1JkybFDTfcEL169YoFCxZE3759s/1r1qyJf/3Xf81+TufaVRpdcu6553oHAADgYCdznzVrVpSWlka7du1i0KBBsWzZsr2Wf/DBB6N3795Z+VNPPTUeeeSRWvsvvfTSKCoqqrVccMEF+1M1AACAJpem2Uq5RLZt2xZPP/101m+qtmTJkrjvvvtqlf/qV78aK1euzMq/8MILcdFFF9XsS32vlLy9rkWQBAAAmiBQMn/+/Gwo+dSpU7Oh3/37949hw4bFhg0b6iz/5JNPxujRo+Oyyy6LZ599NpunNy3p5n9XKTDy9ttv1yy/+tWv9v+3AgAAAAAAaIxAycyZM2PcuHHZ/Lp9+vSJ2bNnR4cOHWLOnDl1lv/Rj36UBUGuvfbaOOWUU+Kmm26Kz372s3HXXXfVKte2bdvo0qVLzXL00UfXt2oAAAAAAACNl6Nk+/btsXz58pg4cWLNtjS37tChQ2Pp0qV1HpO2pxEou0ojUNKcu7tKw8+PO+64LEDy+c9/Pm6++eY49thj6zxnGo6elmpbt26tz68BH8sc9QAAAAAA+VCvESWbNm2KHTt21CQhrJbWU5LCuqTtH1c+jTi5//77Y/HixXHLLbfE448/HhdeeGH2WnWZNm1alJSU1Cw9evSoz68BAAAAAABQ/xEljWXUqFE1P6dk7/369YuTTjopG2Vy3nnn7VY+jWjZdZRKGlEiWAIAAAAAADRqoKRTp07RqlWrWL9+fa3taT3lFalL2l6f8smJJ56YvdZrr71WZ6Ak5TNJCzT36boAAAAAAGhGU2+1adMmBgwYkE2RVW3nzp3Z+uDBg+s8Jm3ftXzy6KOP7rF88tZbb8U777wTXbt2rU/1AAAAAAAAGnfqrTTl1dixY2PgwIFRXl4elZWVUVVVFRUVFdn+MWPGRPfu3bM8IsnVV18dQ4YMidtvvz2GDx8e8+bNi2eeeSbuueeebP97770XN954Y1xyySXZKJPXX389rrvuujj55JOzpO8AAAAAAI0x28eq6cM1LFD/QMnIkSNj48aNMWXKlCwhe1lZWSxatKgmYfvq1aujuPijgSpnnnlmzJ07NyZNmhQ33HBD9OrVKxYsWBB9+/bN9qepvJ5//vn4+c9/Hps3b45u3brF+eefHzfddJPptQAAAAAAgEMvmfuVV16ZLXVJCdj/1le/+tVsqUv79u3j97///f5UAwAAAAAA4ODlKAEAAAAAAIi8jygBAOD/MQ8yAAAANG9GlAAAAAAAALllRAkAAABAI48oBQAOXUaUAAAAAAAAuSVQAgAAAAAA5JZACQAAAAAAkFsCJQAAAAAAQG4JlAAAAAAAALklUAIAAAAAAOSWQAkAAAAAAJBbAiUAAAAAAEButW7qCkBjK52wUCMDAAAAAFAngRIAAAAAIJfq+4DtqunDG60uQNMx9RYAAAAAAJBbRpQAAAAAuWW6ZgBAoAQAoBl8IWOIPwAAADQOU28BAAAAAAC5JVACAAAAAADklkAJAAAAAACQWwIlAAAAAABAbknmTotPfgsAefwbKPk7AAAA7BuBEgAAAACARnqI10NMcOgz9RYAAAAAAJBbRpQAAAAALYYpmwGA+hIoocm4eQUAAAAAoKkJlNBgBD4AAAAAAGhuBEoAAAAAABqJBPBw6BMoAQBogXTGAAAAYN8IlAAAkBFcAQAAII8ESqiTfCMAAAA0NX1TAOBgKD4orwIAAAAAAHAIMqIEAIBD/knfVdOHH5TXAaDxGB0C0Lifme6ZYf8JlOSAm1EAAAAAAKibQAkAAAAAQDNnFAoc5Bwls2bNitLS0mjXrl0MGjQoli1bttfyDz74YPTu3Tsrf+qpp8YjjzxSa3+hUIgpU6ZE165do3379jF06NB49dVX96dqAAAATU6fiTx8GVffBQCgxYwomT9/fowfPz5mz56dBUkqKytj2LBhsXLlyjjuuON2K//kk0/G6NGjY9q0afEP//APMXfu3BgxYkSsWLEi+vbtm5WZMWNG3HHHHfHzn/88evbsGZMnT87O+dJLL2XBlebA/NwAAIcO92Y0JX0mmhtBDID8MgoF/p+iQhrOUQ8pOHL66afHXXfdla3v3LkzevToEVdddVVMmDBht/IjR46MqqqqePjhh2u2nXHGGVFWVpYFW9LLd+vWLb773e/G9773vWz/li1bonPnznHffffFqFGjdjvntm3bsqVaKn/88cfHm2++GR07doym0Hfq75vkdQEAaDov3DisyV5769at2X345s2bo6SkpMnqwaHZZzpU+00cHPqnADR3TXmfTctRnz5TvUaUbN++PZYvXx4TJ06s2VZcXJxNlbV06dI6j0nb0wiUXaXRIgsWLMh+fuONN2LdunXZOaqlSqfORTq2rpv+NDrlxhtv3G17+qUBAOBgKals+rZ+9913BUoOIYdKnynRbwIAmqtD4T6blmNf+kz1CpRs2rQpduzYkT25tKu0/sorr9R5TLqhr6t82l69v3rbnsr8rdTp2LUjkZ7Q+stf/hLHHntsFBUV1edXyrXqiJonyrR1S+K61tYtketaW7dErusDl0YZpBv+NNKAQ8eh0mdK9Jv2n8+oxqFdG4+21bbNketW2zY3rtnm17b16TPVO0fJoaBt27bZsqujjjqqyerT3KWLz9B7bd3SuK61dUvkutbWLZHr+sCYcou90W86cD6jGod2bTzaVts2R65bbdvcuGabV9vua5+puD4n7dSpU7Rq1SrWr19fa3ta79KlS53HpO17K1/9b33OCQAAcCjSZwIAgOanXoGSNm3axIABA2Lx4sW1pr1K64MHD67zmLR91/LJo48+WlO+Z8+eWUBk1zJpqM3TTz+9x3MCAAAcivSZAACg+an31FspN8jYsWNj4MCBUV5eHpWVlVFVVRUVFRXZ/jFjxkT37t2zxIHJ1VdfHUOGDInbb789hg8fHvPmzYtnnnkm7rnnnmx/yilyzTXXxM033xy9evXKAieTJ0/O5g0bMWJEQ/++/M1Q/KlTp+42jRkNT1sfPNpaW7dErmtt3RK5rmnJ9JmaP59R2rW5cc1q2+bIdattmxvXbMtu26JCymhST3fddVfceuutWeLAsrKyuOOOO2LQoEHZvnPPPTdKS0vjvvvuqyn/4IMPxqRJk2LVqlVZMGTGjBlx0UUX1exPVUgNkYInmzdvjrPOOit+/OMfx6c+9amG+j0BAAAOGn0mAABoPvYrUAIAAAAAAJC7HCUAAAAAAAAtiUAJAAAAAACQWwIlAAAAAABAbgmUAAAAAAAAuSVQQo0vfvGLcfzxx0e7du2ia9eu8fWvfz3Wrl2rhRrYqlWr4rLLLouePXtG+/bt46STToqpU6fG9u3btXUD+8EPfhBnnnlmdOjQIY466ijt28BmzZoVpaWl2WfGoEGDYtmyZdq4gf3xj3+ML3zhC9GtW7coKiqKBQsWaONGMG3atDj99NPjyCOPjOOOOy5GjBgRK1eu1NaN4O67745+/fpFx44ds2Xw4MHxb//2b9oaaHb3kOnv8t8u8+bNO+h1bWntunr16hg+fHhWJv1Nvvbaa+Ovf/3rQa9rS5Du0//2Gp0+fXpTV6tZ0u9peN///vd3uz579+7dCK+U7/5ioVCIKVOmZN/xpe+fhg4dGq+++mqT1bclte2ll1662zV8wQUXNFl9m5N96X9/8MEH8a1vfSuOPfbYOOKII+KSSy6J9evXN3rdBEqo8bnPfS7+5V/+Jbs4f/3rX8frr78eX/nKV7RQA3vllVdi586d8ZOf/CRefPHF+OEPfxizZ8+OG264QVs3sBR8+upXvxrf/OY3tW0Dmz9/fowfPz4L8q1YsSL69+8fw4YNiw0bNmjrBlRVVZW1beqc0Xgef/zx7CbsqaeeikcffTQ+/PDDOP/887P2p2F98pOfzL6kWb58eTzzzDPx+c9/Pi6++OLs7yFAc7uHvPfee+Ptt9+uWVJHn/1v1x07dmRBklTuySefjJ///Odx3333ZV/ysX/+6Z/+qdY1etVVV2nKetLvaTyf+cxnal2fTzzxRCO+Wj77izNmzIg77rgj+87p6aefjsMPPzzrt6cvoTmwtk1SYGTXa/hXv/qVZm2g/vd3vvOd+N3vfhcPPvhgVj49yP/lL385Gl0B9uC3v/1toaioqLB9+3Zt1MhmzJhR6Nmzp3ZuJPfee2+hpKRE+zag8vLywre+9a2a9R07dhS6detWmDZtmnZuJOlP9kMPPaR9D4INGzZk7f34449r74Pg6KOPLvzsZz/T1kCzuof0d7nh2/WRRx4pFBcXF9atW1ez7e677y507NixsG3btgN4xXw64YQTCj/84Q+buhrNnn5P45g6dWqhf//+jXT2fPrbv0s7d+4sdOnSpXDrrbfWbNu8eXOhbdu2hV/96ldNVMvmqa6/+WPHji1cfPHFTVanltz/3rx5c+Gwww4rPPjggzVlXn755azM0qVLG7UuRpRQp7/85S/xy1/+Mhsafdhhh2mlRrZly5Y45phjtDPNQnrKLz0NnobtVisuLs7Wly5d2qR1g4b6TE58Ljeu9ORwmqYmPTmUpuACaG7S05CdOnWK8vLymDNnTjbFCfsv3Ueeeuqp0blz55pt6cnnrVu3Gnm4n9IozjRtyWmnnRa33nqraczqSb+ncaUpoNK0RieeeGJ87Wtfy6beo+G88cYbsW7dulr99pKSkmzabP32hrFkyZJs6qhPf/rT2WjJd955p4HOnO/+9/Lly7NRJrteu2lqvpQuorGv3daNenaaneuvvz7uuuuueP/99+OMM86Ihx9+uKmr1OK99tprceedd8Ztt93W1FWBfbJp06bsC85dO7FJWk9Ty0FzlqZGvOaaa+Lv/u7vom/fvk1dnRbpP//zP7PASBryn+abfeihh6JPnz5NXS2Aek9plKYPTLk0/vCHP8QVV1wR7733Xnz729/WkvspfaFX1/1l9T7qJ12Ln/3sZ7MvntJUZhMnTsymhpk5c6am3Ef6PY0nfVmfptZLXzCn6/LGG2+Ms88+O1544YUsbwEHrvpzs67PVZ+pBy5Nu5Wmgkr5h1PqgjSd/oUXXph9kd+qVasGeIX89r/XrVsXbdq02S2f2cG4do0oaeEmTJhQZ6LBXZddv9hMyfKeffbZ7GY//cceM2aMJ6Maqa2TNWvWZB+uaa7ecePGNeyb30LtTzsD1Ofp4NRBk5C38aQO8XPPPZfNk5yevBo7dmy89NJLLlKgWd1DTp48OevUpyf108Nm1113XfbEft64Nz902zvlEzz33HOjX79+cfnll8ftt9+ePaC3bdu2g1xr2F36Qjl9D5KuzzRy7JFHHonNmzdneXOhORg1alR88YtfzEZCphxl6UHzP//5z9koE5pv/9uIkhbuu9/9blx66aV7LZOGOVZLQ8fT8qlPfSpOOeWU6NGjR5Zcx5QYDd/WKRHR5z73uWx6s3vuuWcfXoH9aWcaXvqMSIHU9evX19qe1rt06aLJabauvPLK7Ab3j3/8Y5Z0nMaRng46+eSTs58HDBiQdSh+9KMfxU9+8hNNDjTbe8j0dPRNN92UfQndtm3byIuGbNd0H7ls2bJa26rvN91jHnh7p2v0r3/9a6xatSp7aIGPp99z8KQnx9P3UGnGDRpG9edm+hzt2rVrzfa0XlZWppkbWPrsTZ8Z6Ro+77zztO8B9L/TtZumPkzB011HlRyM75wESlq4T3ziE9myv8OfEk+cNHxbp5EkKUiSviC69957s/wONHw703hfcqZrd/HixdmTE9WfF2k9/aGD5ibNKX/VVVdlU0ClJ4DS8GkOnvT54V4DaO73kGmk3NFHH52rIElDt2t6OO8HP/hBbNiwIZvzPXn00UejY8eOpmhsgPZO12jqd1a3LR9Pv+fgSVMXpumLvv71rx/EV23ZUp8mfamc+unVgZGU86l6VDcN66233spylOwalGL/+t/p+6aULztdu5dcckm2beXKlVkeo8Z+kF+ghEz6oExPdJ511lnZDX76A5WGk5900klGkzSwFCRJQ6BPOOGELC/Jxo0ba/Z5UqphpQ/Rv/zlL9m/KadG6hwk6UnmNC8++y8N5U/T5QwcODBLYFpZWZklZK6oqNCsDdxh2PWpqpSQL13Haa7plMiMhhvuO3fu3Pjtb3+bzYlcPe9pSnbYvn17zdyA0vzoaaqFdP2+++67Wbunm+Pf//732hloNveQv/vd77KnGlNOx3bt2mVf5v/zP/9zfO9732vqqjfrdj3//POzgEj6onTGjBnZ3+NJkyZlf6fzFoA6UGmO/NTHTw/npXubtP6d73wn/vf//t9Zf599p9/TONLn5Re+8IXse5E028bUqVOzWQtGjx7t8mzA/mLK/XDzzTdHr169si+j0/d83bp1q3ngkf1r27SkvDrpS/z0PV76DjVNwZn+nqWp5Diw/nf697LLLss+f1NbpwcmUmAlBUnSvVejKkChUHj++ecLn/vc5wrHHHNMoW3btoXS0tLC5ZdfXnjrrbe0TwO79957C+m/Xl0LDWvs2LF1tvNjjz2mqRvAnXfeWTj++OMLbdq0KZSXlxeeeuop7drA0rVa1zWcrm0azp4+k9PnNQ3r//yf/1M44YQTss+NT3ziE4Xzzjuv8Ic//EEzA83qHvLf/u3fCmVlZYUjjjiicPjhhxf69+9fmD17dmHHjh1NXfVmf2++atWqwoUXXlho3759oVOnToXvfve7hQ8//LBJ690cLV++vDBo0KBCSUlJoV27doVTTjml8M///M+FDz74oKmr1izp9zS8kSNHFrp27ZrdE3bv3j1bf+211xrhlfLdX9y5c2dh8uTJhc6dO2ff9aV775UrVzZ1tZt9277//vuF888/P+vPHHbYYVn/Zty4cYV169Y1dbVbTP/7f/7nfwpXXHFF4eijjy506NCh8KUvfanw9ttvN3rdiv7/CgIAAAAAAOSOxAgAAAAAAEBuCZQAAAAAAAC5JVACAAAAAADklkAJAAAAAACQWwIlAAAAAABAbgmUAAAAAAAAuSVQAgAAAAAA5JZACQAAAAAAkFsCJQAAAAAAQG4JlAAAAAAAALklUAIAAAAAAERe/X86unjxJweutwAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Exemple pour discuter\n", "x = torch.randn(1000, 10) # 1000 examples de dimension 10\n", "h = torch.randn(10, 200) # couche cachée de 200 neurones et de 10 entrées\n", "y = x @ h\n", "print(x.mean(), x.std())\n", "print(y.mean(), y.std())\n", "plt.figure(figsize=(20,5))\n", "plt.subplot(121)\n", "plt.hist(x.view(-1).tolist(), 50, density=True);\n", "plt.subplot(122)\n", "plt.hist(y.view(-1).tolist(), 50, density=True);" ] }, { "cell_type": "markdown", "id": "c9af09ec", "metadata": {}, "source": [ "On voit que la distribution à droite est plus resserée que celle de gauche (regarder les coordonnées...).\n", "On voudrait préserver la distribution, pour que la déviation standard reste proche de 1." ] }, { "cell_type": "code", "execution_count": 93, "id": "048f8c81", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor(-0.0151) tensor(1.0040)\n", "tensor(-0.0461) tensor(15.6056)\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABkoAAAGsCAYAAACSD/sZAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAASANJREFUeJzt/QuUFtWdL+5/GwggKq2Ichtio5IgQSCBgDhGzUhE5SSSqIOMCaTHwclFg5Ko4HAzOgfESNBI5GhCokkYGdYxjBEPGUQxyyOBCHKM16UuO6jITQdQHEHh/a9dv393aGmk34am6a7nWavorqpd9VZXF2/Xfj+19y4pFAqFAAAAAAAAyKFmDX0AAAAAAAAADUVQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNxqEU3Arl27Yu3atXHkkUdGSUlJQx8OAADUu0KhEO+880507tw5mjXz/BP7pt4EAECeFIqoMzWJoCSFJF27dm3owwAAgIPutddei7/5m79x5tkn9SYAAPLotVrUmZpEUJJaklT+wG3btm3owwEAgHq3devW7GGhynth2Bf1JgAA8mRrEXWmJhGUVHa3lUISQQkAAHmi61mKvVbUmwAAyJOSWgzXoTNjAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbLRr6AAA4tJSNW1j0NhXThtbLsQAAADQF6lkAhzYtSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBu1SkomTVrVpSVlUXr1q1j4MCBsWLFilptd99990VJSUkMGzas2vJCoRCTJk2KTp06xWGHHRaDBw+Ol156qS6HBgAAAAAAUH9Bybx582Ls2LExefLkWLVqVfTp0yeGDBkSGzZs+NjtKioq4gc/+EF84Qtf2GPd9OnT4/bbb4/Zs2fH8uXL4/DDD8/2+f777xd7eAAAAAAAAPUXlMyYMSNGjx4d5eXl0bNnzyzcaNOmTcyZM2ev2+zcuTMuvfTSuOGGG+KEE07YozXJzJkzY8KECXHBBRdE796949577421a9fGggULij08AAAAAACA+glKduzYEStXrsy6xqraQbNm2fyyZcv2ut0Pf/jDOO644+Kyyy7bY92rr74a69atq7bP0tLSrEuvve1z+/btsXXr1moTAAAAAABAvQYlmzZtylqHdOjQodryNJ/Cjpo8/vjj8fOf/zzuvvvuGtdXblfMPqdOnZqFKZVT165di/kxAAAAAAAA6j6Ye22988478Y1vfCMLSdq3b3/A9jt+/PjYsmVL1fTaa68dsH0DAAAAAAD50aKYwinsaN68eaxfv77a8jTfsWPHPcq/8sor2SDuX/7yl6uW7dq16/974RYt4sUXX6zaLu2jU6dO1fbZt2/fGo+jVatW2QQAAAAAB1PZuIVOOECeW5S0bNky+vXrF0uWLKkWfKT5QYMG7VG+R48e8ec//zlWr15dNX3lK1+JL37xi9n3qcusbt26ZWHJ7vtMY44sX768xn0CAAAAAAA0SIuSZOzYsTFq1Kjo379/DBgwIGbOnBnbtm2L8vLybP3IkSOjS5cu2TgirVu3jl69elXb/qijjsq+7r78qquuiptuuim6d++eBScTJ06Mzp07x7Bhw/b/JwTIMU86AQAAAMABDkqGDx8eGzdujEmTJmWDrafusRYtWlQ1GPuaNWuiWbPihj659tprs7Dl8ssvj82bN8fpp5+e7TMFLQAAAACQN3V58K1i2tB6ORaApq6kUCgUopFLXXWVlpZmA7u3bdu2oQ8HIHctStyMAxx87oFxzQA0jEO55b66GUDd6kzFNf0AAAAAAABoQgQlAAAAAABAbhU9RgkADeNQbt4NAAAAAI2VFiUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5ZYwSABpk/JSKaUOdeQAAAAAanBYlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAUKRZs2ZFWVlZtG7dOgYOHBgrVqz42PLz58+PHj16ZOVPOeWUeOihh6rWffDBB3Hddddlyw8//PDo3LlzjBw5MtauXVttH+n1SkpKqk3Tpk3zuwMAgP0kKAEAACjCvHnzYuzYsTF58uRYtWpV9OnTJ4YMGRIbNmyosfwTTzwRI0aMiMsuuyyeeuqpGDZsWDY988wz2fr33nsv28/EiROzr/fff3+8+OKL8ZWvfGWPff3whz+MN998s2q68sor/e4AAGA/CUoAAACKMGPGjBg9enSUl5dHz549Y/bs2dGmTZuYM2dOjeVvu+22OPfcc+Oaa66Jk08+OW688cb43Oc+F3fccUe2vrS0NBYvXhx///d/H5/+9Kfj1FNPzdatXLky1qxZU21fRx55ZHTs2LFqSi1QAACA/SMoAQAAqKUdO3ZkAcbgwYP/Wqlq1iybX7ZsWY3bpOW7l09SC5S9lU+2bNmSda111FFHVVueuto65phj4rOf/Wzccsst8eGHH+51H9u3b4+tW7dWmwAAgD21qGEZAAAANdi0aVPs3LkzOnToUG15mn/hhRdqPGfr1q2rsXxaXpP3338/G7MkddfVtm3bquXf+973spYo7dq1y7rzGj9+fNb9VmrhUpOpU6fGDTfc4PcIAAD7ICgBAAA4RKSB3VMXXIVCIe68885q69K4KJV69+4dLVu2jH/+53/OApFWrVrtsa8UpOy+TWpR0rVr13r+CQAAoPERlAAAANRS+/bto3nz5rF+/fpqy9N8GjOkJml5bcpXhiR/+ctf4pFHHqnWmqQmAwcOzLreqqioyMY2+agUntQUoAAAANUZowQAAKCWUiuOfv36xZIlS6qW7dq1K5sfNGhQjduk5buXT9Lg7buXrwxJXnrppXj44YezcUj2ZfXq1dn4KMcdd5zfHwAA7ActSgAAAIqQurMaNWpU9O/fPwYMGBAzZ86Mbdu2RXl5ebZ+5MiR0aVLl6xLrGTMmDFx5plnxq233hpDhw6N++67L5588sm46667qkKSiy66KFatWhUPPvhgNgZK5fglaTySFM6kgd+XL18eX/ziF+PII4/M5q+++ur4+te/HkcffbTfHwAA7AdBCQAAQBGGDx8eGzdujEmTJmWBRt++fWPRokVVA7avWbMma+lR6bTTTou5c+fGhAkT4vrrr4/u3bvHggULolevXtn6N954Ix544IHs+7Sv3T366KNx1llnZV1opYBlypQpsX379ujWrVsWlOw+BgkAAFA3JYU0SmAjlwYlLC0tjS1btuyzH1+Axqps3MJoSiqmDW3oQwBo1NwD45oBaBiHct1MPQugbnUmY5QAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklsHcAWgU/frqaxcAAACA+qBFCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4ZowQAAACA3Cp2/EQAmh4tSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHKrTkHJrFmzoqysLFq3bh0DBw6MFStW7LXs/fffH/3794+jjjoqDj/88Ojbt2/86le/qlbmm9/8ZpSUlFSbzj333LocGgAAAAAAQP0N5j5v3rwYO3ZszJ49OwtJZs6cGUOGDIkXX3wxjjvuuD3Kt2vXLv7lX/4levToES1btowHH3wwysvLs7Jpu0opGPnFL35RNd+qVatiDw0AAAAAAKB+W5TMmDEjRo8enYUdPXv2zAKTNm3axJw5c2osf9ZZZ8VXv/rVOPnkk+PEE0+MMWPGRO/evePxxx+vVi4FIx07dqyajj766GIPDQAAAAAAoP5alOzYsSNWrlwZ48ePr1rWrFmzGDx4cCxbtmyf2xcKhXjkkUey1ic333xztXVLly7NWpmkgOTv/u7v4qabbopjjjmmxv1s3749mypt3bq1mB8D4JBQNm5hQx8CAAAAOa9nVkwbWi/HAtBkg5JNmzbFzp07o0OHDtWWp/kXXnhhr9tt2bIlunTpkoUbzZs3j5/+9KfxpS99qVq3W1/72teiW7du8corr8T1118f5513Xha+pPIfNXXq1LjhhhuKOXQAAAAAAID9H6OkLo488shYvXp1vPvuu7FkyZJsjJMTTjgh65YrueSSS6rKnnLKKVnXXKmbrtTK5Oyzz95jf6lFS9rH7i1KunbtejB+FAAAAAAAIK9BSfv27bMWHuvXr6+2PM2ncUX2JnXPddJJJ2Xf9+3bN55//vmsVUhlUPJRKURJr/Xyyy/XGJSk8UwM9g4AAAAAABzUoKRly5bRr1+/rFXIsGHDsmW7du3K5q+44opa7ydts/sYIx/1+uuvx1tvvRWdOnUq5vAAaML0tQsAAADAIdH1VuryatSoUdG/f/8YMGBAzJw5M7Zt2xbl5eXZ+pEjR2bjkaQWI0n6msqmrrRSOPLQQw/Fr371q7jzzjuz9ak7rjTeyIUXXpi1SkljlFx77bVZC5QhQ4Yc6J8XAAAAAACg7kHJ8OHDY+PGjTFp0qRYt25d1pXWokWLqgZ4X7NmTdbVVqUUonznO9/JWokcdthh0aNHj/j1r3+d7SdJXXk9/fTTcc8998TmzZujc+fOcc4558SNN96oey0AAAAAAKBelRQKhUI0cmkw99LS0tiyZUu0bdu2oQ8HoN66kqI4FdOGOmVAk+UeGNcMwIGR97qZehPQVBVTZ/pr0w8AAAAAAICcEZQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcqtFQx8AQFNQNm5hQx8CAAAAAFAHWpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3GrR0AcAcKgpG7ewoQ8BAAAAADhItCgBAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQWwZzB6DJKhu3sOhtKqYNrZdjAQAAAODQpEUJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAudWioQ8AAAAAAA6EsnELnUgAiqZFCQAAAAAAkFuCEgAAgCLNmjUrysrKonXr1jFw4MBYsWLFx5afP39+9OjRIyt/yimnxEMPPVS17oMPPojrrrsuW3744YdH586dY+TIkbF27dpq+3j77bfj0ksvjbZt28ZRRx0Vl112Wbz77rt+dwAA0BBBSTGVgvvvvz/69++f3cinm/6+ffvGr371q2plCoVCTJo0KTp16hSHHXZYDB48OF566aW6HBoAAEC9mjdvXowdOzYmT54cq1atij59+sSQIUNiw4YNNZZ/4oknYsSIEVmw8dRTT8WwYcOy6ZlnnsnWv/fee9l+Jk6cmH1NdagXX3wxvvKVr1TbTwpJnn322Vi8eHE8+OCD8Yc//CEuv/xyv20AANhPJYWUUhRZKUhPN82ePTsLSWbOnJk9HZVu5I877rg9yi9dujT+67/+K3t6qmXLltkN/fe///1YuHBhVplIbr755pg6dWrcc8890a1bt6yC8Oc//zmee+65LIzZl61bt0ZpaWls2bIle7oKYH/o0zbfKqYNbehDAKgV98ANJ9WDPv/5z8cdd9yRze/atSu6du0aV155ZYwbN26P8sOHD49t27ZldaFKp556avYQWapX1eRPf/pTDBgwIP7yl7/EJz/5yXj++eejZ8+e2fL0IFqyaNGiOP/88+P111/PWqHsi2sGyAP1ueKpAwFNVTH3v0W3KJkxY0aMHj06ysvLsxv1dGPfpk2bmDNnTo3lzzrrrPjqV78aJ598cpx44okxZsyY6N27dzz++OPZ+pTTpLBlwoQJccEFF2Tr7r333qyZ+YIFC2rc5/bt27MfcvcJAACgvu3YsSNWrlyZtYKv1KxZs2x+2bJlNW6Tlu9ePkkPje2tfJIqcyUlJVnL/Mp9pO8rQ5Ik7TO99vLly2vch3oTAADUTrP6rhTsLoUiS5YsyVqfnHHGGdmyV199NdatW1dtnynlSU9p7W2fqfVJKlM5pae3AAAA6tumTZti586d0aFDh2rL03yq19QkLS+m/Pvvv5+NWZK666p88i2V/WgL/hYtWkS7du32uh/1JgAAqIegpC6VgsqnoY444ois662hQ4fGT37yk/jSl76Uravcrph9jh8/Pttn5fTaa68V82MAAAAcktLA7n//93+fPWR255137te+1JsAAKB2WsRBcOSRR8bq1avj3XffzVqUpIEPTzjhhKxbrrpo1apVNgEAABxM7du3j+bNm8f69eurLU/zHTt2rHGbtLw25StDkjQuySOPPFKtH+VU9qODxX/44Yfx9ttv7/V11ZsAAKAeWpTUpVKQvUizZnHSSSdlgxWmgdwvuuiirBl4UrldsfsEAAA42FIr+X79+mUPgFVKg7mn+UGDBtW4TVq+e/lk8eLF1cpXhiQvvfRSPPzww3HMMcfssY/NmzdnXSFXSmFKeu3UbTEAAHCQgpK6VApqkrZJAwsm3bp1ywKR3feZBmdPAxIWs08AAICDIbWQv/vuu+Oee+6J559/Pr797W/Htm3bory8PFs/cuTIrNurSmPGjIlFixbFrbfeGi+88EJMmTIlnnzyybjiiiuqQpL0MFla9pvf/Cbr7jh1Q5ymNE5kcvLJJ8e5554bo0ePjhUrVsT//b//N9v+kksuic6dO/vFAwDAwex6K1UKRo0aFf37948BAwbEzJkz96gUdOnSparFSPqayp544olZOPLQQw/Fr371q6r+dktKSuKqq66Km266Kbp3754FJxMnTsxu9ocNG7Y/PxsAAMABN3z48Ni4cWNMmjQpCzNSy/kUhFSOu7hmzZqsVX2l0047LebOnRsTJkyI66+/Pqv3LFiwIHr16pWtf+ONN+KBBx7Ivk/72t2jjz5a1WVxClFSOHL22Wdn+7/wwgvj9ttv9xsGAICDHZQUWylIIcp3vvOdeP311+Owww6LHj16xK9//etsP5WuvfbarNzll1+eNSc//fTTs322bt16f38+AACAAy4FFpUtQj5q6dKleyy7+OKLs6kmZWVl2eDt+9KuXbsscAEAAA6skkJt7sgPcamrrtLS0tiyZUu1AQ8B6qJs3EInLscqpg1t6EMAqBX3wBTLNQPkgfpc8dSBgKaqmPvfoluUAAAAAAD5DJcEK0DkfTB3AAAAAACApkRQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAORWi4Y+AAA4lJSNW1j0NhXThtbLsQAAAABQ/7QoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQW4ISAAAAAAAgt+oUlMyaNSvKysqidevWMXDgwFixYsVey959993xhS98IY4++uhsGjx48B7lv/nNb0ZJSUm16dxzz63LoQEAAAAAANRfUDJv3rwYO3ZsTJ48OVatWhV9+vSJIUOGxIYNG2osv3Tp0hgxYkQ8+uijsWzZsujatWucc8458cYbb1Qrl4KRN998s2r6t3/7t2IPDQAAAAAAoH6DkhkzZsTo0aOjvLw8evbsGbNnz442bdrEnDlzaiz/m9/8Jr7zne9E3759o0ePHvGzn/0sdu3aFUuWLKlWrlWrVtGxY8eqKbU+AQAAAAAAOGSCkh07dsTKlSuz7rOqdtCsWTafWovUxnvvvRcffPBBtGvXbo+WJ8cdd1x8+tOfjm9/+9vx1ltv7XUf27dvj61bt1abAAAAAAAA6jUo2bRpU+zcuTM6dOhQbXmaX7duXa32cd1110Xnzp2rhS2p26177703a2Vy8803x2OPPRbnnXde9lo1mTp1apSWllZNqTsvAAAAAACAYrWIg2jatGlx3333Za1H0kDwlS655JKq70855ZTo3bt3nHjiiVm5s88+e4/9jB8/PhsnpVJqUSIsAQAAAAAA6jUoad++fTRv3jzWr19fbXmaT+OKfJwf/ehHWVDy8MMPZ0HIxznhhBOy13r55ZdrDErSeCZpAtiXsnELnSQAAAAA4MB0vdWyZcvo169ftYHYKwdmHzRo0F63mz59etx4442xaNGi6N+//z5f5/XXX8/GKOnUqVMxhwcAAAAAAFB/QUmSury6++6745577onnn38+G3h927ZtUV5enq0fOXJk1jVWpTTmyMSJE2POnDlRVlaWjWWSpnfffTdbn75ec8018cc//jEqKiqy0OWCCy6Ik046KYYMGVLs4QEAAAAAANTfGCXDhw+PjRs3xqRJk7LAo2/fvllLkcoB3tesWRPNmv01f7nzzjtjx44dcdFFF1Xbz+TJk2PKlClZV15PP/10Frxs3rw5G+j9nHPOyVqg6F4LAAAAIJ90pQzAIT2Y+xVXXJFNNUkDsO8utRL5OIcddlj8/ve/r8thAAAAAAAAHPygBKAheJoIAAAAAGjwMUoAAAAAAACaCkEJAAAAAACQW7reAoAG6BauYtpQ5x0AAADgEKBFCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4ZowQAAAAAqBVjNAJNkRYlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAKNKsWbOirKwsWrduHQMHDowVK1Z8bPn58+dHjx49svKnnHJKPPTQQ9XW33///XHOOefEMcccEyUlJbF69eo99nHWWWdl63afvvWtb/ndAQDAfhKUAAAAFGHevHkxduzYmDx5cqxatSr69OkTQ4YMiQ0bNtRY/oknnogRI0bEZZddFk899VQMGzYsm5555pmqMtu2bYvTTz89br755o997dGjR8ebb75ZNU2fPt3vDgAA9lOL/d0BAABAnsyYMSMLLMrLy7P52bNnx8KFC2POnDkxbty4Pcrfdtttce6558Y111yTzd94442xePHiuOOOO7Jtk2984xvZ14qKio997TZt2kTHjh1rdZzbt2/Ppkpbt24t4qcEAID80KIEAACglnbs2BErV66MwYMH/7VS1axZNr9s2bIat0nLdy+fpBYoeyv/cX7zm99E+/bto1evXjF+/Ph477339lp26tSpUVpaWjV17dq16NcDAIA80KIEAACgljZt2hQ7d+6MDh06VFue5l944YUat1m3bl2N5dPyYvzDP/xDHH/88dG5c+d4+umn47rrrosXX3wxG9+kJilISV2E7d6iRFgCAAB7EpQAAAA0ApdffnnV92lA+E6dOsXZZ58dr7zySpx44ol7lG/VqlU2AQAAH0/XWwAAALWUur1q3rx5rF+/vtryNL+3sUPS8mLK19bAgQOzry+//PJ+7QcAAPJOUAIAAFBLLVu2jH79+sWSJUuqlu3atSubHzRoUI3bpOW7l0/SYO57K19bq1evzr6mliUAAEDd6XoLAACgCGncj1GjRkX//v1jwIABMXPmzNi2bVuUl5dn60eOHBldunTJBlNPxowZE2eeeWbceuutMXTo0LjvvvviySefjLvuuqtqn2+//XasWbMm1q5dm82nsUeS1OokTal7rblz58b5558fxxxzTDZGydVXXx1nnHFG9O7d2+8PAAD2g6AEAACgCMOHD4+NGzfGpEmTsgHZ+/btG4sWLaoasD0FHs2a/bXx/mmnnZaFHBMmTIjrr78+unfvHgsWLIhevXpVlXnggQeqgpbkkksuyb5Onjw5pkyZkrVkefjhh6tCmTQo+4UXXpjtEwAA2D8lhUKhEI3c1q1bo7S0NLZs2RJt27Zt6MMB6knZuIXOLU1GxbShDX0IQCPnHhjXDNDUqQM2Heo/wKFeZzJGCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAORWi4Y+ACCfysYtbOhDAAAAAADQogQAAAAAAMgvXW8BAAAAAAC5JSgBAAAAAAByq05ByaxZs6KsrCxat24dAwcOjBUrVuy17N133x1f+MIX4uijj86mwYMH71G+UCjEpEmTolOnTnHYYYdlZV566aW6HBoAAAAAAED9BSXz5s2LsWPHxuTJk2PVqlXRp0+fGDJkSGzYsKHG8kuXLo0RI0bEo48+GsuWLYuuXbvGOeecE2+88UZVmenTp8ftt98es2fPjuXLl8fhhx+e7fP9998v9vAAAAAAAADqLyiZMWNGjB49OsrLy6Nnz55ZuNGmTZuYM2dOjeV/85vfxHe+853o27dv9OjRI372s5/Frl27YsmSJVWtSWbOnBkTJkyICy64IHr37h333ntvrF27NhYsWFDs4QEAAAAAANRPULJjx45YuXJl1jVW1Q6aNcvmU2uR2njvvffigw8+iHbt2mXzr776aqxbt67aPktLS7Muvfa2z+3bt8fWrVurTQAAAAAAAPUalGzatCl27twZHTp0qLY8zaewozauu+666Ny5c1UwUrldMfucOnVqFqZUTqk7LwAAAAAAgIMymHtdTZs2Le6777747W9/mw0EX1fjx4+PLVu2VE2vvfbaAT1OAAAAAAAgH1oUU7h9+/bRvHnzWL9+fbXlab5jx44fu+2PfvSjLCh5+OGHs3FIKlVul/bRqVOnavtM45rUpFWrVtkEAAAAAABw0FqUtGzZMvr161c1EHtSOTD7oEGD9rrd9OnT48Ybb4xFixZF//79q63r1q1bFpbsvs805sjy5cs/dp8AAAAAAAAHtUVJMnbs2Bg1alQWeAwYMCBmzpwZ27Zti/Ly8mz9yJEjo0uXLtk4IsnNN98ckyZNirlz50ZZWVnVuCNHHHFENpWUlMRVV10VN910U3Tv3j0LTiZOnJiNYzJs2LD9/gEBAAAAAAAOWFAyfPjw2LhxYxZ+pNAjdY+VWopUDsa+Zs2aaNbsrw1V7rzzztixY0dcdNFF1fYzefLkmDJlSvb9tddem4Utl19+eWzevDlOP/30bJ/7M44JAAAAAADAvpQUCoVCNHKpq67S0tJsYPe2bds29OEAtVA2bqHzRK5VTBva0IcANHLugXHNAE2demPTof4DHOp1pqLGKAEAAAAAAGhKBCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADkVouGPgAAAAAAmraycQsb+hAAYK+0KAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcatHQBwAAAAAANF1l4xYWvU3FtKH1ciwANRGUAEAjqCioJAAAAADUD11vAQAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5VaegZNasWVFWVhatW7eOgQMHxooVK/Za9tlnn40LL7wwK19SUhIzZ87co8yUKVOydbtPPXr0qMuhAQ2kbNzCoiYAgMasmDpRMn/+/KyOk8qfcsop8dBDD1Vbf//998c555wTxxxzTFYfWr169R77eP/99+O73/1uVuaII47I6lnr168/4D8bAADkTdFBybx582Ls2LExefLkWLVqVfTp0yeGDBkSGzZsqLH8e++9FyeccEJMmzYtOnbsuNf9fuYzn4k333yzanr88ceLPTQAAIB6V2yd6IknnogRI0bEZZddFk899VQMGzYsm5555pmqMtu2bYvTTz89br755r2+7tVXXx2/+93vstDlsccei7Vr18bXvva1evkZAQAgT0oKhUKhmA3S01Kf//zn44477sjmd+3aFV27do0rr7wyxo0b97Hbpieurrrqqmz6aIuSBQsW1PjUVG1s3bo1SktLY8uWLdG2bds67QPYP1qJwKGnYtrQhj4EoB65B244xdaJhg8fngUhDz74YNWyU089Nfr27RuzZ8+uVraioiK6deuWBSppfaVU1zn22GNj7ty5cdFFF2XLXnjhhTj55JNj2bJl2f72xTUDNCR1RoqlPgPsr2Luf4tqUbJjx45YuXJlDB48+K87aNYsm0835/vjpZdeis6dO2etTy699NJYs2bNXstu3749+yF3nwAAAOpbXepEafnu5ZPUAqWYOlR6zQ8++KDaflJXXp/85Cf3uh/1JgAAqJ2igpJNmzbFzp07o0OHDtWWp/l169bF/jyR9ctf/jIWLVoUd955Z7z66qvxhS98Id55550ay0+dOjVLgiqn9PQWAABAfatLnSgt3986VCrbsmXLOOqoo2q9H/UmAACox8HcD7TzzjsvLr744ujdu3f2ZFUa2HDz5s3x7//+7zWWHz9+fNZcpnJ67bXXDvoxAwAAHMrUmwAAoHZaRBHat28fzZs3j/Xr11dbnuY/bqD2YqWnpD71qU/Fyy+/XOP6Vq1aZRMAAMDBVJc6UVq+v3WoVDZ1+5UeKNu9VcnH7Ue9CQAA6qFFSWrq3a9fv1iyZEnVsjRwYZofNGhQHCjvvvtuvPLKK9GpU6cDtk8AAID9VZc6UVq+e/lk8eLFRdWh0mt+4hOfqLafF198MRvb8UDWxQAAII+KalGSjB07NkaNGhX9+/ePAQMGxMyZM2Pbtm1RXl6erR85cmR06dIl6w83SU89Pffcc1Xfv/HGG7F69eo44ogj4qSTTsqW/+AHP4gvf/nLcfzxx8fatWtj8uTJ2VNaI0aMOLA/LQAAwH4qtk40ZsyYOPPMM+PWW2+NoUOHxn333RdPPvlk3HXXXVX7fPvtt7PQI9WHKkOQJLUWSVMam/Gyyy7LXrtdu3bRtm3buPLKK7OQ5NRTT/U7BQCAgxmUDB8+PDZu3BiTJk3KBg3s27dvNgh75eCE6ea+WbO/NlRJN/qf/exnq+Z/9KMfZVOqKCxdujRb9vrrr2ehyFtvvRXHHntsnH766fHHP/4x+x4AAOBQUmyd6LTTTou5c+fGhAkT4vrrr4/u3bvHggULolevXlVlHnjggaqgJbnkkkuyr+khsilTpmTf//jHP872e+GFF8b27duz8R1/+tOfHsSfHAAAmqaSQqFQiEZu69at2RNWaWD39GQVcPCVjVvotMMhpmLa0IY+BKAeuQfGNQM0JuqMFEt9BjiYdaaixigBAAAAAABoSgQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFstGvoAAAAAAGg8ysYtbOhDAIADSosSAAAAAAAgt7QoAfbg6SAAAAAAIC+0KAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtY5QAAAAAAI1+/NSKaUPr5ViApk9QAgBNlIoFAAAAwL7pegsAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyK06BSWzZs2KsrKyaN26dQwcODBWrFix17LPPvtsXHjhhVn5kpKSmDlz5n7vEwAAAAAA4EBoUewG8+bNi7Fjx8bs2bOzQCMFH0OGDIkXX3wxjjvuuD3Kv/fee3HCCSfExRdfHFdfffUB2ScAUD/Kxi0sepuKaUPr5VgAAAAADskWJTNmzIjRo0dHeXl59OzZMws32rRpE3PmzKmx/Oc///m45ZZb4pJLLolWrVodkH0CAAAAAAAc9KBkx44dsXLlyhg8ePBfd9CsWTa/bNmyOh1AXfa5ffv22Lp1a7UJAAAAAACgXoOSTZs2xc6dO6NDhw7Vlqf5devWFf3idd3n1KlTo7S0tGrq2rVrnV4bAAAAAADItzoN5t7Qxo8fH1u2bKmaXnvttYY+JAAAAAAAoKkP5t6+ffto3rx5rF+/vtryNN+xY8c6HUBd9pnGOtnbeCcAAAAAAAD1EpS0bNky+vXrF0uWLIlhw4Zly3bt2pXNX3HFFcXsql73CQAAAEDtlI1b6FQBkGtFBSXJ2LFjY9SoUdG/f/8YMGBAzJw5M7Zt2xbl5eXZ+pEjR0aXLl2ycUQqB2t/7rnnqr5/4403YvXq1XHEEUfESSedVKt9AgAAAAAAHBJByfDhw2Pjxo0xadKkbLD1vn37xqJFi6oGY1+zZk00a/bXoU/Wrl0bn/3sZ6vmf/SjH2XTmWeeGUuXLq3VPgEAAAAAAA6JoCRJXWLtrVusyvCjUllZWRQKhf3aJwAAAAAAQH34a9MPAAAAAACAnBGUAAAAAAAAuVWnrrcAAAAAAA4lZeMWFr1NxbSh9XIsQOMiKAEA9ovKCAAAANCY6XoLAAAAAADILS1KoImry5PeAAAAAAB5oUUJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAKNKsWbOirKwsWrduHQMHDowVK1Z8bPn58+dHjx49svKnnHJKPPTQQ9XWFwqFmDRpUnTq1CkOO+ywGDx4cLz00kvVyqTXKykpqTZNmzbN7w4AAPaToAQAAKAI8+bNi7Fjx8bkyZNj1apV0adPnxgyZEhs2LChxvJPPPFEjBgxIi677LJ46qmnYtiwYdn0zDPPVJWZPn163H777TF79uxYvnx5HH744dk+33///Wr7+uEPfxhvvvlm1XTllVf63QEAwH4SlAAAABRhxowZMXr06CgvL4+ePXtm4UabNm1izpw5NZa/7bbb4txzz41rrrkmTj755Ljxxhvjc5/7XNxxxx1VrUlmzpwZEyZMiAsuuCB69+4d9957b6xduzYWLFhQbV9HHnlkdOzYsWpKgQoAALB/BCUAAAC1tGPHjli5cmXWNVZVpapZs2x+2bJlNW6Tlu9ePkmtRSrLv/rqq7Fu3bpqZUpLS7MuvT66z9TV1jHHHBOf/exn45ZbbokPP/xwr8e6ffv22Lp1a7UJAADYU4salgEAAFCDTZs2xc6dO6NDhw7Vlqf5F154ocZzlkKQmsqn5ZXrK5ftrUzyve99L2uJ0q5du6w7r/Hjx2fdb6UWLjWZOnVq3HDDDX6PAACwD4ISaETKxi1s6EMAAKCBpHFRKqXuuVq2bBn//M//nAUirVq12qN8ClJ23ya1KOnatetBO14AAGgsdL0FAABQS+3bt4/mzZvH+vXrqy1P82nMkJqk5R9XvvJrMftMUtdcqeutioqKGten8KRt27bVJgAAYE+CEgAAgFpKrTj69esXS5YsqVq2a9eubH7QoEE1bpOW714+Wbx4cVX5bt26ZYHI7mVS64/ly5fvdZ/J6tWrs/FRjjvuOL8/AADYD7reAgAAKELqzmrUqFHRv3//GDBgQMycOTO2bdsW5eXl2fqRI0dGly5dsi6xkjFjxsSZZ54Zt956awwdOjTuu+++ePLJJ+Ouu+7K1peUlMRVV10VN910U3Tv3j0LTiZOnBidO3eOYcOGZWXSoO4pOPniF78YRx55ZDZ/9dVXx9e//vU4+uij/f4AAGA/CEoAAACKMHz48Ni4cWNMmjQpG2y9b9++sWjRoqrB2NesWZO19Kh02mmnxdy5c2PChAlx/fXXZ2HIggULolevXlVlrr322ixsufzyy2Pz5s1x+umnZ/ts3bp1VTdaKWCZMmVKbN++PQtTUlCy+xgkAABA3ZQUCoVCNHKpWXppaWls2bJFv7s0aQZzB5qKimlDG/oQoNFzD4xrBjhQ1DXJM3UTaLqKqTMZowQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEButWjoAwAAAADgwDAwOwAUT1ACAAAAAORSseFixbSh9XYsQMPR9RYAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcMpg7AHDQGTARAAAAaNRByaxZs+KWW26JdevWRZ8+feInP/lJDBgwYK/l58+fHxMnToyKioro3r173HzzzXH++edXrf/mN78Z99xzT7VthgwZEosWLarL4UGT/JAQAAAAAIBDoOutefPmxdixY2Py5MmxatWqLChJocaGDRtqLP/EE0/EiBEj4rLLLounnnoqhg0blk3PPPNMtXLnnntuvPnmm1XTv/3bv9X9pwIAAAAAAKiPoGTGjBkxevToKC8vj549e8bs2bOjTZs2MWfOnBrL33bbbVkIcs0118TJJ58cN954Y3zuc5+LO+64o1q5Vq1aRceOHaumo48+uthDAwAAAAAAqL+gZMeOHbFy5coYPHjwX3fQrFk2v2zZshq3Sct3L5+kFigfLb906dI47rjj4tOf/nR8+9vfjrfeemuvx7F9+/bYunVrtQkAAAAAAKBeg5JNmzbFzp07o0OHDtWWp/k0XklN0vJ9lU8tTu69995YsmRJNn7JY489Fuedd172WjWZOnVqlJaWVk1du3Yt5scAAAAAAACo+2DuB9oll1xS9f0pp5wSvXv3jhNPPDFrZXL22WfvUX78+PHZOCmVUosSYQkAAAAAAFCvLUrat28fzZs3j/Xr11dbnubTuCI1ScuLKZ+ccMIJ2Wu9/PLLNa5P45m0bdu22gQAAAAAAFCvQUnLli2jX79+WRdZlXbt2pXNDxo0qMZt0vLdyyeLFy/ea/nk9ddfz8Yo6dSpUzGHBwAAAAAAUH9BSZK6vLr77rvjnnvuieeffz4beH3btm1RXl6erR85cmTWNValMWPGxKJFi+LWW2+NF154IaZMmRJPPvlkXHHFFdn6d999N6655pr44x//GBUVFVmocsEFF8RJJ52UDfoOAAAAAABwyIxRMnz48Ni4cWNMmjQpG5C9b9++WRBSOWD7mjVrolmzv+Yvp512WsydOzcmTJgQ119/fXTv3j0WLFgQvXr1ytanrryefvrpLHjZvHlzdO7cOc4555y48cYbsy62AAAAAAAADqnB3FNrkMoWIR+VBmD/qIsvvjibanLYYYfF73//+7ocBgAAAAAAwMEPSgAAAACoX2XjFjrFAHAojlECAAAAAADQVGhRAgAAAABQTy29KqYNdW7hECcoAQAOeSojAAAAQH3R9RYAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADkVouGPgBoKsrGLWzoQwAAAAAAoEiCEgCgSapLgF0xbWi9HAsAAABw6NL1FgAAAAAAkFtalAAAAADUM901A8ChS1ACNXADCwAAAACQD4ISAAAAAIB6YvxEOPQZowQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3jFECAPD/p+9gAAAAyB8tSgAAAAAAgNzSooQmry5PBwMAAIB6JgDkg6AEAAAAAOAQoltgOLh0vQUAAAAAAOSWoAQAAAAAAMgtXW/RqBhvBAAAAACAA0mLEgAAAAAAILe0KAEAOMgMzAgAhw49FwBNhXoG1J2gBABgP/hwBQAAABo3QQkN9kFRxbShzj4AAAAAAA3KGCUAAAAAAEBuaVFCg9FVCQDU399NLTcByCt1TYD6fc9U16Ap0qIEAAAAAADIrTq1KJk1a1bccsstsW7duujTp0/85Cc/iQEDBuy1/Pz582PixIlRUVER3bt3j5tvvjnOP//8qvWFQiEmT54cd999d2zevDn+9m//Nu68886sLAAAwKGmIepEb7/9dlx55ZXxu9/9Lpo1axYXXnhh3HbbbXHEEUfU+88LDUXrEIBDj9buNEVFByXz5s2LsWPHxuzZs2PgwIExc+bMGDJkSLz44otx3HHH7VH+iSeeiBEjRsTUqVPjf/yP/xFz586NYcOGxapVq6JXr15ZmenTp8ftt98e99xzT3Tr1i2rQKR9Pvfcc9G6desD85M2EQfrJlETOgBo3DShh/rTUHWiSy+9NN58881YvHhxfPDBB1FeXh6XX355tj9oDIQeAPmkbkJjUFJIjy4VIVUEPv/5z8cdd9yRze/atSu6du2aPdk0bty4PcoPHz48tm3bFg8++GDVslNPPTX69u2bVSzSy3fu3Dm+//3vxw9+8INs/ZYtW6JDhw7xy1/+Mi655JI99rl9+/ZsqpTKf/KTn4zXXnst2rZtGw2h1+TfN8jrAgA0pGduGHJQ7pvq8jpN5bj2ZuvWrdl9eGp9UFpa2mDHkUcNUSd6/vnno2fPnvGnP/0p+vfvn5VZtGhR1irl9ddfz7ZvDPUmmg51YAAONQ15b86hqag6U6EI27dvLzRv3rzw29/+ttrykSNHFr7yla/UuE3Xrl0LP/7xj6stmzRpUqF3797Z96+88koKagpPPfVUtTJnnHFG4Xvf+16N+5w8eXK2jck5cA24BlwDrgHXgGvANeAayPs18NprrxVzS89+aqg60c9//vPCUUcdVW39Bx98kB3L/fffX+Prqjc1/P9Pk3PgGnANuAZcA64B14BrIBpFnamorrc2bdoUO3fuzJ5s2l2af+GFF2rcJvXZW1P5tLxyfeWyvZX5qPHjx2dN3SulJ7hSf73HHHNMlJSUxIFMmzxt5Rw2FNeg89eQXH/On+uv8fL/Nz/nL7VCeOedd2psSUD9aag6Ufr60W69WrRoEe3atduvelNjuuYbI+fX+W3sXMPOb2Pm+nV+GzvX8MGtM9VpMPeG1qpVq2za3VFHHVUvr5UqCyoMzmFDcg06f66/xsv/X+fP9dd4NZb/v7rc4kDVmxrLNd9YOb/Ob2PnGnZ+GzPXr/Pb2LmGD06dqVkxO23fvn00b9481q9fX215mu/YsWON26TlH1e+8msx+wQAAGgIDVUnSl83bNhQbf2HH36YtRBRbwIAgP1TVFDSsmXL6NevXyxZsqRa8+00P2jQoBq3Sct3L58sXry4qny3bt2yG/vdy6RmRcuXL9/rPgEAABpCQ9WJ0tc0COXKlSuryjzyyCPZa6fB5QEAgLoruuut1MftqFGjon///jFgwICYOXNmbNu2LcrLy7P1I0eOjC5dusTUqVOz+TFjxsSZZ54Zt956awwdOjTuu+++ePLJJ+Ouu+7K1qe+ca+66qq46aabonv37lklYeLEiVm/YcOGDYuGkpqoT548eY+m6jiHrsHGwf9h58/113j5/+v8uf441DVEnejkk0+Oc889N0aPHh2zZ8+ODz74IK644oq45JJL9mucGu+59cv5dX4bO9ew89uYuX6d38bONXxwlaQR3Yvd6I477ohbbrklGzSwb9++cfvtt1c9xXTWWWdFWVlZ/PKXv6wqP3/+/JgwYUJUVFRkN/7Tp0+P888/v2p9OoQUSqSKQnpK6vTTT4+f/vSn8alPfepA/ZwAAAAHTEPUiVI3Wykc+d3vfhfNmjWLCy+8MHvdI444wm8WAAAOdlACAAAAAACQuzFKAAAAAAAAmhJBCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSoq0ffv26Nu3b5SUlMTq1avr57fSBH3lK1+JT37yk9G6devo1KlTfOMb34i1a9c29GE1ChUVFXHZZZdFt27d4rDDDosTTzwxJk+eHDt27GjoQ2s0/vVf/zVOO+20aNOmTRx11FENfTiHvFmzZkVZWVn2/3XgwIGxYsWKhj6kRuMPf/hDfPnLX47OnTtnfycWLFjQ0IfUqEydOjU+//nPx5FHHhnHHXdcDBs2LF588cWGPqxG484774zevXtH27Zts2nQoEHxf/7P/2now2q0pk2blv0/vuqqqxr6UOCg3Q+tWbMmhg4dmpVJ78PXXHNNfPjhh9XKLF26ND73uc9Fq1at4qSTTopf/vKXfkM1SOcpvYfUNP3pT3+qus+vaf0f//hH57QW0v3qR89deu/e3dNPPx1f+MIXsvvarl27xvTp053bA1QHdf3uP/Wug1eHOOuss/Z4v/jWt751gI6g6ZsyZcoe569Hjx5V699///347ne/G8ccc0wcccQRceGFF8b69esb9Jgb+9+zNKVzmrh+Dx5BSZGuvfba7AMwivPFL34x/v3f/z37Y/W///f/jldeeSUuuugip7EWXnjhhdi1a1f8r//1v+LZZ5+NH//4xzF79uy4/vrrnb9aSjf0F198cXz72992zvZh3rx5MXbs2KwitGrVqujTp08MGTIkNmzY4NzVwrZt27Jzlio9FO+xxx7LbgbTB0SLFy+ODz74IM4555zsvLJvf/M3f5N9QLRy5cp48skn4+/+7u/iggsuyP52UJz0IWb6u5uCJ8jL/dDOnTuzkCSVe+KJJ+Kee+7JQpBJkyZVlXn11VezMunePj00loLEf/qnf4rf//73B/EnaRxSKPXmm29Wm9K5Sh889+/fv1rZhx9+uFq5fv36NdhxNzY//OEPq527K6+8smrd1q1bs/uI448/PvvbeMstt2Qf9t11110NesxNrQ7q+q0b9a6DX4cYPXp0tfcLwWlxPvOZz1Q7f48//njVuquvvjp+97vfxfz587PfR3ow+mtf+9oB+g3no+6x+7lN13GS7tsquX4PkgK19tBDDxV69OhRePbZZwvp1D311FPOXh39x3/8R6GkpKSwY8cO57AOpk+fXujWrZtzV6Rf/OIXhdLSUuftYwwYMKDw3e9+t2p+586dhc6dOxemTp3qvBUp/Z347W9/67zthw0bNmTn8bHHHnMe6+joo48u/OxnP3P+ivDOO+8UunfvXli8eHHhzDPPLIwZM8b5Ixf3Q6mu06xZs8K6deuqlt15552Ftm3bFrZv357NX3vttYXPfOYz1bYbPnx4YciQIQfhyBu3VO859thjCz/84Q+rlr366qvqlfvh+OOPL/z4xz/e6/qf/vSn2d/Byus3ue666wqf/vSn9+dlc+ujdVDX7/5R7zq4dQj3dPtn8uTJhT59+tS4bvPmzYVPfOIThfnz51cte/7557PfwbJly/bzlfMp1T9OPPHEwq5du7J51+/Bo0VJLaUmYym9+9WvfpU1Rafu3n777fjNb36TPWX1iU98wqmsgy1btkS7du2cOw6o9ARpetpu8ODBVcuaNWuWzS9btszZpkHe6xLvd8VLT4bfd9992ZN0qQsuai89kZiemN/9vRDyIP2tP+WUU6JDhw5Vy1Kr0vRUfmXLtFTmo/83Uhn3Cfv2wAMPxFtvvRXl5eU1dlOcuos5/fTTs3LUXmpJmbp6+exnP5u1GNm9q7h0XZ5xxhnRsmXLatdr6uXgv/7rv5zmA1QHdf0WT72rYeoQ6XOo9u3bR69evWL8+PHx3nvv1fORNC0vvfRS1sPOCSecEJdeemnWXWeSPkNIrXh2vz9I3XKl7vfdH9Tt/eHXv/51/OM//mPW/VYl1+/B0eIgvU6jlh4M/uY3v5n1X5iaSae+OCneddddF3fccUf2x+jUU0+NBx980Gmsg5dffjl+8pOfxI9+9CPnjwNq06ZN2Yeru39AkqT51PweDqbU3UPq0uVv//Zvs8oMtfPnP/85C0ZSP8Gpf+Df/va30bNnT6evllK4lLodrBw/APJk3bp1Nd4DVK77uDIpTPnv//7vbCwDavbzn/88+5A+dZNYKb1P33rrrdnfuvRwSuqiOPWtn8Y4Sx8+8/G+973vZePlpA9DU3dx6YPP1GXJjBkzqq7X1NXZ3q7po48+2inejzqo67fu1LsOfh3iH/7hH7Ju+NIH/WnsovT5VApN77///no8mqYjjV2auuP89Kc/nb3P3nDDDdn4T88880z2fpoC6Y+Of5bebyvvH6i9dA+wefPm7HPoSq7fgyfXLUrGjRu31wH2Kqf04WC6IXjnnXeyGy+KP3+V0mCQTz31VPznf/5nNG/ePEaOHJmFUHlV7PlL3njjjTj33HOzfgpTC6c8q8v5AxrXU/3pxjt9cE3tpcpLGjdg+fLl2TgEo0aNiueee84prIXXXnstxowZkz2tlQb9hcbA/VDjOOevv/56No5LGhx7d+nJ5jQ2XPoAKg1EnFpHfP3rX89aRuRVMec3nbs0wG0aTyo91JhCp1R33759e0P/GLmog7p+aUx1iMsvvzwLq1PLydQa4t57780eKErj57Jv5513XvYekN5v03l86KGHsg/z01jEHPgHK9L53n18bNfvwZPrFiXf//73qyV0NUlNyh555JGsuVirVq2qrUutS9IbbBroMI9qe/52v5FK06c+9ak4+eSTo2vXrtlgW3ntEqTY85cGw0oDZ6YuywxAWPz5Y9/S/88UYqauBneX5jt27OgUctBcccUVWavDP/zhD9WevGXf0tNcJ510UvZ9Ggw4tYy47bbbssFY+Xip24ANGzZkTydXSq3s0nWYWsSmD97SeyQ01fuh9Ld+xYoV1ZZV3hNU3gekrzXdJ7Rt2zY3rUnqcs5/8YtfZN1D1aaVSApNKgdxzaP9uabTuUtdb6UeINKDA3u7XpO83tvWdx0079dvbal3NXwdIl2rla2lTjzxxHo6oqYrtR5Jn+2l8/elL30p6y4qBSe7tyrxOULx/vKXv8TDDz+8z5ZOrt/6k+ug5Nhjj82mfbn99tvjpptuqnazkBLUefPmVV2ceVTb87e35pBJnp/2Keb8pad40g1q+tArVbRS0/y825/rj71/wJqusSVLlmTdPlT+X03z6aYT6ltqZXjllVdmT3ctXbp0j+4yKF76P5znv7XFOPvss7Ouy3aXxhJIfSyn7hmEJDT1+6H08NK//uu/ZoFhGi8jSR94phCksgu/VCY9Rbq7VCZPDz4Ve87T37Z0/55a09dmfMbUKrBTp06RV/tzTadzl+pJlddvui7/5V/+Jes7v/Lcp+s1hSh57Xarvuugeb9+a0u9q+HrEOlaTVyvdfPuu+9mrXG+8Y1vZO8R6T02fW5w4YUXZutTt2ZpDJM83R8cCOm9Nv0NS+MlfhzXb/3JdVBSW2kAot2lvjiTlDp70nXfUvcf6YnWNDhhuiFNb6YTJ07Mzp83zdrdoKYm5ak/zdQn7MaNG6vW5fVJqGKlP9Bvv/129jU9HVz5RyU9dV35/5mo6sIgddWTWswNGDAgZs6cmQ0GXdPAo9R8w5ieqqn06quvZtdb6jv7o39LqLmp/Ny5c+M//uM/4sgjj6zq07a0tDQ3Tyrvj9RFaGqmna611GVoOpepspi6e2Hf0jX30fFwDj/88OwpcOPkkIf7oXPOOScLRNKHHtOnT8/egydMmJC9N1e2rE/dG6UWVtdee202yGhqeZ+63Vi4cGED/3SHrnSO0v3AP/3TP+2xLvVMkD4wTQORJ+kJ0jlz5sTPfvazBjjSxiX1+JDqmemD/PT+neavvvrqrOuyyhAk9eme+tFPXZ6lwDt1x5NaWf74xz9u6MNvEnVQ1+/+Ue86eHWI9BlUWn/++edn93VpjJL0fnHGGWdkXUmxbz/4wQ/iy1/+cvaekB4enzx5cvYQ0YgRI7LznN5n0zWd6r3pAYsUXKXP+9L4xNT+AbcUlKTPY1q0+OvH9a7fg6xA0V599dU0sEbhqaeecvZq4emnny588YtfLLRr167QqlWrQllZWeFb3/pW4fXXX3f+auEXv/hFdr3VNFE7o0aNqvH8Pfroo05hDX7yk58UPvnJTxZatmxZGDBgQOGPf/yj81RL6Zqq6VpL1yD7trf3uvQ+yL794z/+Y+H444/P/u8ee+yxhbPPPrvwn//5n07dfjjzzDMLY8aMcQ7Jzf1QRUVF4bzzziscdthhhfbt2xe+//3vFz744INq+0nl+/btm73XnHDCCd6j92HEiBGF0047rcZ1v/zlLwsnn3xyoU2bNoW2bdtm913z588/EL/uJm/lypWFgQMHFkpLSwutW7fOzuP//J//s/D+++9XK/f//t//K5x++ulZPbRLly6FadOmNdgxN7U6qOt3/6l3HZw6xJo1awpnnHFG1WdSJ510UuGaa64pbNmy5QAdQdM3fPjwQqdOnbK//em9NM2//PLLVev/+7//u/Cd73yncPTRR2d/07761a8W3nzzzQY95sbm97//fXbdvvjii9WWu34PrpL0z8EOZwAAAAAAAA4FBjoAAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAiLz6/wFdwF3gJtow2QAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Exemple pour discuter\n", "x = torch.randn(1000, 10)\n", "h = torch.randn(10, 200) * 5\n", "#h = torch.randn(10, 200) * 0.2\n", "y = x @ h\n", "print(x.mean(), x.std())\n", "print(y.mean(), y.std())\n", "plt.figure(figsize=(20,5))\n", "plt.subplot(121)\n", "plt.hist(x.view(-1).tolist(), 50, density=True);\n", "plt.subplot(122)\n", "plt.hist(y.view(-1).tolist(), 50, density=True);" ] }, { "cell_type": "markdown", "id": "a3b0e867", "metadata": {}, "source": [ "Empiriquement, pour garder la variance il faut diviser par la racine carrée du \"fan-in\", c'est-à-dire par le nombre d'entrées de chaque neurone:" ] }, { "cell_type": "code", "execution_count": 94, "id": "5a08f37b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor(-0.0049) tensor(0.9964)\n", "tensor(-0.0029) tensor(1.0097)\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABkoAAAGsCAYAAACSD/sZAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAPMNJREFUeJzt3Q+wV2WdP/DPBZZ/KqCS/Iu6gCaSAQVCtP5NFI0UzAodC2Ra3NXF0blbBiUQigOi0l0UZaNQ1Exq19hWXMyYqHVEMMjR8s+oeRcU+WcDKE3QwP3Nc37zvcuVe5Hv5V7un/N6zZyBc77nHJ7vuRf4Pvf9fJ6npLKysjIAAAAAAAByqFVjNwAAAAAAAKCxCEoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQW22iBdi/f39s2rQpjjvuuCgpKWns5gAAQIOrrKyM9957L3r27BmtWhn/xIfTbwIAIE8qi+gztYigJIUkvXv3buxmAADAUbdx48b46Ec/6snzofSbAADIo42H0WdqEUFJqiQpvOFOnTo1dnMAAKDB7dq1KxssVPgsDB9GvwkAgDzZVUSfqUUEJYXptlJIIigBACBPTD1Lsd8r+k0AAORJyWEs12EyYwAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQW20auwEAeVU6ZXlR51fMGd1gbQEAAKBh+nJ1pQ8IcPSoKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5VaegZMGCBVFaWhrt27eP4cOHx9q1aw/rukcffTRKSkpi7Nix1Y5XVlbG9OnTo0ePHtGhQ4cYOXJkvPbaa3VpGgAAAAAAQMMFJUuXLo2ysrKYMWNGrF+/PgYNGhSjRo2KrVu3HvK6ioqK+OY3vxlnnXXWQa/NnTs35s+fHwsXLow1a9bEMccck93zr3/9a7HNAwAAAAAAaLigZN68eTFp0qSYOHFiDBgwIAs3OnbsGIsXL671mn379sVVV10VM2fOjL59+x5UTVJeXh4333xzjBkzJgYOHBgPPvhgbNq0KZYtW1Zs8wAAAAAAABomKNm7d2+sW7cumxqr6gatWmX7q1evrvW6W265JU466aT4xje+cdBrb775ZmzevLnaPTt37pxN6VXbPffs2RO7du2qtgEAAAAAADRoULJ9+/asOqRbt27Vjqf9FHbU5Omnn44f/ehHsWjRohpfL1xXzD1nz56dhSmFrXfv3sW8DQAAAAAAgLov5n643nvvvfj617+ehSRdu3att/tOnTo1du7cWbVt3Lix3u4NAAAAAADkR1FBSQo7WrduHVu2bKl2PO137979oPPfeOONbBH3Sy65JNq0aZNtaf2RX/ziF9nv0+uF6w73nkm7du2iU6dO1TYAAICmYsGCBVFaWhrt27fPphVeu3btYV336KOPRklJSYwdO/agtR2nT58ePXr0iA4dOmRTF7/22msN1HoAAMiXooKStm3bxpAhQ2LlypVVx/bv35/tjxgx4qDz+/fvHy+++GI8//zzVdull14a5513Xvb7NGVWnz59skDkwHumNUfWrFlT4z0BAACasqVLl0ZZWVnMmDEj1q9fH4MGDYpRo0bF1q1bD3ldGmT2zW9+M84666yDXps7d27Mnz8/Fi5cmPWVjjnmmOyef/3rXxvwnQAAQD60KfaC9IF/woQJMXTo0Bg2bFiUl5fH7t27Y+LEidnr48ePj169emXriKTRU6effnq167t06ZL9euDxG2+8MWbNmhWnnHJKFpxMmzYtevbsedAoKoA8K52yvOhrKuaMbpC2AAC1mzdvXkyaNKmqj5TCjeXLl8fixYtjypQpNV6T1oK86qqrYubMmfE///M/sWPHjmrVJKnfdfPNN8eYMWOyY6lSP63ruGzZsrjiiit8OQBaIH1AgCYclIwbNy62bduWlX2nxdYHDx4cK1asqFqMfcOGDdGqVXFLn9x0001Z2HLNNddkHYIzzzwzu2cKWgAAAJqLvXv3xrp167J1FQtS/yhNlbV69epar7vlllvipJNOim984xtZUHKgN998M+t7pXsUdO7cOZvSK92ztqBkz5492XZg5T4AAFAPQUkyefLkbKvJqlWrDnntAw88cNCxNAdv6hikDQAAoLnavn17Vh1SGEhWkPZfeeWVGq95+umn40c/+lE2PXFNUkhSuMcH71l4rSapyj9VqAAAAA0QlADQPCjVBoCm7b333ouvf/3rsWjRoujatWu93jtVtaSpkw+sKEnrRAJw5P0mAFoWQQkAAEA9SWFH69atY8uWLdWOp/3u3bsfdP4bb7yRLeJ+ySWXVB3bv3///++stWkTr776atV16R49evSods80FXJt2rVrl20AAMChFbeYCAAAALVq27ZtDBkyJFauXFkt+Ej7I0aMOOj8/v37x4svvphNu1XYLr300jjvvPOy36cKkD59+mRhyYH3TNUha9asqfGeAABAcVSUAAAA1KM03dWECRNi6NChMWzYsCgvL4/du3fHxIkTs9fHjx8fvXr1ytYQad++fZx++unVru/SpUv264HHb7zxxpg1a1accsopWXAybdq06NmzZ4wdO9bXDgAAjpCgBOADrOsBAByJcePGxbZt22L69OnZYutpeqwVK1ZULca+YcOGaNWquOL+m266KQtbrrnmmtixY0eceeaZ2T1T0AIAABwZQQkAAEA9mzx5crbVZNWqVYe89oEHHjjoWElJSdxyyy3ZBgAA1C9rlAAAAAAAALklKAEAAAAAAHLL1FsAjbSuCQAAAADQ+FSUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMitNo3dAICGVDpluQcMAAAAANRKRQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADIrTaN3QAAmpbSKcuLvqZizugGaQsAAAAANDQVJQAAAAAAQG4JSgAAAAAAgNyqU1CyYMGCKC0tjfbt28fw4cNj7dq1tZ772GOPxdChQ6NLly5xzDHHxODBg+Ohhx6qds7VV18dJSUl1baLLrqoLk0DAAAAAABouDVKli5dGmVlZbFw4cIsJCkvL49Ro0bFq6++GieddNJB559wwgnx3e9+N/r37x9t27aNxx9/PCZOnJidm64rSMHI/fffX7Xfrl27YpsGAAAAAADQsEHJvHnzYtKkSVnYkaTAZPny5bF48eKYMmXKQeefe+651fZvuOGGWLJkSTz99NPVgpIUjHTv3r3Y5gAAAAAAEVE6ZXnRz6FizmjPDsi9oqbe2rt3b6xbty5GjhxZdaxVq1bZ/urVqz/0+srKyli5cmVWfXL22WdXe23VqlVZlcmpp54a1157bbz77ru13mfPnj2xa9euahsAAAAAAECDVpRs37499u3bF926dat2PO2/8sortV63c+fO6NWrVxZwtG7dOu6999644IILqk279aUvfSn69OkTb7zxRnznO9+Jiy++OAtf0vkfNHv27Jg5c2YxTQcAAAAAAKifxdyLddxxx8Xzzz8fzz33XNx2223ZGiepgqTgiiuuiEsvvTQ+9alPxdixY7N1TNK5B55zoKlTp2bhS2HbuHHj0XgbAAAAh2XBggVRWloa7du3z9Z2XLt2ba3nPvbYYzF06NDo0qVLHHPMMTF48OB46KGHqp1z9dVXR0lJSbUtDTgDAACOckVJ165dswqPLVu2VDue9g+1vkianuvkk0/Ofp8+9L/88stZVcgH1y8p6Nu3b/Znvf7663H++ecf9Hpaz8Ri7wAAQFO0dOnSbHBYWs8xhSTl5eXZ+oxpCuI03fAHnXDCCfHd7343+vfvH23bts0GjqU1IdO5B67rmIKR+++/v2pfnwgAABqhoiR9aB8yZEi2zkjB/v37s/0RI0Yc9n3SNWkartq89dZb2RolPXr0KKZ5AAAAjW7evHkxadKkLOwYMGBAFph07NgxFi9eXOP5aQDZZZddFqeddlr069cvbrjhhhg4cGA8/fTT1c5LwUgaoFbYjj/++KP0jgAAoGUreuqtNDJq0aJFsWTJkqwyJC28vnv37qwTkIwfPz6bGqsgVY489dRT8ac//Sk7/6677srKyL/2ta9lr7///vvxrW99K5599tmoqKjIQpcxY8ZkFSgHjp4CAABo6vbu3Rvr1q2LkSNHVquwT/tpDcYPU1lZmfWJUvXJ2WefXe21NDVxqjI59dRTs35YGlx2KGlw2q5du6ptAADAEU69lYwbNy62bdsW06dPj82bN2dTaa1YsaJqgfcNGzZkHYGCFKJcd911WZVIhw4dsnLyhx9+OLtPkqbyeuGFF7LgZceOHdGzZ8+48MIL49Zbb1VKDgAANCvbt2+Pffv2VfWPCtL+K6+8Uut1ae3FXr16ZeFG6iPde++9ccEFF1SbdutLX/pS9OnTJ9544434zne+ExdffHEWvqTza5IGrc2cObMe3x1A01c6ZXljNwGAZqikMg1ZaubSyKjOnTtnnYtOnTo1dnOAJsSH5KOjYs7oo/QnAVDgM3DTtGnTpizweOaZZ6pNT3zTTTfFb37zm1izZk2t0xOnKvxUcZ8qStLAsWXLltW6rmM6N03T9atf/arGdR2TFLocOOVx+p7p3bu3fhPQoukDFk9/DmipiukzFV1RAgAAQM26du2aVXhs2bKl2vG0n9YVqU2qyk/TDyepaj9NW5wqQmoLSvr27Zv9Wa+//nqtQUla08SC7wAA0ABrlAAAAFCztm3bxpAhQ7KqkAOrRdL+gRUmHyZdc2A1yAelqY3TGiU9evTwpQAAgCOkogQAAKAelZWVxYQJE2Lo0KExbNiwKC8vz9ZunDhxYvb6+PHjs+m5UsVIkn5N56aptFI48sQTT8RDDz0U9913X/Z6mo4rrTVy+eWXZ1UpaY2SNJVXqkAZNWqUrx0AABwhQQkAAEA9GjduXGzbti2mT58emzdvzqbSWrFiRdUC7xs2bMim2ipIIcp1112XVYl06NAh+vfvHw8//HB2nyRN5fXCCy/EkiVLYseOHdGzZ8+48MILs3VMTK0FAABHzmLuQItmIb+jw+J/AEefxdzxPQNwMH3A4unPAS1VMX0ma5QAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADILUEJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAORWm8ZuAADNX+mU5UVfUzFndIO0BQAAAACKoaIEAAAAAADILRUlQIuuWgAAAAAAOBQVJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAORWm8ZuAAD5VDpleVHnV8wZ3WBtAQAAACC/6lRRsmDBgigtLY327dvH8OHDY+3atbWe+9hjj8XQoUOjS5cuccwxx8TgwYPjoYceqnZOZWVlTJ8+PXr06BEdOnSIkSNHxmuvvVaXpgEAAAAAADRcULJ06dIoKyuLGTNmxPr162PQoEExatSo2Lp1a43nn3DCCfHd7343Vq9eHS+88EJMnDgx25588smqc+bOnRvz58+PhQsXxpo1a7JAJd3zr3/9a7HNAwAAAAAAaLigZN68eTFp0qQs7BgwYEAWbnTs2DEWL15c4/nnnntuXHbZZXHaaadFv3794oYbboiBAwfG008/XVVNUl5eHjfffHOMGTMme+3BBx+MTZs2xbJly4ptHgAAAAAAQMMEJXv37o1169ZlU2NV3aBVq2w/VYx8mBSKrFy5Ml599dU4++yzs2NvvvlmbN68udo9O3funE3pVds99+zZE7t27aq2AQAAAAAANGhQsn379ti3b19069at2vG0n8KO2uzcuTOOPfbYaNu2bYwePTruvvvuuOCCC7LXCtcVc8/Zs2dnYUph6927dzFvAwAAAAAAoO6LuRfruOOOi+effz6ee+65uO2227I1TlatWlXn+02dOjULXwrbxo0b67W9AAAAR2LBggVRWloa7du3z6rl165dW+u5jz32WAwdOjS6dOmSrdc4ePDgeOihhw6qzp8+fXr06NEjOnTokFXkv/baa75IAABQD9oUc3LXrl2jdevWsWXLlmrH03737t1rvS5Nz3XyySdnv08f+l9++eWsKiStX1K4Lt0jfeg/8J7p3Jq0a9cu2wAAAJqapUuXZoPD0nqOKSRJazKOGjUqm4L4pJNOOuj8E044Ib773e9G//79syr8xx9/PFsTMp2brkvmzp0b8+fPjyVLlkSfPn1i2rRp2WsvvfRSFsYAQF2VTlle1PkVc0Z72EC+K0rSh/YhQ4Zk64wU7N+/P9sfMWLEYd8nXZPWGUnSh/wUlhx4z7TmyJo1a4q6JwAAQFMwb968mDRpUhZ2DBgwIAtMOnbsGIsXL67x/DSA7LLLLovTTjst+vXrFzfccEMMHDgwnn766apqkhS23HzzzTFmzJjstQcffDA2bdoUy5YtO8rvDgAAWp6ip95KI6MWLVqUjWRKlSHXXntt7N69O+sEJOPHj8+mxipIlSNPPfVU/OlPf8rOv+uuu7Iy8q997WvZ6yUlJXHjjTfGrFmz4he/+EW8+OKL2T169uwZY8eOrc/3CgAA0KD27t0b69aty6bGOrDCPu2vXr36Q69PoUgaRJaqT84+++zs2Jtvvpmt33jgPdNajala5VD3TIPT0iC0AzcAAOAIp95Kxo0bF9u2bcvmx00f1tP0WCtWrKhajH3Dhg1ZR6AghSjXXXddvPXWW9lcuqmc/OGHH87uU3DTTTdl511zzTWxY8eOOPPMM7N7KiEHAACak+3bt8e+ffuq+kcFaf+VV16p9bq09mKvXr2ycCNNd3zvvffGBRdckL2W+l2Fe3zwnoXXapIGrc2cOfMI3xEAALR8RQclyeTJk7OtJh9cpD1ViqTtUFJVyS233JJtAFAf8+Ym5s4FoLk47rjj4vnnn4/3338/qyhJlfx9+/bNpuWqq1Tpn+5TkCpKevfuXU8tBgCAnAclAAAAHKxr165ZRciWLVuqHU/7aW3G2qSq/JNPPjn7faraT9MWp4qQFJQUrkv36NGjR7V7pnNr065du2wDAADqeY0SAAAAata2bdsYMmRIVhVSsH///mx/xIgRh/3Y0jVpGq6kT58+WVhy4D1TdciaNWuKuicAAFAzFSUAAAD1KE13NWHChBg6dGgMGzYsysvLszUZJ06cmL0+fvz4bD2SVDGSpF/Tuf369cvCkSeeeCIeeuihuO+++6qmKr7xxhuzKY1POeWULDiZNm1a9OzZM8aOHetrBwAAR0hQAgAAUI/GjRsX27Zti+nTp2eLrafpsVasWFG1GPuGDRuyqbYKUohy3XXXxVtvvRUdOnSI/v37x8MPP5zdp+Cmm27Kzrvmmmtix44dceaZZ2b3bN++va8d0GLVZZ1CAKiLksrKyspo5lLZeefOnWPnzp3RqVOnxm4O0EB8SKZYFnMHWjKfgfE9A7R0+oBNk34W0BL7TNYoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC51aaxGwDkU+mU5Y3dBAAAAAAAFSUAAAAAAEB+mXoLAAAAAADILUEJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFt1CkoWLFgQpaWl0b59+xg+fHisXbu21nMXLVoUZ511Vhx//PHZNnLkyIPOv/rqq6OkpKTadtFFF9WlaQAAAAAAAA0XlCxdujTKyspixowZsX79+hg0aFCMGjUqtm7dWuP5q1atiiuvvDJ+/etfx+rVq6N3795x4YUXxttvv13tvBSMvPPOO1XbT37yk2KbBgAAAAAA0LBBybx582LSpEkxceLEGDBgQCxcuDA6duwYixcvrvH8H//4x3HdddfF4MGDo3///vHDH/4w9u/fHytXrqx2Xrt27aJ79+5VW6o+AQAAAAAAaEhtijl57969sW7dupg6dWrVsVatWmXTaaVqkcPxl7/8Jf72t7/FCSeccFDlyUknnZQFJJ///Odj1qxZceKJJ9Z4jz179mRbwa5du4p5GwDkROmU5UVfUzFndIO0BQAAAIAWUFGyffv22LdvX3Tr1q3a8bS/efPmw7rHt7/97ejZs2cWrhw47daDDz6YVZncfvvt8Zvf/CYuvvji7M+qyezZs6Nz585VW5rOCwAAoKmwriMAALTwxdzras6cOfHoo4/Gz3/+82wh+IIrrrgiLr300vjUpz4VY8eOjccffzyee+65rMqkJqmiZefOnVXbxo0bj+K7AAAAqJ11HQEAoAUHJV27do3WrVvHli1bqh1P+2ldkUO58847s6Dkl7/8ZQwcOPCQ5/bt2zf7s15//fUaX0/rmXTq1KnaBgAA0BRY1xEAAFpwUNK2bdsYMmRItYXYCwuzjxgxotbr5s6dG7feemusWLEihg4d+qF/zltvvRXvvvtu9OjRo5jmAQAANKrCuo4HTjVc3+s6nnrqqXHttddmfaZDSes6pvUcD9wAAIB6mHqrrKwsFi1aFEuWLImXX345+4C+e/fumDhxYvb6+PHjqy32ntYcmTZtWixevDhKS0uztUzS9v7772evp1+/9a1vxbPPPhsVFRVZ6DJmzJg4+eSTY9SoUcU2DwAAoNE0lXUdE2s7AgDA4WkTRRo3blxs27Ytpk+fnn3QHzx4cFYpUugIbNiwIRsxVXDfffdlo6q+/OUvV7vPjBkz4nvf+142ldcLL7yQBS87duzIOgQXXnhhVoGSptgCAADIi8K6jql65IPrOhaktR3TdMb9+vXLzjv//PNrvFcawJYGuhWkipLevXs38DsAAIAcBCXJ5MmTs60mH1yAPVWJHEqHDh3iySefrEszAAAAmpT6WNfxV7/6VVHrOtYWlKSBZwafAQBAA0y9BQAAQM2s6wgAAM2PoAQAAKAeWdcRAAByMPUWAAAANbOuIwAtWemU5UVfUzFndIO0BaC+CEoAAADqmXUdAQCg+TD1FgAAAAAAkFsqSgAAAABoctM1AcDRoqIEAAAAAADILUEJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADkVpvGbgDQMpROWd7YTQAAAAAAKJqKEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNyqU1CyYMGCKC0tjfbt28fw4cNj7dq1tZ67aNGiOOuss+L444/PtpEjRx50fmVlZUyfPj169OgRHTp0yM557bXX6tI0AAAAAACAhgtKli5dGmVlZTFjxoxYv359DBo0KEaNGhVbt26t8fxVq1bFlVdeGb/+9a9j9erV0bt377jwwgvj7bffrjpn7ty5MX/+/Fi4cGGsWbMmjjnmmOyef/3rX4ttHgAAAAAAQMMFJfPmzYtJkybFxIkTY8CAAVm40bFjx1i8eHGN5//4xz+O6667LgYPHhz9+/ePH/7wh7F///5YuXJlVTVJeXl53HzzzTFmzJgYOHBgPPjgg7Fp06ZYtmxZjffcs2dP7Nq1q9oGAAAAAADQoEHJ3r17Y926ddnUWFU3aNUq20/VIofjL3/5S/ztb3+LE044Idt/8803Y/PmzdXu2blz52xKr9ruOXv27OycwpaqVAAAAJoK0xUDAEALDUq2b98e+/bti27dulU7nvZT2HE4vv3tb0fPnj2rgpHCdcXcc+rUqbFz586qbePGjcW8DQAAgAZjumIAAMjBYu51NWfOnHj00Ufj5z//ebYQfF21a9cuOnXqVG0DAABoCprCdMWJKYsBAKABgpKuXbtG69atY8uWLdWOp/3u3bsf8to777wzC0p++ctfZh/sCwrX1eWeAAAATUlTma44MWUxAAA0QFDStm3bGDJkSNXIpqQw0mnEiBG1Xjd37ty49dZbY8WKFTF06NBqr/Xp0ycLRA68Z1qcfc2aNYe8JwAAQFPTVKYrTkxZDAAAh6dNFKmsrCwmTJiQBR7Dhg3LSsB3796dlZUn48ePj169emWjl5Lbb789pk+fHo888kiUlpZWfZA/9thjs62kpCRuvPHGmDVrVpxyyilZcDJt2rSsYzB27NhimwcAANBsFaYrXrVq1RFNV1yYsjhtAABAPQcl48aNi23btmXhRwo90jy6qVKkMLppw4YNWWl5wX333ZeVn3/5y1+udp8ZM2bE9773vez3N910Uxa2XHPNNbFjx44488wzs3seaccAAADgaKqP6Yp/9atf1TpdcY8ePardM/XHAACAoxyUJJMnT862mqSRTweqqKj40PulqpJbbrkl2wCgMZVOWV70NRVzRjdIWwBofg6crrhQIV+Yrri2PlRhuuLbbrstnnzyyUNOV1wIRgrTFV977bUN/I4AAKDlq1NQAgAAQM1MVwwAAM2LoAQAAKAema4YAACal5LKysrKaOZS2Xnnzp1j586d0alTp8ZuDuRSXaYrgpbC1FtAY/AZGN8zQHOiz5hv+kxAU+8zqSgBDuIDLAAAAACQF60auwEAAAAAAACNRVACAAAAAADklqAEAAAAAADILWuUAAAAAHDYrGvJ0fiesQA8cDSpKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEButWnsBgBAc1c6ZXnR11TMGd0gbQEAAACgOIISaOHq8gNcAAAAAIC8MPUWAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbdQpKFixYEKWlpdG+ffsYPnx4rF27ttZz//jHP8bll1+enV9SUhLl5eUHnfO9730ve+3ArX///nVpGgAAAAAAQMMFJUuXLo2ysrKYMWNGrF+/PgYNGhSjRo2KrVu31nj+X/7yl+jbt2/MmTMnunfvXut9P/nJT8Y777xTtT399NPFNg0AAAAAAKBhg5J58+bFpEmTYuLEiTFgwIBYuHBhdOzYMRYvXlzj+WeccUbccccdccUVV0S7du1qvW+bNm2yIKWwde3atdimAQAANAmq8AEAoIUGJXv37o1169bFyJEj/+8GrVpl+6tXrz6ihrz22mvRs2fPrPrkqquuig0bNtR67p49e2LXrl3VNgAAgKZAFT4AALTgoGT79u2xb9++6NatW7XjaX/z5s11bkRa5+SBBx6IFStWxH333RdvvvlmnHXWWfHee+/VeP7s2bOjc+fOVVvv3r3r/GcDAADUJ1X4AACQg8Xc69vFF18cX/nKV2LgwIHZeidPPPFE7NixI37605/WeP7UqVNj586dVdvGjRuPepsBAACaahV+ohIfAAAaIChJ64a0bt06tmzZUu142j/UQu3F6tKlS3ziE5+I119/vcbX01onnTp1qrYBAAA0tqZShZ+oxAcAgAYIStq2bRtDhgyJlStXVh3bv39/tj9ixIioL++//3688cYb0aNHj3q7JwAAQHNVbBV+ohIfAAAOT5soUllZWUyYMCGGDh0aw4YNi/Ly8ti9e3dMnDgxe338+PHRq1evbPRSofT8pZdeqvr922+/Hc8//3wce+yxcfLJJ2fHv/nNb8Yll1wSH//4x2PTpk0xY8aMrHLlyiuvLLZ5AAAAjaapVOEXKvHTBgAA1PMaJePGjYs777wzpk+fHoMHD85Cj1T+XSgtT/PkvvPOO1Xnp+Dj05/+dLal4+na9Pt/+Id/qDrnrbfeykKRU089Nb761a/GiSeeGM8++2x85CMfKbZ5AAAAjUYVPgAA5KCiJJk8eXK21WTVqlXV9ktLS6OysvKQ93v00Ufr0gwAAIAmRxU+ABy50inLi76mYs5ojx44ekEJAAAAtVfhb9u2LavCTwu4p0r8D1bht2rV6qAq/IJUhZ+2c845p2ogWqEK/913380q788880xV+AAAUE8EJQAAAPVMFT4AALTgNUoAAAAAAABaChUlAAAAADlWl7UgAKAlUVECAAAAAADklqAEAAAAAADILUEJAAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcqtNYzcAAPKodMryos6vmDO6wdoCAAAAkGcqSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHLLGiXQgtc0AAAAAADg0FSUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG4JSgAAAAAAgNwSlAAAAAAAALklKAEAAAAAAHJLUAIAAAAAAOSWoAQAAAAAAMgtQQkAAAAAAJBbghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcqlNQsmDBgigtLY327dvH8OHDY+3atbWe+8c//jEuv/zy7PySkpIoLy8/4nsCAAAAAAA0SlCydOnSKCsrixkzZsT69etj0KBBMWrUqNi6dWuN5//lL3+Jvn37xpw5c6J79+71ck8AAAAAAID6UFJZWVlZzAWp2uOMM86Ie+65J9vfv39/9O7dO66//vqYMmXKIa9NFSM33nhjttXXPZNdu3ZF586dY+fOndGpU6di3g40K6VTljd2E4BGUjFntGcPVOMzcNOWKubvuOOO2Lx5czYQ7O67745hw4bVWoU/ffr0WLduXfzv//5vfP/73z+oz1TsPWviewaojb4meaavBS1XMZ9/i6oo2bt3b/bhfeTIkf93g1atsv3Vq1fXqbF1ueeePXuyN3ngBgAA0BSowgcAgOalTTEnb9++Pfbt2xfdunWrdjztv/LKK3VqQF3uOXv27Jg5c2ad/jwAAICGNG/evJg0aVJMnDgx21+4cGEsX748Fi9eXGPFfKquT1tSW0V9sfcsDDBLW4EBZpAPqkMA4Cgt5t7Ypk6dmpXLFLaNGzc2dpMAAACaTBV+YYBZmmqgsKXpjQEAgCMMSrp27RqtW7eOLVu2VDue9mtbqL0h7tmuXbtsTrEDNwAAgMZ2qIr5tLbI0bynAWYAANAAQUnbtm1jyJAhsXLlyqpjaeH1tD9ixIhibtWg9wQAAMg7A8wAAKAB1ihJysrKYsKECTF06NAYNmxYlJeXx+7du6vmyh0/fnz06tUrK/MulIm/9NJLVb9/++234/nnn49jjz02Tj755MO6JwAAQHPQVKrwAQCABgxKxo0bF9u2bYvp06dnZd6DBw+OFStWVJWBb9iwIZsvt2DTpk3x6U9/umr/zjvvzLZzzjknVq1adVj3BIC8q8uinBVzRjdIWwA4vIr5sWPHVquYnzx5cpO5JwAAcARBSZI+jNf2gbwQfhSUlpZGZWXlEd0TAACguVCFDwAAOQhKgMYZHQ4AQNOnCh8AAJoXQQkAAEA9U4UPAADNx/8tJgIAAAAAAJAzghIAAAAAACC3BCUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC51aaxGwAANIzSKcuLvqZizugGaQsAAABAU6WiBAAAAAAAyC0VJdCII7cBAAAAAGhcKkoAAAAAAIDcEpQAAAAAAAC5JSgBAAAAAAByS1ACAAAAAADklqAEAAAAAADIrTaN3QAAAAAAgMZQOmV5UedXzBndYG0BGo+KEgAAAAAAILcEJQAAAAAAQG6ZegsAAACgBUwJBADUjYoSAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQG61aewGAABNR+mU5UVfUzFndIO0BQAAAOBoUFECAAAAAADklqAEAAAAAADILUEJAAAAAACQW3UKShYsWBClpaXRvn37GD58eKxdu/aQ5//sZz+L/v37Z+d/6lOfiieeeKLa61dffXWUlJRU2y666KK6NA0AAAAAAKDhFnNfunRplJWVxcKFC7OQpLy8PEaNGhWvvvpqnHTSSQed/8wzz8SVV14Zs2fPji9+8YvxyCOPxNixY2P9+vVx+umnV52XgpH777+/ar9du3bFNg0adTFjAAA4cHDZHXfcEZs3b45BgwbF3XffHcOGDTvk4LJp06ZFRUVFnHLKKXH77bfHF77whWqDy5YsWVLtmtQPW7FihYcOAABHu6Jk3rx5MWnSpJg4cWIMGDAgC0w6duwYixcvrvH8f/3Xf81CkG9961tx2mmnxa233hqf+cxn4p577ql2XgpGunfvXrUdf/zxdX9XAAAAjaQwuGzGjBnZALEUlKRQY+vWrTWeXxhc9o1vfCN+//vfZwPL0vaHP/yh2nmpX/XOO+9UbT/5yU+O0jsCAICWraiKkr1798a6deti6tSpVcdatWoVI0eOjNWrV9d4TTqeOgkHSp2EZcuWVTu2atWqrCIlBSSf//znY9asWXHiiSfWeM89e/ZkW8GuXbuKeRsAAAAN5sDBZUkaXLZ8+fJscNmUKVMOObgsSYPLnnrqqWxwWbr2g4PLAIDmNQtJxZzRDdIWoJGCku3bt8e+ffuiW7du1Y6n/VdeeaXGa1KpeU3np+MFqVPwpS99Kfr06RNvvPFGfOc734mLL744C1lat2590D3TNF4zZ84spukAAAANrqkMLksMMIOmxRTPANB0Fb1GSUO44oorqn6fFnsfOHBg9OvXL+sInH/++QednzodB3YkUkVJ7969j1p7AQAAmvLgssQAMwAAaICgpGvXrtmH8C1btlQ7nvZrKwFPx4s5P+nbt2/2Z73++us1BiWp5Nxi7wDQNCg9B2h6g8sSA8wAAKABFnNv27ZtDBkyJFauXFl1bP/+/dn+iBEjarwmHT/w/CTNt1vb+clbb70V7777bvTo0aOY5gEAADSqxhhcVps0uKxTp07VNgAA4AiDkiRNebVo0aJYsmRJvPzyy3HttdfG7t27qxYqHD9+fLX5eG+44YZYsWJF3HXXXVmp+fe+97343e9+F5MnT85ef//997NFC5999tmoqKjIQpUxY8bEySefnM3LCwAA0FwYXAYAADlYo2TcuHGxbdu2mD59ejZn7uDBg7MgpDCn7oYNG7LFCgs+97nPxSOPPBI333xzNo/uKaecki1KePrpp2evp9FWL7zwQha87NixI3r27BkXXnhh3HrrrabXAgAAmp00uGzChAkxdOjQGDZsWJSXlx80uKxXr17ZGiKFwWXnnHNONrhs9OjR8eijj2aDy37wgx9UDS6bOXNmXH755VmVSVqj5KabbjK4DAAAGnMx91QNUqgI+aA0R+4HfeUrX8m2mnTo0CGefPLJujQDAACgyTG4DAAAchCUAAAAUDuDywAAoAWvUQIAAAAAANBSCEoAAAAAAIDcMvUWLV7plOWN3QQAAAAAAJooFSUAAAAAAEBuCUoAAAAAAIDcEpQAAAAAAAC5ZY0SAAAAAIAmtH5uxZzRDdIWoGaCEgAAAIAG/qEnANB0CUoAgCb/wwWjqQAAAICGYo0SAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcqtNYzcAilE6ZbkHBgAAAABAvRGUAAAAAAA088HCFXNGN0hbIA8EJQAAAEBumbkAALBGCQAAAAAAkFsqSgCAJk/ZOQAAANBQVJQAAAAAAAC5JSgBAAAAAAByy9RbNBoL5gEAAABA4/2srWLOaI8fVJQAAAAAAAB5pqIEAAAAaDHMXgAAFEtQQo2U6gEAAAAAkAcWcwcAAAAAAHJLUAIAAAAAAOSWqbeoN+aBBaApMY0kAAAAcDgEJQAAAAAAOWSAGfx/ghIAAACgSTJzAQBwNAhKAAAAAABokBC7Ys5oT5YmT1CSA0bgAEDT+j9TRwGAPNI3BQCaKkFJM+ODJQA0f+YBBgAAgGYelCxYsCDuuOOO2Lx5cwwaNCjuvvvuGDZsWK3n/+xnP4tp06ZFRUVFnHLKKXH77bfHF77wharXKysrY8aMGbFo0aLYsWNH/P3f/33cd9992bkAABRPGAONS5+Jls4gPgAa8v8MVfg0+aBk6dKlUVZWFgsXLozhw4dHeXl5jBo1Kl599dU46aSTDjr/mWeeiSuvvDJmz54dX/ziF+ORRx6JsWPHxvr16+P000/Pzpk7d27Mnz8/lixZEn369MlClXTPl156Kdq3bx8tlQ+WAADQ8ugz0dzomwLQ1AhXONpKKlM5RxFSOHLGGWfEPffck+3v378/evfuHddff31MmTLloPPHjRsXu3fvjscff7zq2Gc/+9kYPHhwFrakP75nz57xL//yL/HNb34ze33nzp3RrVu3eOCBB+KKK6446J579uzJtoJ0/sc+9rHYuHFjdOrUKRrD6TOebJQ/FwCgvvxh5igPsxnZtWtX9jk8VWR37ty5sZtDE+szNdV+E0eH/ikANBz9ppbZZyqqomTv3r2xbt26mDp1atWxVq1axciRI2P16tU1XpOOpwqUA6VqkWXLlmW/f/PNN7MpvNI9ClKjU+ciXVvTh/5UnTJz5syDjqc3DQBA3XQu9+Sao/fee09Q0oQ0lT5Tot8EAFD/9JtaZp+pqKBk+/btsW/fvmzk0oHS/iuvvFLjNekDfU3np+OF1wvHajvng1Kn48CORBqh9ec//zlOPPHEKCkpKeYt5So5M3LMM/R92Pz5++wZNhW+Fz3DpiDv34epyiB94E+VBjQdTaXPlOg3tQx5/7cuD3yN88HXOR98nfPB17ll9pnqtJh7Y2vXrl22HahLly6N1p7mIn2o9sHaM2xsvg89x6bC96Ln2FT4XvQMj4QptzgU/aaWxf8XLZ+vcT74OueDr3M++Dq3rD5Tq2Ju2rVr12jdunVs2bKl2vG037179xqvSccPdX7h12LuCQAA0BTpMwEAQPNTVFDStm3bGDJkSKxcubLatFdpf8SIETVek44feH7y1FNPVZ3fp0+fLBA58JxUvrRmzZpa7wkAANAU6TMBAEDzU/TUW2ltkAkTJsTQoUNj2LBhUV5eHrt3746JEydmr48fPz569eqVLRyY3HDDDXHOOefEXXfdFaNHj45HH300fve738UPfvCD7PW0psiNN94Ys2bNilNOOSULTqZNm5bNGzZ27Nj6fr+5LbmfMWPGQdOV4Rn6Pmx+/H32DJsK34ueYVPg+5CmSp+J+uTfupbP1zgffJ3zwdc5H3ydW6aSyrSiSZHuueeeuOOOO7KFAwcPHhzz58+P4cOHZ6+de+65UVpaGg888EDV+T/72c/i5ptvjoqKiiwMmTt3bnzhC1+oej01If0gP4UnO3bsiDPPPDPuvffe+MQnPlFf7xMAAOCo0WcCAIAWHpQAAAAAAADkbo0SAAAAAACAlkRQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCkpy59NJL42Mf+1i0b98+evToEV//+tdj06ZNjd2sZqOioiK+8Y1vRJ8+faJDhw7Rr1+/mDFjRuzdu7exm9bs3HbbbfG5z30uOnbsGF26dGns5jQLCxYsiNLS0uzv7/Dhw2Pt2rWN3aRm5be//W1ccskl0bNnzygpKYlly5Y1dpOandmzZ8cZZ5wRxx13XJx00kkxduzYePXVVxu7Wc3OfffdFwMHDoxOnTpl24gRI+K///u/G7tZzdqcOXOyv9c33nhjYzcF4KjYs2dPDB48OPu37/nnn/fUWxB9zpZLf67l0k/KJ32QlkdQkjPnnXde/PSnP81+sPUf//Ef8cYbb8SXv/zlxm5Ws/HKK6/E/v3749/+7d/ij3/8Y3z/+9+PhQsXxne+853Gblqzk8Klr3zlK3Httdc2dlOahaVLl0ZZWVkWzK1fvz4GDRoUo0aNiq1btzZ205qN3bt3Z88tdVCom9/85jfxz//8z/Hss8/GU089FX/729/iwgsvzJ4th++jH/1o9qF63bp18bvf/S4+//nPx5gxY7L/Vyjec889l/2/nMIngLy46aabssEftDz6nC2T/lzLpp+UP/ogLVNJZWVlZWM3gsbzi1/8IhsRnEYk/d3f/Z0vRR3ccccd2ejgP/3pT55fHTzwwAPZCOAdO3Z4foeQKkjSSP577rkn20+BXe/eveP666+PKVOmeHZFSqMvf/7zn2f//lF327ZtyypLUsfg7LPP9iiPwAknnJD9f5KqFjl877//fnzmM5+Je++9N2bNmpWNri4vL/cIgRYtVSGmATRp4NsnP/nJ+P3vf5/9+0fLpc/Z/OnP5Yt+UsumD9JyqSjJsT//+c/x4x//OJv+SEhSdzt37sx+wAUNWX2TRp6PHDmy6lirVq2y/dWrV3vwNOq/f4l/A+tu37598eijj2ZVOWkKLoqTKpxGjx5d7d9HgJZsy5YtMWnSpHjooYeyKWzJB33O5k1/Ln/0k1o2fZCWS1CSQ9/+9rfjmGOOiRNPPDE2bNgQ//mf/9nYTWq2Xn/99bj77rvjH//xHxu7KbRg27dvz36Y2q1bt2rH0/7mzZsbrV3kW6pqStVgf//3fx+nn356Yzen2XnxxRfj2GOPjXbt2sU//dM/ZRVOAwYMaOxmNSspYEpTEaY5oQHyIE0GcfXVV2f/bwwdOrSxm8NRos/Z/OnP5Yt+UsumD9KyCUpagDTtTppG5lBbmue04Fvf+lZWnv3LX/4yWrduHePHj88+dOdZsc8wefvtt+Oiiy7K1tlIo7qo23MEmu8omj/84Q/ZB0WKd+qpp2aL765ZsyZbq2nChAnx0ksveZSHaePGjXHDDTdklbHt27f33IBcfIZOA7Tee++9mDp1amM3mTrQ54R80E9qufRBWj5rlLSQuQ/ffffdQ57Tt2/faNu27UHH33rrrWydg2eeeSbXU34U+ww3bdoU5557bnz2s5/N1thI0yBRt+9Fa5QcXql2mlrh3//936utqZF+sJrWdlEVVjxrlByZyZMnZ993v/3tb6NPnz5HeDeSNHVUv379skXJ+XDLli2Lyy67LBvwUZAq79Lf7fR/clp77cDXAFrCZ+ivfvWr8V//9V/Zv3UH/tuX/r276qqrYsmSJUehtdSVPmd+6c/lh35Sy6YP0vK1aewGcOQ+8pGPZFtdSwKT9AOFPCvmGaZKkvPOOy+GDBkS999/v5Ckjs+Rw5eCpfT9tnLlyqqgJP3dTfvpgxgcLan68Prrr8+miVq1apWQpB6lv9N5/7+4GOeff342fdmBJk6cGP3798+mGBWSAC3xM/T8+fNj1qxZVftp8NaoUaNi6dKl2ULRNG36nPmlP9fy6Sflgz5IyycoyZE0vcdzzz0XZ555Zhx//PHxxhtvxLRp07IRrHmuJilGCklSJcnHP/7xuPPOO7NRQQXdu3dv1LY1N2l9nD//+c/Zr2kkXJqCJjn55JOzefuprqysLKsgSfNRDxs2LMrLy7PFn9MPBjk877//fjbHc8Gbb76Zfd+lhcg/9rGPeYyHWUb+yCOPZNUkxx13XNUaOZ07d44OHTp4hocpTZly8cUXZ993aQqV9ExT8PTkk096hocpff99cG2cwvpr1swBWqoPfl4pfGZO/bmPfvSjjdQq6ps+Z8ukP9ey6Sflgz5IyycoyZE0dc9jjz0WM2bMyH7A2qNHj2yNjZtvvjlbTJYP99RTT2U/aE3bBzsjeV/npVjTp0+vNj3Apz/96ezXX//611kYRXXjxo3Lgrn03NIPpwcPHhwrVqw4aIF3ave73/0uqwY7sLOSpAAqTQHHh7vvvvuyXz/4dzRV16XFZTk8W7duzdYHe+edd7KQaeDAgVlIcsEFF3iEAJBz+pwtk/5cy6afBC2DNUoAAAAAAIDcsgI1AAAAAACQW4ISAAAAAAAgtwQlAAAAAABAbglKAAAAAACA3BKUAAAAAAAAuSUoAQAAAAAAcktQAgAAAAAA5JagBAAAAAAAyC1BCQAAAAAAkFuCEgAAAAAAILcEJQAAAAAAQOTV/wM+BA96vJydzgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Exemple pour discuter\n", "fanin = 10\n", "nb_exemples = 1000\n", "n_hidden = 200\n", "x = torch.randn(nb_exemples, fanin)\n", "h = torch.randn(fanin, n_hidden) / fanin**0.5 \n", "y = x @ h\n", "print(x.mean(), x.std())\n", "print(y.mean(), y.std())\n", "plt.figure(figsize=(20,5))\n", "plt.subplot(121)\n", "plt.hist(x.view(-1).tolist(), 50, density=True);\n", "plt.subplot(122)\n", "plt.hist(y.view(-1).tolist(), 50, density=True);" ] }, { "cell_type": "markdown", "id": "17707b1f", "metadata": {}, "source": [ "Le papier de [Kaiming _et al_](" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, max_steps, mini_batch_size)\n", "train_loss = nn.training_loss(datasets)\n", "val_loss = nn.test_loss(datasets)\n", "print(f\"{train_loss=}\")\n", "print(f\"{val_loss=}\")\n", "plt.plot(lossi);" ] }, { "cell_type": "markdown", "id": "c9a3e4e3", "metadata": {}, "source": [ "Le score est similaire à ce que nous avions fait \"à la main\", mais cette version permet d'avoir une méthode robuste pour ne pas avoir à trouver nous même le facteur d'initialisation.\n", "\n", "Des méthodes plus modernes et robustes existent néanmoins, comme la méthode BatchNorm." ] }, { "cell_type": "markdown", "id": "d90f6653", "metadata": {}, "source": [ "## Méthode BatchNorm" ] }, { "cell_type": "markdown", "id": "f870d3d3", "metadata": {}, "source": [ "Méthode apparue en 2015 pour entraîner de manière fiable des réseaux avec de nombreuses couches.\n", "\n", "- BatchNorm: \n", "- Illustration de certains problèmes liés à BatchNorm: " ] }, { "cell_type": "markdown", "id": "7848e061", "metadata": {}, "source": [ "L'article de 2015 \"_Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift_\" de Sergey Ioffe de Christian Szegedy ([arxiv.org](https://arxiv.org/abs/1502.03167) a introduit la notion de \"batch normalisation\" qui a permis par la suite d'entraîner des réseaux de grande profondeur de manière beaucoup plus fiable et reproductible.\n", "\n", "Dans la couche cachée (celle avec la non-linéarité due à $tanh$), les états de pré-activation `hpreact`:\n", "\n", "```python\n", "self.hpreact = self.embcat @ self.W1 + self.b1 # hidden layer pre-activation\n", "```\n", "\n", "ne doivent ni être trop petits ni trop grands, sinon ils sont \"annulés\" par la forme de $tanh$: on souhaiterait que la distribution de ces valeurs suive une loi normale. L'apport du papier de Ioffe et Szegedy est de partir de l'intuition qu'il suffit de normaliser les valeurs à l'initialisation pour qu'ils suivent une loi normale, avec des prescriptions sur la manière de le faire.\n", "\n", "Ainsi, le papier indique qu'une normalisation qui \"fonctionne bien\" à l'initialisation est de partir d'un \"batch\" d'exemples $\\mathcal{B} = {x_1, ..., x_m}$ qui correspondent dans notre code à:\n", "\n", "```python\n", "self.emb = self.C[X] # Embed characters into vectors\n", "self.embcat = self.emb.view(self.emb.shape[0], -1) # Concatenate the vectors\n", "self.hpreact = self.embcat @ self.W1 + self.b1 # hidden linear pre-activation layer\n", "```\n", "\n", "de faire la moyenne de ces valeurs ($\\mu_\\mathcal{B} = {1\\over m} \\sum_{i=1}^m{x_i}$):\n", "\n", "```python\n", "self.hpreact.mean(0)\n", "```\n", "\n", "et ensuite d'utiliser la variance ($\\sigma_\\mathcal{B}^2 = {1\\over m} \\sum_{i=1}^m{(x_i-\\mu_\\mathcal{B})^2}$) pour normaliser les valeurs avec la formule suivante:\n", "\n", "$$\\hat{x}_i = {{x_i - \\mu_\\mathcal{B}}\\over{\\sqrt{\\sigma_\\mathcal{B}^2 + \\epsilon}}}$$\n", "\n", "(avec $\\epsilon$ qui permet d'éviter une division par zéro). Cette formule peut se traduire en code par ceci, en utilisant l'écart-type (`std` pour _standard deviation_):\n", "\n", "```python\n", "self.hpreact = self.hpreact - self.hpreact.mean(0) / self.hpreact.std(0)\n", "```\n", "\n", "Toutes ces fonctions sont dérivables donc nous pouvons calculer nos gradients.\n", "\n", "En l’état cette formule est intéressante pour l'initialisation, mais le fait de forcer cette normalisation de cette manière ne va pas permettre d'obtenir de bons résultats lors de l'apprentissage.\n", "\n", "Il faudrait adapter la distribution au fur et à mesure, en laissant la back-propagation nous dire quoi faire. L'approche \"_scale and shift_\" proposée dans le papier utilise la formule suivante:\n", "\n", "$$y_i = {\\gamma\\hat{x}_i+\\beta}$$\n", "\n", "où $\\gamma$ et $\\beta$ sont deux nouveaux paramètres que nous allons faire apprendre à notre réseau, que nous appelerons `bngain` et `bnbias`:\n", "\n", "```python\n", "## initialisation\n", "self.bngain = torch.ones()\n", "self.bnbias = torch.zeros()\n", "## forward\n", "self.bnmeani = self.hpreact.mean(0, keepdim=True)\n", "self.bnstdi = self.hpreact.std(0, keepdim=True)\n", "self.hpreact = self.bngain * (self.hpreact - self.bnmeani) / self.bnstdi + self.bnbias\n", "```\n", "\n", "Cette normalisation de la couche cachée stabilise l’entraînement, en adaptant l’échelle de la couche de pré-activation.\n", "\n", "Le prix à payer de cette approche est important: auparavant, chaque exemple avait une contribution indépendante pour l'apprentissage, maintenant sa contribution est reliée à ceux présents dans son batch à cause de la normalisation. En pratique cela permet de rendre l'apprentissage plus régulier quand on a un grand nombre de couches, dans une sorte d'_augmentation de données_ (_data augmentation_).\n", "\n", "On peut supprimer `b1` dans le code car lorsque l’on soustrait `bmeani` ils sont annulés.\n", "\n", "Cette approche implique également de revoir la manière dont l'inférence fonctionne car nous avons besoin de la valeur de la moyenne\n", "de l'ensemble d'entraînement, ce qui est une opération qui peut être longue:\n", "\n", "```python\n", "# calibrate the batch norm at the end of training\n", "with torch.no_grad():\n", " # pass the training set through\n", " emb = C[Xtr]\n", " embcat = emb.view(emb.shape[0], -1)\n", " hpreact = embcat @ W1\n", " # measure the mean/std over the entire training set\n", " bnmean = hpreact.mean(0, keepdim=True)\n", " bnstd = hpreact.std(0, keepdim=True)\n", "```\n", "\n", "Une approche plus courante est de \"maintenir\" cette moyenne et écart-type au cours de l'apprentissage dans la passe \"forward\":\n", "\n", "```python\n", "with torch.no_grad():\n", " self.bnmean_running = 0.999 * self.bnmean_running + 0.001 * self.bnmeani\n", " self.bnstd_running = 0.999 * self.bnstd_running + 0.001 * self.bnstdi\n", "```\n", "\n", "l'attribut `no_grad` permettant d'indiquer que les variables `*_running` ne font pas partie de notre réseau." ] }, { "cell_type": "markdown", "id": "211f4047", "metadata": {}, "source": [ "\n", "#### Lien avec Pytorch\n", "\n", "Pytorch propose des [Linear layer](https://pytorch.org/docs/stable/generated/torch.nn.Linear.html#torch.nn.Linear) qui correspondent aux couches que nous utilisons dans notre réseau: ce sont ces couches que l'on associe à des fonctions d'activation. Ces couches linéaires vont être la plupart du temps \"normalisée\" afin d'avoir des lois normales similaires en entrée et en sortie avant de passer à la couche non-linéaire.\n", "\n", "La couche [BatchNorm1d](https://pytorch.org/docs/stable/generated/torch.nn.BatchNorm1d.html#torch.nn.BatchNorm1d) correspond à ce que nous avons utilisé ici.\n", "\n", "D'autres couches de normalisation sont utilisées, comme [Layer Normalization](https://pytorch.org/docs/stable/generated/torch.nn.LayerNorm.html), [Group Normalization](https://pytorch.org/docs/stable/generated/torch.nn.GroupNorm.html#torch.nn.GroupNorm).\n" ] }, { "cell_type": "markdown", "id": "f0d64d20", "metadata": {}, "source": [ "### Code final\n", "\n", "Voici en synthèse le nouveau code de notre réseau, avec l'inférence associée." ] }, { "cell_type": "code", "execution_count": 114, "id": "68b4b569", "metadata": {}, "outputs": [], "source": [ "\n", "def layers(self):\n", " self.C = torch.randn((self.nb_chars, self.e_dims), generator=self.g)\n", " fan_in = self.context_size * self.e_dims\n", " tanh_gain = 5/3\n", " self.W1 = torch.randn((self.context_size * self.e_dims, self.n_hidden), generator=self.g) * (tanh_gain / (fan_in ** 0.5))\n", " self.W2 = torch.randn((self.n_hidden, self.nb_chars), generator=self.g) * 0.01 # Pour l'entropie\n", " self.b2 = torch.randn(self.nb_chars, generator=self.g) * 0\n", " self.bngain = torch.ones((1, n_hidden))\n", " self.bnbias = torch.zeros((1, n_hidden))\n", "BengioFFN.layers = layers\n", "\n", "def create_network(self):\n", " self.layers()\n", " self.loss = None\n", " self.steps = 0\n", " self.parameters = [self.C, self.W1, self.W2, self.b2, self.bngain, self.bnbias]\n", " self.nb_parameters = sum(p.nelement() for p in self.parameters) # number of parameters in total\n", " for p in self.parameters:\n", " p.requires_grad = True\n", " self.bnmean_running = torch.zeros((1, n_hidden))\n", " self.bnstd_running = torch.zeros((1, n_hidden))\n", "BengioFFN.create_network = create_network\n", "\n", "def forward(self, X, Y):\n", " self.emb = self.C[X] # Embed characters into vectors\n", " self.embcat = self.emb.view(self.emb.shape[0], -1) # Concatenate the vectors\n", " # Linear layer\n", " self.hpreact = self.embcat @ self.W1 # hidden layer pre-activation\n", " # BatchNorm layer\n", " self.bnmeani = self.hpreact.mean(0, keepdim=True)\n", " self.bnstdi = self.hpreact.std(0, keepdim=True)\n", " self.hpreact = self.bngain * (self.hpreact - self.bnmeani) / self.bnstdi + self.bnbias\n", " # Non linearity\n", " self.h = torch.tanh(self.hpreact) # hidden layer\n", " self.logits = self.h @ self.W2 + self.b2 # output layer\n", " self.loss = F.cross_entropy(self.logits, Y) # loss function\n", " # mean, std\n", " with torch.no_grad():\n", " self.bnmean_running = 0.999 * self.bnmean_running + 0.001 * self.bnmeani\n", " self.bnstd_running = 0.999 * self.bnstd_running + 0.001 * self.bnstdi\n", "BengioFFN.forward = forward\n", "\n", "@torch.no_grad() # this decorator disables gradient tracking\n", "def compute_loss(self, X, Y):\n", " emb = self.C[X] # Embed characters into vectors\n", " embcat = emb.view(emb.shape[0], -1) # Concatenate the vectors\n", " hpreact = embcat @ self.W1 # hidden layer pre-activation\n", " hpreact = self.bngain * (hpreact - self.bnmean_running) / self.bnstd_running + self.bnbias\n", " h = torch.tanh(hpreact) # hidden layer\n", " logits = h @ self.W2 + self.b2 # output layer\n", " loss = F.cross_entropy(logits, Y) # loss function\n", " return loss\n", "BengioFFN.compute_loss = compute_loss\n", "\n", "@torch.no_grad()\n", "def generate_word(self, itoc, g):\n", " out = []\n", " context = [0] * self.context_size\n", " while True:\n", " emb = self.C[torch.tensor([context])]\n", " embcat = emb.view(1, -1)\n", " hpreact = embcat @ self.W1\n", " hpreact = self.bngain * (hpreact - self.bnmean_running) / self.bnstd_running + self.bnbias\n", " h = torch.tanh(hpreact)\n", " logits = h @ self.W2 + self.b2\n", " probs = F.softmax(logits, dim=1)\n", " # Sample from the probability distribution\n", " ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n", " # Shift the context window\n", " context = context[1:] + [ix]\n", " # Store the generated character\n", " if ix != 0:\n", " out.append(ix)\n", " else:\n", " # Stop when encounting '.'\n", " break\n", " return ''.join(itoc[i] for i in out)\n", "BengioFFN.generate_word = generate_word\n" ] }, { "cell_type": "markdown", "id": "de0895e9", "metadata": {}, "source": [ "## Entraînement" ] }, { "cell_type": "code", "execution_count": 115, "id": "2dddf2d8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0/ 200000: 3.7070\n", " 10000/ 200000: 2.1842\n", " 20000/ 200000: 2.0719\n", " 30000/ 200000: 2.0493\n", " 40000/ 200000: 2.0728\n", " 50000/ 200000: 1.8556\n", " 60000/ 200000: 1.9816\n", " 70000/ 200000: 1.6609\n", " 80000/ 200000: 1.5744\n", " 90000/ 200000: 2.1140\n", " 100000/ 200000: 1.8943\n", " 110000/ 200000: 1.5171\n", " 120000/ 200000: 1.8516\n", " 130000/ 200000: 1.5455\n", " 140000/ 200000: 1.3465\n", " 150000/ 200000: 1.3427\n", " 160000/ 200000: 1.6364\n", " 170000/ 200000: 1.9168\n", " 180000/ 200000: 1.5088\n", " 190000/ 200000: 1.6241\n", "train_loss=1.5353056192398071\n", "val_loss=1.6674079895019531\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGhCAYAAACUFDUXAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUddJREFUeJzt3QlYVOX3B/DDIuCKIgouKC65pYKiIua+Ydli+SszS7PSMi3LNmnR0grLMn+laVmmZaYtmv9fGqYoqUkuqLlTaIobIC6gqKAw/+e8OjgzzAz3ztw7c+/M9/M88+jM3Jm7DXPPvO95z+tjMBgMBAAAAKBTvu7eAAAAAABnIJgBAAAAXUMwAwAAALqGYAYAAAB0DcEMAAAA6BqCGQAAANA1BDMAAACgawhmAAAAQNcQzAAAAICuIZgBAAAAXXNJMDN79myKjIykoKAgio2Npa1bt9pd/vz58zR27FiqU6cOBQYGUrNmzWjVqlWu2FQAAADQGX+1V7B06VKaMGECzZ07VwQyM2fOpPj4eEpPT6fatWuXWb6oqIj69esnnvvxxx+pXr16dPToUapevbramwoAAAA65KP2RJMcwHTs2JFmzZol7peUlFBERAQ988wzNHHixDLLc9Azffp0OnjwIFWoUEH2+vj9T548SVWrViUfHx9F9gEAAADUxeHIhQsXqG7duuTr66udYIZbWSpVqiRaWAYNGlT6+IgRI0RX0ooVK8q85o477qCQkBDxOn6+Vq1a9NBDD9Err7xCfn5+ZZYvLCwUN6MTJ05Qq1at1NolAAAAUNGxY8eofv362ulmys3NpeLiYgoLCzN7nO9zy4s1hw8fpnXr1tGwYcNEnkxGRgY9/fTTdPXqVZo8eXKZ5RMTE+mtt96yejCqVaum4N4AAACAWvLz80XPDfesaC5nxpFuIs6X+fzzz0VLTExMjGht4a4na8FMQkKCyMmxPBgcyCCYAQAA0BdHUkRUDWZCQ0NFQJKdnW32ON8PDw+3+hoewcS5MqZdSi1btqSsrCzRbRUQEGC2PI924hsAAAB4J1WHZnPgwS0rycnJZi0vfD8uLs7qa2677TbRtcTLGf39998iyLEMZAAAAABUrzPDXUDz5s2jhQsX0oEDB2jMmDFUUFBAI0eOFM8PHz5cdBUZ8fNnz56l8ePHiyBm5cqV9O6774q6MwAAAAAuz5kZMmQInT59miZNmiS6iqKjoykpKak0KTgzM9NsCBbnu6xevZqef/55atu2ragzw4ENj2YCAAAAcHmdGVfjBODg4GDKy8tDAjAAAIAXXL8xNxMAAADoGoIZAAAA0DUEMwAAAKBrCGYAAABA1xDMAAAAgK4hmAEAAABdQzADAAAAuoZgRoYfth+jPzJy1TsbAAAAoP9Zs7Vq38k8eunH3eL/R6YNdPfmAAAAwA1omZEo92KR1EUBAADAhRDMSBRRo6L4t2oQGrMAAAC0BMEMAAAA6BqCGQAAANA1BDMAAACgawhmZLpw5Zo6ZwIAAAAcgmBGoj0n8hw7wgAAAKAqBDMS5aNFBgAAQJMQzEjko+55AAAAAAchmJHIB9EMAACAJiGYkcgHbTMAAACahGBGIrTMAAAAaBOCGYlMe5kMBoM6ZwMAAABkQzDjQMvMgVMX5B9pAAAAUAWCGYl8TKKZwmvF6pwNAAAAkA3BjEQYzAQAAKBNCGakHihkAAMAAGgSghmpBwpHCgAAQJNwiXagzoxp/gwAAAC4F4IZiUzjFwzNBgAA0A4EMxKhNQYAAECbEMxIhI4lAAAAbUIwI/VAmfQzoZUGPFHa0bP0wNxU2ncyz92bAgAgC4IZiZDzC55u8JxU2nrkLD38xRZ3bwoAgCwIZqQeKPQzgZc4d+mquzcBAEAWBDOS3Yxm0o6ek3eUAQAAQDUIZhxomVm85ahKpwMAAADkQjAjEZJ+AQAAtAnBjERImQEAANAmBDNSDxSOFAAAgCbhEu3A3EwAAADgZcHM7NmzKTIykoKCgig2Npa2bt1qc9kFCxaI/BTTG79OS3VmDp0ucOemAAAAgCuDmaVLl9KECRNo8uTJtGPHDoqKiqL4+HjKycmx+Zpq1arRqVOnSm9Hj7p/9BASgAEAALw0mJkxYwaNGjWKRo4cSa1ataK5c+dSpUqVaP78+XYDh/Dw8NJbWFgYuRuK5gEAAHhhMFNUVERpaWnUt2/fmyv09RX3U1NTbb7u4sWL1LBhQ4qIiKB77rmH9u3bZ3PZwsJCys/PN7upATkzAAAAXhjM5ObmUnFxcZmWFb6flZVl9TXNmzcXrTYrVqygRYsWUUlJCXXp0oWOHz9udfnExEQKDg4uvXEA5OqWmeISgyrrBAAAAB2OZoqLi6Phw4dTdHQ09ejRg5YtW0a1atWizz77zOryCQkJlJeXV3o7duyYOhtmEcz8uueU+Hf38fPUevJq+mLjYXXWCwAAAO4LZkJDQ8nPz4+ys7PNHuf7nAsjRYUKFahdu3aUkZFh9fnAwECRMGx6c0U305hvd4h/X/5xN12+WkxvrzygynoBAADAjcFMQEAAxcTEUHJyculj3G3E97kFRgruptqzZw/VqVOH3AkJwAAAANrkr/YKeFj2iBEjqEOHDtSpUyeaOXMmFRQUiNFNjLuU6tWrJ3Jf2JQpU6hz587UtGlTOn/+PE2fPl0MzX7iiSfInTA0GwAAwEuDmSFDhtDp06dp0qRJIumXc2GSkpJKk4IzMzPFCCejc+fOiaHcvGyNGjVEy87mzZvFsG6tFM0DAAAALwpm2Lhx48TNmpSUFLP7H330kbhpDbqZAAAAtElzo5m0q2zTzJ2fbKSDWRfcsjUAAABwHYIZJ7qZ9p5Qp0AfAAAASIdgRiKkzAAAAGgTghmpBwoZwAAAAJqEYEYixDKep6TEQBv+Pk3nCorcvSkAAOAEBDNSDxSiGY/z3bZMGj5/Kw38eKO7NwUAAJyAYEYFJ89fprzLV0mLLRE5+VfIE63YdYLSjp6T9ZqkvdcnOz2Z55nHBADAWyCYkUhqw0zOhSvUZdo6inrrN9KaZ5bspE7vJtPa/eZzZendX8fO0/glu2jwnM3u3hQAAHADBDMOTjRpy94TeaRVK3dfn+n70xTrk3bq1ZEzBe7eBAAAcCMEM1IPFI4UAACAJuESrXDLjFxZeVfIYDCo8t4AAADeAMGMGy3bcZw6JybTaz/vlf3aPw+fEYGQPZwQ+/AXW+jvbEy5AAAAngvBjNQDpULDzAer08W/i7dkynpd6qEz9ODnf4pAyB5OiN2UkUuPzt/q1HYCAABoGYIZBUczFV0rIVdIPXxG1vKnPHQ4NgAAAEMwI1n50cxTi9IU+VRxDs3h0xcVy6Xht9mZKa8GCwAAgF4gmFGwZWbdwRxSwow1f1PvD3+naUkHSSn3fnqzBgvSjeXh3CSuHwQAANqEYEbF6QwuFl6T/Zo1+7Ppk3XX68B89vth6wth9JPLXLlaLHKTOr2TTMUl2gwDubIzAIA3QzAjkSP5v60nr6azEicxvFp8Pd9m1NfbHViTd/NRcd6s0xcKXZ4TJcf69Bxq/ebq0oKIAADeCMGM1AMl8YL55v/tLzPyyNRzS3bSEwu3lcmHeWzBNtIC3q4vNh6m3/8+Xe5yu46dpwtXtDcHlTcZ+dU2ulRUTGMX73D3pgAAuI2/+1atL1J//GeevVTmMQ5oqgb5U7OwqvTzrpPisWNnL5sts/GfXMnbolSnwucbDlFolUC6r319s219e+UB8f8j0wbanaRxzLc7qGHNSvT7S70U2iJwtUOnL5K/rw81rFkZBx8AdAstMyqbufZvGjrvT7rzk01kMAlD+P9qdo+UJyPnAr276iBN+P4vs8ePnzcPsmz55Ua3xtEzZYM3PVr051F65MstdKnomqQuQSVGmvH7zEk5RLuPny/t0pr7+yGzri01cU5Xnw9/px7TU5B3AwC6hmBGIkfjjn9yLrolb3f/yXy7z+ddLts95M3TKrz+817ROrZg8xG7y+XkX6FbJ68uEwQ6GkC9l3SQ7p71h7g/+pvtNO3Xg+JfV+B9MSrx4nMPAPqHYEYiJVpRvpVZ6dcWy+sOByEcvBiTiNkdH2+U/HrGicpdpq2j935Vbji4uxlHH+VdukrnrCRiWzunBXZGoHFrGp9DTgRevvOE1WX49VIThQ+eujnNxAOfpdLOzOstNMZ/AQBAGgQzEgX4OX+opv5inhwsBQcqPL/SiPlbrbaccLcItyZw8PL0t9KSQK39Bp+/6V86lXeFzkgcfaV1s9b9Q23fXE0HTuVT1JTfqN3UNQ6NRpITw3Igw6025U0zYc3Wf8/Kfg0AAFyHBGAXBjOO1KDJuVAo5ldi+VeuUXDFCmbPt5q02qxGjaNM83k8wQe//S3+TVi2p/Sx85eKqHa1INXWyYETkzoc35HPzPIdxyn+1nBV9wMAQG/QMiOVwrm61qY+GDT7eu6EqWsWBdF4WoJZ668X1dOK7UfOKl47ZfMh6aO79p3II2/w+vI99MaKfWKSUQAAuAnBjERKDzyyHJrNuG6LpSctkkE5t8KVXRb7TubR+CU7KdPOqKXDuQUOV9c1JqEWXium77cdE9vOtVMemrdFUsVdrn772QYblZI1Tm5rWPKBHKeOd9n1a0d2/hX6fvsx8ZmwNfpOi0ULAUAb0M0kkbsGUe89cXNUEn+ZXy0u/xIkJeCRSgwpN1xPVl39fHdFD0jX99ZT7sVC2vhyL/oh7Th9nPyP2fMczPj5+tjNi+GhzfZYO1rlva83cmeZADbw403is8B1bxJub2n23P/9dZKe/W4ndWlSkxaP6uy2bQQA7ULLjESVA5SP+05IrOlilLTX9SXrjTnHGadvDjGXgkcPXSsuEa0tR89Yb0ngixfb8M9p2lBOxWFbeTEFRdZ/yZcyTZr2IXrh+78o5u01lJ51QfY6yxu9bNwfU+8nHaQH5qaiVYGkHbuUg2XPyYe/pYt/N1tU0wYAMELLjES+Gvglb5k/Y81fVrqqyrsqS5moUM4ki9+kHhG5HbWrBooE5vKqCbvSTzuOi3/jZ26QdIwtWyzsHYVPrbQSGR9L2pdFd0fVdWCLwVMKMwKAetAy42H+zr5Zu8S+mxfpwXM3O73e//11kuI/2iC6CTiQYcZAxlHuKOTGM5Vvs5HQXN7WHD5tO5flqsx8D7VnwubclLf+J79UQHmSD2TTJhlTcwAAKAHBDDhfpM1A9Mx3Oyk9+wK9+IPtyrgzfkun7u+vl3yh5tYTR6oSv/7zzeHYjnjlp92qDrc3srdrL/3o2DbYk3b0HH2akiFa2T7fcNihrj17zlwspMcXbqeHv9yC6REAwKUQzHihYgVaPFbemJuJGevgsEuFtnNYPl6XISbifGfV9YkspXQvXJAZIORcuEKL/rxZadmRPeWKwZyfxAnXlp2L6w9eH1HE1GyBMHaHKWnwnM30flI6/Zh2TPGuGx7tNtGkpo+WRkoBgOdDMKMjJ2UmDEsZIaUEHm1SnqXbbgYY3CJwj0lNHSWH3O61qDljnv8rLe+JqyA/tWgH/Tf5euE9o3tmbaI9Ju/PLRCOdvGs2HWCzl1yT7XlV37aUyZYcjYjjLsqnSnaaMufh5H0C+7F89ip3e0LzkMwoyPzNv6r2HtZq2kjlekcUHIuoKaTb5omKp+/VHbSS0sc8GzOyLVZh8Tor2N5NkeMWQYnUmcGNzpkJydGjsRVB2j8kl209kbdGMncn4Nuk1IzfXNXpSkUCAR34hzAqLd+o6HzUKhS6xDMeBgecuzIPFHWfnjM/d16DRcubqa08n73vPW/ffTQF1tk57OYTi1g2v0klRrlV1bucf0Qe7m45UgJfPxfXb5HVvCcYTLTPIA7/Zh2vQVzC+ZO0zwEMx7mi02Otd5Y6+qZZjGDNieOcpKrvV/hatVeM844vmJX+V1aSrlWbKDcC0WKXPR5ZJYjycxycJDJ+T6W+LyVt27LoerccmTZxcOtXFL3wbjcgJkbaPGWzNKpOvKvXKW1+7Ptdi1y1WlbuHaRnDIBAOAdEMyAZE1eXUWtJ69WrEvBdDi5vfo41vqrbRUcVDKY4nXcNWuT3WX4oi8Fj066b85mcZHPvVikSG6QZWDx8o+76enFaWWqJN/y2ipqlLDK7vtx15+9FpLPfj9Et01bR9NXXy9gJwUPcbccnv/IF1voia+304w19rv8/sm+QMt3lk2C7vlBCvWb8bvqgSEA6AuCGRC+lNGiY2wlsabIgXyaX/dm2X3+t/1ZZaZr4AurlvAoKilD4N9eKW0kF/ti42Ea++0Oqy0RPM3E/yxyetgfGTdbU37ZfVJUSVaiISPxRiudtcKAtmyxkrz71/HrrS4/77zZorX7+Pky9Xr6fbSBnl9adpj/8XOXxdxUcofBe7NjZy+JitwAngzBDCjKXuE4R2zOOFPm172SE2kqgSdB7PROsuJBIwc+nF9z76d/iBoxln660Z9vy7jFO0kP7p5lPlv8fy3m6ALnguxu76+ndlPXeNVh5IECh2VOwaJm92/nd5PpwCllR5GCOQQzoGkcKMghdfi1kvrOsD41glJ2H88TNWIuXNFOa4SzeStZ+Vfo+aW7MGeVyg6ckvf34ynu+3Qz9f7wd/pd4cKQjuDuX/6887xwoPNgZvbs2RQZGUlBQUEUGxtLW7dulfS6JUuWiLlxBg0apPo2gjZxN8m2I2VbJZzp7nGFzzdI747RKlv5R+OX7KTYd9eKZF5bpIQ6y3eeoGav/+rQts1an+HQ68CzfbvlKK1Pz6H9N1pBlqlQfNJRSFzXeTCzdOlSmjBhAk2ePJl27NhBUVFRFB8fTzk59mtsHDlyhF588UXq1q2b2psIGsfzPimRz+NK764yHwmmhh2Z0oM8JfGIMk5i/uWvU2K2ay4qZs2Vq8oVQ7Q2h5ZSeITUU9+kiRwluJlnw8fEWvemVnFS/GvL99LIr7ap8v625mzzJll5VxyqM+YRwcyMGTNo1KhRNHLkSGrVqhXNnTuXKlWqRPPnz7f5muLiYho2bBi99dZb1LhxY7U3EUCXbHU7FSiUHMsDht78v320ZKv1APFi4VXq8PZaUVTMmjkWdYr23Ej+1RpOQOdZzeUkZ3u6sYt3iGPC3Zt6kZ1/RdMV2A0m7ZUcEGg1KLCFa0V1Tkym/8xNJa8LZoqKiigtLY369u17c4W+vuJ+aqrtAzJlyhSqXbs2Pf744+Wuo7CwkPLz881uAN7MsiCio1IPnaEFm4+YzbkktfUp//LVMs3q9iYhdadLRbaDv8JrxbTgj381k0zqKkrP3aUnPOz/vaSD9P025YuDGktNdJm2jjq9s1a0CurF9zeKpdoro+GxwUxubq5oZQkLCzN7nO9nZVkfjrtp0yb68ssvad68eZLWkZiYSMHBwaW3iIgIRbYdQIn+e3dYss168Ty5TKsnyxXz9lqXFVR0pEQ9fzFLmW+Hu7Pe/N9+kUyqBVyTiafDOO+meb28AXetzUk5RC/LrDYup0WVa3Wdu3RVkZpTcJ0/aciFCxfokUceEYFMaGiopNckJCSInBwjbplBQANawP337hI1xXrXjxypHjrJYx+TwOSBDuY/frhbjevYLHi0I/n6+tB2jeWM9P9oQ2lBx1kPtSdvxi19HHB/OaKDGCiiFClzxTkCdR51HMxwQOLn50fZ2eaz6fL98PDwMssfOnRIJP7eddddpY+VlFxvhvP396f09HRq0qSJ2WsCAwPFDQBAjp2Z58oEM9ytxjiI6dQoRLMH1HJ2eFs8uVKycd4kbmlrWruqKuuQUxfK3dYfvD6K6+meTRQN7ozKm+TXo7uZAgICKCYmhpKTk82CE74fFxdXZvkWLVrQnj17aNeuXaW3u+++m3r16iX+jxYXAP06KHESVC3w5mG0CzcfsTlCTYuM8Zq1uM3ZS7q9OcS0ZuSCbWK6kY3/5Ja7LHexcu6LnP1btkOZyWd1283EXUAjRoygDh06UKdOnWjmzJlUUFAgRjex4cOHU7169UTuC9ehad26tdnrq1evLv61fBwAwB6ezmH7kXP0xp2tyM/XR1OFFrWAW21eXb6XWoRXpRFdIksfn/x/+9y6XZ7qsotaNrIkjOrikYYc+PRvFUafD+9AnkD1YGbIkCF0+vRpmjRpkkj6jY6OpqSkpNKk4MzMTDHCCQBAScbpHNo1qE73RNcr87y9HhglWum5WZ7XUTHAj9wRqHCxSXv5H5sycum7G8PuH4ptQBX83Ps9fOHKVZEY27hWFfJEPK8Y5/j4aSAT/osbNZV+22+eAqJnLkkAHjdunLhZk5KSYve1CxYsUGmrAMAbuGPECDfjR0/5jQqvlVD61NspwN+xQCHzzCXafeI8DWxTR1IeBF8suYsoPesCPbXIfAZ1y2An//LNIeld31tHW17ta3NfOBlabV0S19GFwmu08tmudGvdYPJEKek51KdFmNXaM0Y8ieoHq9Pprqg6FNNQu3lbWoMmEQDwOH+ajMRyJAnW2Uv3lWvFogIyr9pyig2eJXzehsPl5uVwEbju09eLFqb/k1gFu/3UNdTrgxRasLls4uq+kzeThkd8tU0Uxru5LvPJXE1tzCg/B0MJHMgwW/Mp8fQZxuH0Us+pJjOfyvlwzfjtb5GIPniOesXpDOR5EMwAgMd58PM/Fev6kIIDhf4f/U7JB7LLrezKs4S/s+pAaREyW2LfTXZ4pnhr1ZYHfryptHbQBhkTMBZqYBTLwax8avvmb/T4QnWmKpCLiygu3pIpu+jd3hPmRV2tndcMhQo05l2+Kj6/HPhx8Uej26atU234uTtpqs4MAICrlP6497HdJbAzU1q109Ffp4naL48v3C7uT7uvTbmv4a4gRzk69JZL8odUDiBX237kLJ3Ku0J3RdV16PXfpF4vQLk+vWwQxoeCh6o/8fX1Y285j1ibesH0RDdlp8UxFlHkwHVopwa0dFsm3dY0tNx8n/l//EuDY27mb41fsstqPpfR5aJiCvT3ld3NV3ituHSakU6RIZSWeY52vN6PgitVEJ9TT4SWGQAAK8HCozImLLRswbGcAuL2/26kyIkr6d/cAt3VgBn9TRrlODnvEc/n88x3O0ULi6VN/+SKmj/OGPOt7fwgnnOr1aQkMVEkBwebM3LttqjYOisceFjOFM+TvXKA8saKfZKrRP+TLb3lpeWkJLrXgfmxck3yxLYeOSu6NFP+tj+5s94hmAEAj+ZIzHAqT9lfrwdOXb+Icz6LrdmIn1+6S0zmp0X/Tf5Hkfc5cc78uHKQ9PCXW+jeTzc7dU45SLHnUlExDf38TxH0PPTFFtHNN37JTvqfxFwkoy83ls1F4uH/lniU2My1f0t6z883HKIZa2wvq4W5kEZbafXSGgQzAKBLUieu/GzDIZHLoqUZmJnp9fiFH3bR8p0naNDsPxR5b6UrwJbIiAiX7zxOXRKTJVUptpZ4rFbdn2vcOnGjm+qrP46ILihuLZKjSGKOTMKyPTRz7T9WW6KsTdj6cfI/lJEjP1fGVa17v+lgCDeCGQDQbTl7KRNqcpO7MZfFlLVhsa7E1yHunuKRTX9knHH6IrZi1wmnL3K2KsJeLZb+fs8v/YtO5l2hofOUScK2xnRrvtt6zGyYuVzjFu+gjf9IT4iWO6mkErO3W8O5Lx3eXmu3VcebIJgBAN1yJiBxVcoK15qx5a3/7RddHkp44XtpLVW28CSbzV7/1WbguD49R7ULubPzJ0ltMbHml92n6JEvtyqyLc7mFskJRD/8LZ3OFBSJVh1AMAMAOpZ7sVDkHJy5aLtOitGyHcdFIq4RJ4Ryy4i1Tg0lp2YynaXbFAcHxskSy/PtlkxF5ovaaSf/wjjJpi3PLt5J5y8V2R0pZawDY4uxK4UTcHmEU1FxsdXWIdML+tEzBWL/jUyHGWtNp3eT6ZNyggtHegC5zhDnVTkjYdkemvFbepnH+fja8sP2YzT390OkB2iZAQDd4u4jzjmwV+3WaML3f5Um4rIjZy7RfTYST6Veb/iiyyOU8h1oheDy9nJM+d8+p3Nm3vh5r8NdUFzULnrKGrNjaLRy9ynqMm0djV+6q0zXCQeRRom/HhTJ1R/89rcY4cRdUpY+Wvs3vfjD7tL7z1m85/fbpQWAjvIpJ3eIgzaz5S1e8KGT3T7WJop89rud1OfDm8njPKrK1sSP0349SL/uOWU1CfrjdRllHu8xPYUW/Xl96Lull37cLd5PDxDMAIBuHT1zSfzL8xA54p+ci1ZbPKT+ep76ywGbI5SUtvBGrRWWc6FsS5TBoiS+WpbcmM/J1CfrrrdGWI4OajVptQgiLWdP59Y0lnn2+vmz9JNJAMStb44UMlTDZ78fFkGbnPm5pC9bQvd9ajsBvMBkxNZry/faXO5sQZEYju6O0WruhGAGALyaZU0Yxq09UnCNEXfgX9mWF00luqEcMWsdj9qRVwDQmU399s+ywZRWWcvFOXDK+rHiKSh2SCzSmHJQnZox2flX6O/sC6ILUM2AWA2oAAwAoHPtpqyRvOy7TiYcWyY0c5eRHNa6qazhLqpKAf5WR/GoOSRZ7ZDQVg7Kqj1Z5b6W91vpYfeMZyt/+5f99MWmm8F587Cq5OrtcAZaZgDAI7irZcJdZq/PoN4fpIhuhcsyujPmWSn8Jodpd4cj3k8qm4RqDXdR8azhcJPcIfxymAYyLD3bdmsbT02hNQhmAMAjcMl6teqFaNH01el0OLeA3tNJgqYjeNZwa7WEDCoGxDzPklSunpHiTIH1UXuDFCq2qGcIZgDAI3D3x+ML1Cu7PvWX/fSTxKHUrrQpo+zoFzWZdvEYE3nV5MhIMUfxTOZrD8jLR9FCZ8suDUx54G7ImQEAj8HF075OtV8vxZnibGxwTH1SE9f9kFNx19WzIBtutGCM+no7rVMpEdUdOEfH2Qkv1ca1gLhFjofJgzkEMwDgUSatkF6PRYu47oeW8cSHv/+d49ZAxl5VZUdxjk6gv7zOCq6ZU6NSALnKTomjndTGk6JOvL0FtWtQg7QC3UwAADIcP3dJdLUUeOmvYy72Z2sOJ1eZ9qsyU0A4GyQ5Wt9I77b8e1b2TOdqQ8sMAIDMOYee/CZNFzMJe2rrF894DWAKLTMAADJ5cyBjqwKxK7lqEkspNFZuxWshmAEAkOG9JM8dCg3yrd7n3YGtViCYAQCQISXde2rZAOgFghkAAADQNQQzAAAAoGsIZgAAAEDXEMwAAACAriGYAQAAAF1DMAMAAAC6hmAGAAAAdA3BDAAAAOgaghkAAADQNQQzAAAAoGsIZgAAAEDXEMwAAACAriGYAQAAAF1DMAMAAAC6hmAGAAAAdA3BDAAAAOgaghkAAADQNZcEM7Nnz6bIyEgKCgqi2NhY2rp1q81lly1bRh06dKDq1atT5cqVKTo6mr755htXbCYAAADokOrBzNKlS2nChAk0efJk2rFjB0VFRVF8fDzl5ORYXT4kJIRee+01Sk1Npd27d9PIkSPFbfXq1WpvKgAAAOiQj8FgMKi5Am6J6dixI82aNUvcLykpoYiICHrmmWdo4sSJkt6jffv2NHDgQJo6dWq5y+bn51NwcDDl5eVRtWrVSEmRE1cq+n4AAAB6dWTaQEXfz5nrt6otM0VFRZSWlkZ9+/a9uUJfX3GfW17Kw3FWcnIypaenU/fu3a0uU1hYKA6A6Q0AAAC8h6rBTG5uLhUXF1NYWJjZ43w/KyvL5us4KqtSpQoFBASIFplPPvmE+vXrZ3XZxMREEckZb9zqAwAAAN5Dk6OZqlatSrt27aJt27bRO++8I3JuUlJSrC6bkJAggh/j7dixYy7fXgAAAHAffzXfPDQ0lPz8/Cg7O9vscb4fHh5u83XcFdW0aVPxfx7NdODAAdEC07NnzzLLBgYGihsAAAB4J1VbZribKCYmRuS9GHECMN+Pi4uT/D78Gs6NAQAAAHBpywzjLqIRI0aI2jGdOnWimTNnUkFBgRhuzYYPH0716tUTLS+M/+VlmzRpIgKYVatWiTozc+bMUXtTAQAAQIdUD2aGDBlCp0+fpkmTJomkX+42SkpKKk0KzszMFN1KRhzoPP3003T8+HGqWLEitWjRghYtWiTeBwAAAMDldWZcDXVmAAAA1Oc1dWY8TdVA1RuyAAAAQCYEMzIMbFtH7vEFAAAAlSGYkeGRuIbqnQkAAABwCIIZGW6tG+zYUQYAAADVIJgBAAAAXUMwAwAAALqGYAYAAAB0DcEMAAAA6BqCGQAAANA1BDMAAACgawhmAAAAQNcQzAAAAICuIZgBAAAAXUMwAwAAALqGYAYAAAB0DcEMAAAA6BqCGQAAANA1BDMAAACgawhmAAAAQNcQzAAAAICuIZgBAAAAXUMwAwAAALqGYAYAAAB0DcEMAAAA6BqCGQAAANA1BDMAAACgawhmAAAAQNcQzAAAAICuIZgBAAAAXUMwAwAAALqGYAYAAAB0DcEMAAAA6BqCGQAAANA1BDMyjerWSJ0zAQAAAA5BMCNTSOVAx440AAAAqALBDAAAAOgaghkAAADQNQQzMlUJ8lfnTAAAAIBDEMzI9ECH+o4daQAAAFAFghmZAv391DkTAAAA4BAEMwAAAKBrLglmZs+eTZGRkRQUFESxsbG0detWm8vOmzePunXrRjVq1BC3vn372l0eAAAAvJvqwczSpUtpwoQJNHnyZNqxYwdFRUVRfHw85eTkWF0+JSWFhg4dSuvXr6fU1FSKiIig/v3704kTJ9TeVAAAANAhH4PBYFBzBdwS07FjR5o1a5a4X1JSIgKUZ555hiZOnFju64uLi0ULDb9++PDhZZ4vLCwUN6P8/Hzx/nl5eVStWjVSw3tJB2lOyiFV3hsAAEAPjkwbqOj78fU7ODjYoeu3qi0zRUVFlJaWJrqKSlfo6yvuc6uLFJcuXaKrV69SSEiI1ecTExPFzhtvHMio7ZUBLWjqPbeqvh4AAABwczCTm5srWlbCwsLMHuf7WVlZkt7jlVdeobp165oFRKYSEhJEFGe8HTt2jFyhWsUKLlkPAAAA2KfpCnDTpk2jJUuWiDwaTh62JjAwUNwAAADAO6kazISGhpKfnx9lZ2ebPc73w8PD7b72gw8+EMHM2rVrqW3btmpuJgAAAOiYqt1MAQEBFBMTQ8nJyaWPcQIw34+Li7P5uvfff5+mTp1KSUlJ1KFDBzU3EQAAAHRO9W4mHpY9YsQIEZR06tSJZs6cSQUFBTRy5EjxPI9QqlevnkjkZe+99x5NmjSJFi9eLGrTGHNrqlSpIm5a4ePj4+5NAAAAAFcEM0OGDKHTp0+LAIUDk+joaNHiYkwKzszMFCOcjObMmSNGQf3nP/8xex+uU/Pmm2/ipAEAAIDrE4DHjRsnbtZwcq+pI0eOuGKTAAAAwENgbiYAAADQNQQzAAAAoGsIZhyE9F8AAABtQDADAAAAuoZgBgAAAHQNwQwAAADoGoIZB6FmHgAAgDYgmAEAAABdQzADAAAAuoZgxkEV/HDoAAAAtABXZAf1blFb2TMBAAAADkEw40TLTJ3gIEdfDgAAAApBMAMAAAC6hmAGAAAAdA3BjBMwPxMAAID7IZgBAAAAXUMwo6BaVQOVfDsAAACQAMGMgga2qaPk2wEAAIAECGac4IMJmgAAANwOwQwAAADoGoIZAAAA0DUEMwAAAKBrCGYUhBQaAAAA10Mwo5K7ouqq9dYAAABgAsGMSi0xg9vXK/1//RoVnVkNAAAA2IFgRiU1K98soNewZiW1VgMAAOD1/L3+CCjIYLj5/ya1K9PUQa2pVpVA+iMjl/7IOINDDQAAoAIEMyp6pHND8W+XpjXJ14doYepRNVcHAADgldDN5ALVgirQW/e0dsWqAAAAvA6CGQUTgCsG+N08sBinDQAA4BIIZhQUWiWQEm5vQZPvakVBFW4GNkYrxt6m5OoAAAAAOTPKe7JHE5vPRUVUx4cOAABAYWiZcSN/zgoGAAAApyCYccI7g9o4dfCXjO7s1OsBAAAAwYxTujerRQenDnD49R0iQyioAuJJAAAAZ+BK6iRrib5yrHy2G/VrFebsZgAAAHgtBDMKql315hQGUjWpVUWMgAIAAADHoAKwAuYN70BpR8/RwDZ1lHg7AAAAkAHBjAK4mwhdRQAAAO6BbiYAAADQNZcEM7Nnz6bIyEgKCgqi2NhY2rp1q81l9+3bR4MHDxbL+/j40MyZM12xiQAAAKBTqgczS5cupQkTJtDkyZNpx44dFBUVRfHx8ZSTk2N1+UuXLlHjxo1p2rRpFB4eTt5o71vx1CK8qrs3AwAAQBdUD2ZmzJhBo0aNopEjR1KrVq1o7ty5VKlSJZo/f77V5Tt27EjTp0+nBx98kAID5Y8O0iNugTJVJdCfVj3bjSJCKrptmwAAAPRC1WCmqKiI0tLSqG/fvjdX6Osr7qempiqyjsLCQsrPzze7eQJfXx+aMyzGoeHeAAAA3kTVYCY3N5eKi4spLMy8KBzfz8rKUmQdiYmJFBwcXHqLiIggT9G6XjBtebWPuzcDAABA03Q/mikhIYHy8vJKb8eOHSO9MRgMkrugAAAAwIV1ZkJDQ8nPz4+ys7PNHuf7SiX3cl6NJ+XWREdUd/cmAAAA6IqqLTMBAQEUExNDycnJpY+VlJSI+3FxcWquWrfevde5mbgBAAC8jerdTDwse968ebRw4UI6cOAAjRkzhgoKCsToJjZ8+HDRVWSaNLxr1y5x4/+fOHFC/D8jI4M8wf0x9cW/w2IbWH2+WVgVF28RAACAvqk+ncGQIUPo9OnTNGnSJJH0Gx0dTUlJSaVJwZmZmWKEk9HJkyepXbt2pfc/+OADcevRowelpKSQ3r1zbxu6v0MEtWvgmu6k+9rXo2U7TpDWfHh/FEWGVqbBcza7e1MAAEDnXDI307hx48TNGssAhSv/2kuI1bsAf1/q1CjEZesb26upJoOZ9g1rUEHhNXdvBgAAeADdj2YC7bq3XT13bwIAAHgBBDMezteNQ7sn39XKbesGAADv4ZJuJnC9prWrUJNalSmyZiW3Hf7qlQJsPufv60Me3JsIAAAuhGDGQ703uC3FNKxBWhURUonOX8pz92YAAIAHQDeTBnhbA8W6F3q4exMAAMCDIJjRGC1NX/Dr+G5Ov0enSPORW3e0CafGtfRVS6dakLINmN1uCVX0/QAAvB2CGbCpgp/zgZXBiXanQ+/eQZ6oeVhVd28CAIBHQTADmuXnq41WKj12A3LyNwCAt0Awo2O/Pd+dtOrZ3k1JL57re4u7NwEAAJyAYEZnhsc1LP1/Mw13V0zo35z0omIFP5vPxTWuqfr6ezavpfh76rE1CQDAUQhmdGRQdF2Kjqju0OssvTyg/GCjfg3HatS8ekcLm8+1re+aOamUuPD/Nbk/LR4Vq+q6Xx/Ykr4c0VHVdQAAeDoEMzpSo3IA3R1VV8y8/cH9UZJfN/PBdmJOKFNNyhlRNLBtHQqy02Jhz+juTaw+PunOVvTYbY1ubkPtyqUF9NzJVvG+AD9fMbrMXsuNo0nU3Ztdb425O7quZnKDAAD0CkXzdOCjIVFissjn+jQjfz9fmi4jkHFUJScu4LY81rWR+ToC/Gn3m/2pgq8vtZyUJOk96gYH0cm8K+RK797bhp74eruigdLCkR2p8FqJwwEjAADchJYZDagcYD+mvLddffrm8VgKrlRBsXVqZSqBakEVqGKAH0XVD1Z9XRP6NXPodX1bhdHTPZtQpQDlAg9u8bEVyLSuV02x9XgCy1ZFAABL+JbQgPDgIEq4vQW9Pai1Il0OjWtVprYuCA70FDSxZ/vYHrVUNdB+QPnygBa07614coXqFQPsdk15m5+fvs3dmwAAGodgRiOe7NGEHu58c6SSM9Y+34Mq+PmWO1S6VtXAct+L81zaNdBe0m55kl/oQcEVpbdk+VoJItUqxtzQyuSfHSO1O4+WOz3Tuym1qouWKgCwD8GMB7J2YR7bq6nF9AQG8pGY57Lc5Jcxt/owV7X8PBTbwKHXcYLziC6RLm9lSnu9L4VUvtmyYvmScb2a0tBOZfcpsmZlh4Oou6LqanpstqPnEABAKgQzXoJzNFrWMf+FW91ODo7BTt2VHW/0oxVjXdP0P/We1jTtvjal9xtpvLJtzSqBNoNEjjFfjG8ukriV9GgXZVr01OKnofnGAMAzIZjxYrMfau/Q67jlwXJCzP/E1HdqW2wFT5xDdGvdm61AH94fLfk9fVwwf5RSpFzvDTaahiIs6gFxN9ZgJ8+H0iUFAADUhGDGi92i4QrC9pKl/02UNgGltzQI1K4WZHY/5cWeVKWchGZX4erGfVrUdvdmAICH08Y3Hnj1KCO5LFuFtMxWa4o3HJ8j0waKf0+cv+zuTQEAD4eWGVCVjmMmSepVr+jW9U+951bSOncfIwDwfAhmdO7zR2I0EWTUULCgnyVrDQ0NQhybN0pplknV9qgxMegjccqO2AJw1sjb8JkE10Mwo3P9bw0nLRjX+xbq3aI2/fdB6Qm6zqgTbJ4nogfzhnew+ZyPSbqyVrqJABzBZSAAXA3BjAeIvzXM3ZsgCtTNf7Qj3RNdz+Xr5oTXGQ/Im6+qa9NQcgVjwTcuyR+hkdYkPfHWqscAIA+CGQ/w0ZBoeqqH9ZmqvUFkaGW6r319u60dlmpXCzSrg6JWY8jch2PowY4R9MszXZ16n74tlR8RVFPjQ6aXP92FOkaGWA3eXRXALx3d2SXrAev+mtwfhwYkQTDjAXj26X6t3N86ozXlBShcKr95WNUyFWoH3BpO97Wr5/CM1qbrrV+jEk0b3NbpfJnW9YJ1nZz941NxNKRDhJiBXKo2VvZ53Qs96LNHOojbk90by96OTlaCI3tiG9eUvQ5vp+SErHKmJAHvhmDGS2ltaLY7tueF/s1p9fPdqWqQ+Rfm3EdiaMYQ1+T+eIsOkSH03n/aykoUN/1IfP1YJ/p0WHtqXKuKU9vB2+BIEKQ2ngPNk35cNQp1vlK3tclfb2+tjRxB0B4EM17KVVVv72tfT/aoH2+n1fTfvQrPGs5TY9hjOlkq587c0aaOMuttor3WFj1O5qr2MfbzM/9LeOfe1jTnYcdGb4LytFKY0wjBDFhV3aJ5t/KNpuNezeXlbgzt2IB+eCpO3MA2Vw1gMm0BkFvQT+kvr9F2WkgWj4rVVdSndKAHZfl6ySg/42S+lrT2g7BjZA3SEgQzUEav5rXomT63mD228ZXe9P2TcdRHZiIqz+DNSZxai+KVEmaSSKyHrrhH4pyblDLQ31ex1gTTa9OzfW6h357vTvunxNOa57tTlyauGW2mFE/9fEvRtLZzXX+rnu1W+n9/no3Vi/06vhute6FnmRzI9g2q089ju5C73dFGu918CGY8RNv6wSKi73aL8xeBr0Z2KpN4x5NLdmoUopkaKNaSQy25Yktfjm9BevXKgBblFjnjSSvleLxrI+rbMoy+G1X+KCDT2IxHlHGSNOdbqD1nmNoFF++JrkuuMNjKCD57CdhqGVhO91+AnVniG4dWFuULeLTf/8Z1pUVPSGiR82C2Wl/4Mxvor1xitaNmDXVscmJX8N6fEx6G8wvWPt/DayZXrBKkjY9ucKUKIsjb+u9ZVVt7+Etu+5FzTr+36cejW7NaoiWEA4iv/jhCSqhbvSK9ofFkVk4i5gKG/n4+NPKrbYq/PwdmrvDe4Db0047jkpZ1VX79a3e0LPNYakJvOnLmEmXlXaFDpy/SjDV/lz73cOeGZqP18i5fddGWeqeqgf5UUHSNSiR+IGY/1J7GLt4h/l+xgp9oaefWs2slBrrNRbW6pELLjAfhD5orW06qaiSgcDuFrxSWw7if7tmE+is09N7fz5de6NeMnuzRWMyZxIGMbvvfnTju3IzfLuJmNxm3ai58rJPjb0hEL8U3p/BqQfRCfHNdzJmlBuPXj2mXaM0qgRTTsAYNbFuH7mxr3opj+XVl2iLctn51l0yrEmXyObDl7UGtSateH1g2gLRl+xt9af+UAZKWXTCyo9VupQ0v96KZQ6Lp0S7amrYCwYyXstf0K6fybv0aFenNu5T9Jf7J0Hb089jbrOZpKMVeQT13/Lo1/VK3DEiHx0VaDVJDqziWr8P5UAm3t3S6lUHJ4fRd3Pwrb8noztTDyWrDXMafWyE4SLQ1Z1ZU/eByu2XU4OOCmjBKG9vTvBColL9YR37MlTeqztiC9NEQeVXGXeWJbtJKDfBnj7uquH7W+4Pb2l121kPtqGfz2uJ4Rt8I9ozdp9z6OqhdPfHDSEu08bMMXGZ8n1to9/HzYh4lZ/Evrk2v9Cal3RVVt0yC4dBODSi0iryKtZ7Q5cbnK9zKPFScLzHG4steSVzg7vzlIiouMdDh3ILSx8uLX3jaBsk0VutICeVdTDlI52XS3k2mrPwriq33kc4N6Zs/j8p+3bDYBhRSOZDeSzqo2HsqxdGilXKZnrK6wUF0Ms/8vPS58V3pyA8gLSWUG0z+/0DHCHr5p902lzXd14UjO9Hv/5ymfi21XZhVW6EVqO75fs1Egq/Womp7+Ms/8b42osidHtzfIUL826FhDUXOl7XaQB8+ECV7FAlXNpZq2uA2osqu3K6i+2OkJ6W6iulxcnd8q1Y3sKNJzfw9YCso5m4hayx3wdZQYndoES4/eTyynCR3Hvzgagm3tyhtEdFCXuDdUXWpooZb8Zh+rmjg0R7ocP0iqOQfsLtGXk3ggPHRjvTVyI6k12HktrqQ7HVP/vBUF5f9mpaCuyd5bh97s5XrAVcsljtKsUmtypK6zVrXtT0q0PKvh7vHdrzRj/5NHGj2+J1tHR+9pfRfqGmhRan+E3P9x4eW1K4WZNbVHlatbOusPT4SDuxnj8RQUAVfMX+cu4qqKgnBjJeTMsTZFYbFNqSfxsRJK5ZmI5eD83e0gL9Qe7WoXWaaBKkCLCqfOsKZY8EJhZxMbs0UHSW2clM9J5TWsdJNpyc83P2bx2NldwFzAqeSdUM+HtrOaiuFn4zaMFq8RJpuv1ZKTxjNf7QD3duuHo3r3VTx946/NZz2vzWABrQOp9pVXVcvSy0IZrzc58Nj6ImujcQwT3fii2dMwxDJo2s4KZab1k1nC+/Z3PEEzpbh1TRTX4J/KbauV43G9nI8J4Zzorip+luZdTs4uLWXUMgzlIP2KuKGVpXfFVKraqCsi7ecoIV/8TtLY3GFW/RuEUYfDYmW/cPo2d7mRU9tMf5o+W50ZxE06ZlLgpnZs2dTZGQkBQUFUWxsLG3dutXu8j/88AO1aNFCLN+mTRtatWqVKzbTK9UJrkiv39mKIlQuJKZGP+7vL/WkibcrU7Ru5oPRYvZsrsApF/+ycbY15KFODejVO1qIwmHcN/3LM93oJScK8vFF6skeTcxqQTRxslKrZffTq1Zqish6L2d/p1u52I3q1oj0QIlmfL4QJb/QQ1TQrRro3tmleei1aZ7NM71voVZ1qqkygSZXw7XVTf2MCi0YWiI1f+c/MfXF8GmpmtSqIoImPVN9NNPSpUtpwoQJNHfuXBHIzJw5k+Lj4yk9PZ1q1y47ombz5s00dOhQSkxMpDvvvJMWL15MgwYNoh07dlDr1tod6693aldFVYOSTcLcJ/3uvfZbp2zNZTSiSyQ1qlWZoh2oi2GaiDm6uzqjk34a04U2Z+SKgMmWiJCKdOzsZbpdRrcDFwvU2q/t1wa2onkb/yVvwRchdirvss2/ay5E10BmJWe5fngyji5fLabKN0bh8EV3lYQfBo78DVsbMffe4LaiWKOjXbtqfmZ5st1lO06Qq9XXSLe7xwQzM2bMoFGjRtHIkSPFfQ5qVq5cSfPnz6eJEyeWWf6///0vDRgwgF566SVxf+rUqbRmzRqaNWuWeC2oo36NSrT4iViqXsn1mft6xMm9xuRJbn6XOwGnq381G38527JibFfafuSsIkP27eHaFZycKqVQmVbwr/7vt0urtKslHCise6GHqPZaUHitzPOmsTlXd+VgRK5xvZqWthIZAxm12AsqeF/VCGRs4UT4ouISScvy94Q7ghlvo2o3U1FREaWlpVHfvn1vrtDXV9xPTU21+hp+3HR5xi05tpYvLCyk/Px8sxs4XriM50nxFA/HNrQ5dJN/LTkjokYlkXPgSuN63aLItlvDv6T73xqu+pB9Hu20dkIPmvGA+k3aSrUCcVVfveLzabv2z81o5ttRsdQsrAotMkk0Nuggr4UrWXNislKk7s+q8V3JHXw8vLXdGap+c+Xm5lJxcTGFhZkX2+H7WVlZVl/Dj8tZnrujgoODS28REdobZgeuYVnUivNqLIen8nDH9S/2dGheEYObv8TjmtSkvyb1pw/vL1uJ9FmVcgWG3Zg7p3Nj+V1K5XUtKFlBWE2VXDAjtjtqmZhq36AG/fZ8D+qqwES1ctzWtPzqu/ZwJWuugeJqTWtLr2cj9buCW8eUHt59lxuOjbvofjRTQkIC5eXlld6OHTvm7k0CDbH8IuGWgUY6HpHDAZq1PIOKKs2xxN1nG1/uJXtosB6YdotYzhJvanhc2RY+JX06rL39OiIaHtXjTEC6dkJ3u0GBrYq7zgbBFcopfeDI/E62yKnSve7FHqS0TxxstTIdKacXqv7kCA0NJT8/P8rOzjZ7nO+Hh1tPNOTH5SwfGBgobgCgDvORboYyhfiy8wt1eei5HtCfCX3EyCKes8YWpSbjtFVw8I42deiKvXwVCRdvd117nBmVFVnT/T8qrE0IquSF/KX+zWlhavkz0r//n7ZiZKm7DY9rSDsyz1GfltrNAXRLy0xAQADFxMRQcnJy6WMlJSXiflxcnNXX8OOmyzNOALa1PAC4D7fYcFfeCpNqpXrC81654iKy7bW+tGtyP/J2zoQJxpYaZ2INy5YKywlBlQ4KpbzfF8M70AM3pkBxtyn3tBZlIewF917bzcTDsufNm0cLFy6kAwcO0JgxY6igoKB0dNPw4cNFV5HR+PHjKSkpiT788EM6ePAgvfnmm7R9+3YaN26c2psKADI1C6tKCx/rpKvRSe7AyeKmLTzvDGpTOpGoGpydAdxTOZJDYgyi+LOuhtoyphkB21TPbBsyZAidPn2aJk2aJJJ4o6OjRbBiTPLNzMwUI5yMunTpImrLvP766/Tqq6/SLbfcQj///DNqzIBDujSpSSnpp3H0NMjZobw8okwuR395z324Pf2y+xTVqBQgZpJ2dCizUd9WYbR/SrxiXVimZjwQ5dR8SXr3yoAWVmcBdxaP9OSSDPWq22/J41Fhf2dfLL3v7kl9I2tWoiNnLonuTE/m75IhpePG2WxZSUlJKfPY/fffL24Azhp5WyNxAerc2LlRE6C82EYh9HDnBnSLjJEhph7sGEHHz12mPw+foV3HzpOaBrSuI26c29I8vKqYOqPre+udek/TQEbOVAHlua+962Yul5uMK2dxR0sf2B6KbtrSYnAo4DWtJ8UjCD9el2H2/Dv3tqYODUMofuaG0scGt69H3/55VLSWfbbhMLlqklWj5U/fJvJgPL21ziXBDIA7kzzv10h/tCdQcjg1j8p6+0Z3iyP4Fy9PZ/HB6nTVgxnT0XAP3xiuXh7uevvr2HmqLmF0DH9Ov36sEz21KI0uFTne4mOLtdnM3T003lb13x+fiqMLV66JfCY1cLBjq+Bdw5DKdDArX1Jw83y/ZtSpUU2a+st+eqJbI9GFxcf57+wLZYLWpOe6i/9bC2bk5GxJaVl8oV8zMSzbqEblAOrT0rzciSfS/dBsANM6LABawV1Tj3aJpGVjukhavnuzWg5XYLY1jNmI5/v66tGOYhZmObOOu0OHyBAx67xa3h5Udlqcn8bE0Z1t69AHVmo42QvGuC7P6ue7ix9M1gJGWyoF+NHq57rT8qe7KF588xmV8rC0Di0z4DFubx1O84Z38KgqxlLdFVVH5AlEIxFXM/gX95t3lx366y5SAwTuKjl5/jK1rnd9ug6tKa9FydYcavZmfo9pGCJurspf4e8p7q4E5SCYAY/Bv5T6tfL85lRbc2vtfrM/VVapeB7YpuRnjnO8ONG4Y2QN2nbknFsO+zCTaUB4FnKeuPOl+OZu2RZPw5Nvnjh3mW5xeGSU9Ra4xU/E0hsr9lLifW3JW+GbD0Ai0x98WqyPWc2FE+3BTaFVlJuKgCcE3fnG9Xo07aaucfthfvWOliJHyNY8P2hdkIfzZxwPZOzPq5f8Qk/yZsiZAQDJqtkp+68Vr97RorSqqh5xwqa1KrTP9bWdC1GvRkXVWjsb1qxcJln3l2e6iuq5d3ngEPB37jVPStdhZX+vhJYZAJCMC4dN6NdMTGOgFZYl9Ud3b0LD4yJlJWTqAZcYsHde/vtgtP05nhTE+TRazalxVvyt4aIGUKtJq929KSADghkAkOVZHYyW8LRAhgVVsN+Qfk90PZdti55JaWhxppih6ezX3DWNlh3XQDcTAIAKMyY7w6QoOvneKKg3qF091Qqr6YFepszgiVm5S/CNO1uVnjulIDCyDS0zAODRTKvrVi0nSfrPV/tQ89eTZL1/XRUmquTt5BmMi0sMFFrlepceT/7HQ++dLRDI9W+m/XqQPhnanvRg88TelJ1/RVfJxs/1bebuTfA6CGYAwKNxdV0uFld0rYRCKtsfeSRntmCep2fN/mwa1b0xqTWDsZpTM+hF3eoVxQ3MDYttoHhrnZ4hmAEAj9e7hfL1h3ieHtO5ekBePomj3DwLg2ZHXXk7BDMAoGuuGsEDzuMWFp6gkWdMV3JyTXXpZTu9G4IZANC1Bzs2oINZF6j7LZ49K7CnmNAf1YRBeQhmABwYScAz74I28Ll4F03uIFF9lQoMgnshmAGQkUj6fN9mVFB0TcyFBOBqGJrr/ESUNasE0spnuzpVSwa0B2cTQIbxdkrKA4C2vDKgBZ3Ku0xfpx41e/zWuvqsXozsHdvQVg4AAB5pTM8m1LM5cqm8AYIZAAAA0DUEMwAAAKBrCGYAAEyMvlHR966oujguGtS/1fUCiLWqumbmdiRd6wMSgAEATLwc35zibw2jNvX0MbGht3m0S6QYTRjTsIYio5vAMyCYAQAw/VL086WYhiE4Jho+PwNah9t83oAJD7wSupkAAAB0oFFoZXdvgmYhmAEAANCBmQ9G091RdWn5013cvSmag24mAAAAHagTXJE+HtrO3ZuhSWiZAQDQCVSAdb1KFfzcsFaQC8EMAACAhSn33EpREdVpXO+mODY6gG4mAADwGH6+yvxGHx4XKW6gD2iZAQAAj/FgxwhqEV6VxvZq4u5NARdCywwAAHiMyoH+lPRcd3dvBrgYWmYAAABA19AyAwAAHiu2cU3y8/URXU/guRDMAACAx6oS6E/73oqnAD90RHgyBDMAAODRglArxuMhVAUAAABdQzADAKATtzUNFf/6+6IWMIApdDMBAOjE2F5NKaxaEPVoVsvdmwKgKQhmAAB0lPvxcOeG7t4MAM1BNxMAAADoGoIZAAAA0DXVgpmzZ8/SsGHDqFq1alS9enV6/PHH6eLFi3Zf8/nnn1PPnj3Fa3x8fOj8+fNqbR4AAAB4CNWCGQ5k9u3bR2vWrKFffvmFNmzYQKNHj7b7mkuXLtGAAQPo1VdfVWuzAAAAwMP4GAwGg9JveuDAAWrVqhVt27aNOnToIB5LSkqiO+64g44fP05169a1+/qUlBTq1asXnTt3TrTqyJGfn0/BwcGUl5cnWngAAABA+5y5fqvSMpOamiqCEGMgw/r27Uu+vr60ZcsWRddVWFgoDoDpDQAAALyHKsFMVlYW1a5d2+wxf39/CgkJEc8pKTExUURyxltERISi7w8AAAAeFMxMnDhRJObaux08eJBcKSEhQTRJGW/Hjh1z6foBAABAR0XzXnjhBXr00UftLtO4cWMKDw+nnJwcs8evXbsmRjjxc0oKDAwUNwAAAPBOsoKZWrVqiVt54uLixLDqtLQ0iomJEY+tW7eOSkpKKDY21vGtBQAAAHBFzkzLli3FEOtRo0bR1q1b6Y8//qBx48bRgw8+WDqS6cSJE9SiRQvxvBHn0+zatYsyMjLE/T179oj73KIDAAAA4NI6M99++60IVvr06SOGZHft2lUUxTO6evUqpaeni9oyRnPnzqV27dqJIIh1795d3P+///s/tTYTAAAAdE6VOjPuhDozAAAA3nX99rhZs42xGerNAAAA6Ifxuu1IG4vHBTMXLlwQ/6LeDAAAgD6v49xC49XdTDxi6uTJk1S1alVR90bpqJGDJK5l44lTJXj6/nnDPmL/9A/nUN88/fypuY8cjnAgwwOFeMYAr26Z4QNQv359VdfBJ89TP6TesH/esI/YP/3DOdQ3Tz9/au2j3BYZ1UczAQAAALgCghkAAADQNQQzMvC0CZMnT/bY6RM8ff+8YR+xf/qHc6hvnn7+tLqPHpcADAAAAN4FLTMAAACgawhmAAAAQNcQzAAAAICuIZgBAAAAXUMwAwAAALqGYEai2bNnU2RkJAUFBVFsbCxt3bqV3C0xMZE6duwopm6oXbs2DRo0iNLT082W6dmzp5jWwfT21FNPmS2TmZlJAwcOpEqVKon3eemll+jatWtmy6SkpFD79u3FULymTZvSggULXHKM3nzzzTLb36JFi9Lnr1y5QmPHjqWaNWtSlSpVaPDgwZSdna2b/eP3s9w/vvE+6fX8bdiwge666y5Rkpy39+effzZ7ngdQTpo0ierUqUMVK1akvn370j///GO2zNmzZ2nYsGGiumj16tXp8ccfp4sXL5ots3v3burWrZvYXi6t/v7775fZlh9++EF8XniZNm3a0KpVq2Rvi5z9u3r1Kr3yyitiXZUrVxbLDB8+XEyxUt55nzZtmub3jz366KNltn3AgAG6OX9S9tHa3yTfpk+frotzmCjh2qCl704p21IuHpoN9i1ZssQQEBBgmD9/vmHfvn2GUaNGGapXr27Izs5266GLj483fPXVV4a9e/cadu3aZbjjjjsMDRo0MFy8eLF0mR49eojtPXXqVOktLy+v9Plr164ZWrdubejbt69h586dhlWrVhlCQ0MNCQkJpcscPnzYUKlSJcOECRMM+/fvN3zyyScGPz8/Q1JSkurHaPLkyYZbb73VbPtPnz5d+vxTTz1liIiIMCQnJxu2b99u6Ny5s6FLly662b+cnByzfVuzZg2XSjCsX79et+ePt+G1114zLFu2TOzL8uXLzZ6fNm2aITg42PDzzz8b/vrrL8Pdd99taNSokeHy5culywwYMMAQFRVl+PPPPw0bN240NG3a1DB06NDS5/kYhIWFGYYNGyY+/999952hYsWKhs8++6x0mT/++EPs5/vvvy/2+/XXXzdUqFDBsGfPHlnbImf/zp8/L87F0qVLDQcPHjSkpqYaOnXqZIiJiTF7j4YNGxqmTJlidl5N/261un9sxIgR4vyYbvvZs2fNltHy+ZOyj6b7xjf+u/Dx8TEcOnRIF+cwXsK1QUvfneVtixQIZiTgL6OxY8eW3i8uLjbUrVvXkJiYaNASvjDyH+bvv/9e+hhfDMePH2/zNfwB9fX1NWRlZZU+NmfOHEO1atUMhYWF4v7LL78sAgpTQ4YMEX8wah8jDmb4S9EavnDwH/4PP/xQ+tiBAwfEMeCLiB72zxKfqyZNmhhKSko84vxZXih4v8LDww3Tp083O4+BgYHiy57xlyK/btu2baXL/Prrr+JicuLECXH/008/NdSoUaN0H9krr7xiaN68een9Bx54wDBw4ECz7YmNjTU8+eSTkrdF7v5Zs3XrVrHc0aNHzS6EH330kc3XaHn/OJi55557bL5GT+fP1j5a4v3t3bu32WN6OYfWrg1a+u6Usi1SoJupHEVFRZSWliaa9kwns+T7qamppCV5eXni35CQELPHv/32WwoNDaXWrVtTQkICXbp0qfQ53gdu2gwLCyt9LD4+XsyKum/fvtJlTPffuIxx/9U+Rtykys3BjRs3Fk3X3PTJeJ3crG+6Xm6ubdCgQel69bB/RryeRYsW0WOPPWY247vez5+pf//9l7KysszWxRPLcdOz6TnjrokOHTqULsPL8zZt2bKldJnu3btTQECA2T5xU/q5c+ck7beUbVHq75LPJ++TKe6S4Gb1du3aie4L0+Z7re8fdy1wt0Pz5s1pzJgxdObMGbNt96Tzx90dK1euFF1llvRyDvMsrg1a+u6Usi1eOWu20nJzc6m4uNjshDK+f/DgQdKKkpISeu655+i2224TFz2jhx56iBo2bCiCAe6/5f58/mNatmyZeJ7/UKztm/E5e8vwh/ry5cviD1OtY8R/tNwHy1+ap06dorfeekv0Qe/du1dsF39RWF4keL3lbbtW9s8U99ufP39e5CR4yvmzZNwma+sy3V6+UJry9/cXX8SmyzRq1KjMexifq1Gjhs39Nn2P8rbFWZwLwOds6NChZrMLP/vssyLPgPdp8+bNIkjlz/eMGTM0v3+cH3PfffeJ7Tt06BC9+uqrdPvtt4sLj5+fn0edP7Zw4UKRe8L7bEov57DEyrVBS9+dUrZFCgQzHoKTp/gCv2nTJrPHR48eXfp/jrI5iaxPnz7iS6hJkyakdfwladS2bVsR3PDF/fvvvxfJcJ7kyy+/FPvLgYunnD9vxr82H3jgAZHAOWfOHLPnJkyYYPa55i/zJ598UiRuamm+G2sefPBBs88kbz9/Frm1hj+bnmb+/PmiRZiTV/V4DsfauDZ4GnQzlYOb9/nXhmVmNd8PDw8nLRg3bhz98ssvtH79eqpfv77dZTkYYBkZGeJf3gdr+2Z8zt4y/EuTAwpXHiOO3ps1aya2n9+bmzG5NcPWevWyf0ePHqW1a9fSE0884dHnz/h+9tbF/+bk5Jg9z833PEJGifNq+nx52+JsIMPndc2aNWatMrbOK+/jkSNHdLF/prj7lz9Dpp9JvZ8/o40bN4qW0PL+LrV6DsfZuDZo6btTyrZIgWCmHBxtx8TEUHJyslmzHd+Pi4sjd+JffPxhXb58Oa1bt65Mk6Y1u3btEv/yL3zG+7Bnzx6zLx/jl2+rVq1KlzHdf+Myxv135THi4Z3cKsHbz+usUKGC2Xr5i4dzaozr1cv+ffXVV6JpnodBevL5488of0GZroubpDmXwvSc8Rcb96Ub8eebt8kYzPEyPLyWgwbTfeLuSG6+l7LfUrbFmUCGc704QOWcivLweeVcAmP3jJb3z9Lx48dFzozpZ1LP58+ytZT/NqKionR1Dg3lXBu09N0pZVskkZwq7MV4aBlnjy9YsEBk6o8ePVoMLTPN8naHMWPGiCF7KSkpZsMDL126JJ7PyMgQQwd5qNu///5rWLFihaFx48aG7t27lxl+179/fzGEj4fU1apVy+rwu5deeklkmc+ePdvq8Ds1jtELL7wg9o+3n4cx8jBBHh7I2fnGIX085HDdunViP+Pi4sRNL/tnzO7nfeCRDqb0ev4uXLgghnLyjb9iZsyYIf5vHM3DQ035vXl/du/eLUaKWBua3a5dO8OWLVsMmzZtMtxyyy1mQ3t5BAQPe33kkUfE8FPeft5Hy2Gv/v7+hg8++EDsN4+MszbstbxtkbN/RUVFYuhs/fr1xfkw/bs0jgDZvHmzGAXDz/NQ30WLFolzNnz4cM3vHz/34osvilEm/Jlcu3atoX379uL8XLlyRRfnr7x9NB1azdvEI3gsaf0cjinn2qC1787ytkUKBDMS8fh5Ptg8Xp6HmnH9BHfjP0JrN64vwDIzM8WFLyQkRHyYuNYDf+hM65SwI0eOGG6//XZRA4EDBQ4grl69arYM1z2Jjo4W+88XVOM61D5GPMyvTp064j3r1asn7vNF3oj/oJ9++mkxBJL/qO69917xR6uX/WOrV68W5y09Pd3scb2eP16Xtc8lD+k1Djd94403xBc971efPn3K7PuZM2fExa9KlSpiKOjIkSPFBcgU19zo2rWreA/+bPCXvqXvv//e0KxZM7FPPIR05cqVZs9L2RY5+8cXeFt/l8baQWlpaWL4LV9sgoKCDC1btjS8++67ZsGAVvePL4Z8ceOLGl90eXgy1w2xDHq1fP7K20cjDjr4b4qDEktaP4dUzrVBa9+dUralPD43dhwAAABAl5AzAwAAALqGYAYAAAB0DcEMAAAA6BqCGQAAANA1BDMAAACgawhmAAAAQNcQzAAAAICuIZgBAAAAXUMwAwAAALqGYAYAAAB0DcEMAAAAkJ79PzHvSrCwVcDeAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "g = torch.Generator().manual_seed(seed)\n", "nn = BengioFFN(e_dims, n_hidden, context_size, words.nb_chars, g)\n", "lossi = nn.train(datasets, max_steps, mini_batch_size)\n", "train_loss = nn.training_loss(datasets)\n", "val_loss = nn.test_loss(datasets)\n", "print(f\"{train_loss=}\")\n", "print(f\"{val_loss=}\")\n", "plt.plot(lossi);" ] }, { "cell_type": "markdown", "id": "4686039c", "metadata": {}, "source": [ "## Génération" ] }, { "cell_type": "code", "execution_count": 116, "id": "9a0cf856", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "aves\n", "des\n", "saien\n", "impôt\n", "récialir\n", "empêchercise\n", "ubularieu\n", "artenue\n", "soire\n", "assume\n", "unitorigrogré\n", "momentes\n", "judividarf\n", "soument\n", "lohes\n", "facirié\n", "alte\n", "jours\n", "océ\n", "déritestiture\n" ] } ], "source": [ "g = torch.Generator().manual_seed(2147483647 + 10)\n", "for _ in range(20):\n", " word = nn.generate_word(words.itoc, g)\n", " print(word)" ] }, { "cell_type": "markdown", "id": "59d1850e", "metadata": {}, "source": [ "## Synthèse des scores de loss\n", "\n", "*Origine*\n", "\n", "```\n", "train_loss=1.4996627569198608\n", "val_loss=1.7031011581420898\n", "```\n", "\n", "*Initialisation correcte de la couche de sortie*\n", "\n", "```\n", "train_loss=1.4785451889038086\n", "val_loss=1.6728813648223877\n", "```\n", "\n", "*Initialisation correcte de la couche cachée pour tanh*\n", "\n", "```\n", "train_loss=1.4660104513168335\n", "val_loss=1.657399296760559\n", "```\n", "\n", "*Initialisation correcte de la couche cachée méthode Kaiming*\n", "\n", "```\n", "train_loss=1.4627671241760254\n", "val_loss=1.657903790473938\n", "```\n", "\n", "*BatchNorm*\n", "\n", "```\n", "train_loss=1.5353056192398071\n", "val_loss=1.6674079895019531\n", "```" ] }, { "cell_type": "markdown", "id": "a38d8cae", "metadata": {}, "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.14.2" } }, "nbformat": 4, "nbformat_minor": 5 }