{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "56710956-7461-4d84-86b1-bfd3ae4924e2",
   "metadata": {},
   "source": [
    "# 9. Wavenet (CNN)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f238647-e5ac-440c-a180-95cc9a57898a",
   "metadata": {},
   "source": [
    "- Bengio et al. 2003 MLP LM <https://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf>\n",
    "- WaveNet 2016 from DeepMind <https://arxiv.org/abs/1609.03499>\n",
    "- <https://deepmind.google/discover/blog/wavenet-a-generative-model-for-raw-audio/>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02800f69",
   "metadata": {},
   "source": [
    "L'objectif est d'améliorer le modèle de langue au niveau des caractères en s'éloignant d'un simple perceptron multicouche (MLP) qui écrase immédiatement tous les caractères d'entrée en une seule couche cachée. Nous allons mettre en oeuvre une architecture hiérarchique inspirée de l'article **WaveNet** (2016). Le but est de fusionner progressivement les informations (deux caractères à la fois) afin de traiter plus efficacement les contextes plus longs."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "202308d9-8529-4826-aeeb-469a34ffb16c",
   "metadata": {},
   "source": [
    "# Reprise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "caddd508",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class Words(object):\n",
    "    \"\"\"Représente une liste de mots, ainsi que la liste ordonnée des caractères les composants.\"\"\"\n",
    "\n",
    "    EOS = '.'\n",
    "\n",
    "    def __init__(self, filename):\n",
    "        self.filename = filename\n",
    "        self.words = open(self.filename, 'r').read().splitlines()\n",
    "        self.nb_words = len(self.words)\n",
    "        self.chars = sorted(list(set(''.join(self.words))))\n",
    "        self.nb_chars = len(self.chars) + 1  # On ajoute 1 pour EOS\n",
    "        self.ctoi = {c:i+1 for i,c in enumerate(self.chars)}\n",
    "        self.ctoi[self.EOS] = 0\n",
    "        self.itoc = {i:s for s,i in self.ctoi.items()}\n",
    "\n",
    "    def __repr__(self):\n",
    "        l = []\n",
    "        l.append(\"<Words\")\n",
    "        l.append(f'  filename=\"{self.filename}\"')\n",
    "        l.append(f'  nb_words=\"{self.nb_words}\"')\n",
    "        l.append(f'  nb_chars=\"{self.nb_chars}\"/>')\n",
    "        return '\\n'.join(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ed358a8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Datasets:\n",
    "    \"\"\"Construits les jeu de données d'entraînement, de test et de validation.\n",
    "    \n",
    "    Prend en paramètres une liste de mots et la taille du contexte pour la prédiction.\n",
    "    \"\"\"\n",
    "\n",
    "    def _build_dataset(self, lwords:list, context_size:int):\n",
    "        X, Y = [], []\n",
    "        for w in lwords:\n",
    "            context = [0] * context_size\n",
    "            for ch in w + self.words.EOS:\n",
    "                ix = self.words.ctoi[ch]\n",
    "                X.append(context)\n",
    "                Y.append(ix)\n",
    "                context = context[1:] + [ix] # crop and append\n",
    "        X = torch.tensor(X)\n",
    "        Y = torch.tensor(Y)\n",
    "        return X, Y\n",
    "    \n",
    "    def __init__(self, words:Words, context_size:int, seed:int=42):\n",
    "        # 80%, 10%, 10%\n",
    "        self.shuffled_words = words.words.copy()\n",
    "        random.shuffle(self.shuffled_words)\n",
    "        self.n1 = int(0.8*len(self.shuffled_words))\n",
    "        self.n2 = int(0.9*len(self.shuffled_words))\n",
    "        self.words = words\n",
    "        self.Xtr, self.Ytr = self._build_dataset(self.shuffled_words[:self.n1], context_size)\n",
    "        self.Xdev, self.Ydev = self._build_dataset(self.shuffled_words[self.n1:self.n2], context_size)\n",
    "        self.Xte, self.Yte = self._build_dataset(self.shuffled_words[self.n2:], context_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7b458ba3-1ae8-4b9b-99aa-68daa355f14b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Embedding:\n",
    "  \n",
    "    def __init__(self, num_embeddings, embedding_dim):\n",
    "        self.weight = torch.randn((num_embeddings, embedding_dim))\n",
    "    \n",
    "    def __call__(self, IX):\n",
    "        self.out = self.weight[IX]\n",
    "        return self.out\n",
    "  \n",
    "    def parameters(self):\n",
    "        return [self.weight]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "129c38bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FlattenConsecutive:\n",
    "    \"\"\"Plus ou moins <https://pytorch.org/docs/stable/generated/torch.nn.Flatten.html>\"\"\"\n",
    "    def __call__(self, x):\n",
    "        self.out = x.view(x.shape[0], -1)\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cfb9e6dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear:\n",
    "    \"\"\"Linear layer.\n",
    "    \n",
    "    Similar to <https://pytorch.org/docs/stable/generated/torch.nn.Linear.html>\n",
    "    \"\"\"\n",
    "    def __init__(self, fan_in, fan_out, bias=True):\n",
    "        self.weight = torch.randn((fan_in, fan_out)) / fan_in**0.5\n",
    "        self.bias = torch.zeros(fan_out) if bias else None\n",
    "  \n",
    "    def __call__(self, x):\n",
    "        self.out = x @ self.weight\n",
    "        if self.bias is not None:\n",
    "            self.out += self.bias\n",
    "        return self.out\n",
    "  \n",
    "    def parameters(self):\n",
    "        return [self.weight] + ([] if self.bias is None else [self.bias])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c6ff29bc-59c6-4416-a3c7-ade2097b03b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchNorm1d:\n",
    "    \"\"\"Batch normalization layer.\n",
    "    \n",
    "    Similar to <https://pytorch.org/docs/stable/generated/torch.nn.BatchNorm1d.html#torch.nn.BatchNorm1d>\n",
    "    \"\"\"\n",
    "  \n",
    "    def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "        self.eps = eps\n",
    "        self.momentum = momentum\n",
    "        self.training = True\n",
    "        # parameters (trained with backprop)\n",
    "        self.gamma = torch.ones(dim)\n",
    "        self.beta = torch.zeros(dim)\n",
    "        # buffers (trained with a running 'momentum update')\n",
    "        self.running_mean = torch.zeros(dim)\n",
    "        self.running_var = torch.ones(dim)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        # calculate the forward pass\n",
    "        if self.training:\n",
    "            xmean = x.mean(0, keepdim=True) # batch mean\n",
    "            xvar = x.var(0, keepdim=True) # batch variance\n",
    "        else:\n",
    "            xmean = self.running_mean\n",
    "            xvar = self.running_var\n",
    "        xhat = (x - xmean) / torch.sqrt(xvar + self.eps) # normalize to unit variance\n",
    "        self.out = self.gamma * xhat + self.beta\n",
    "        # update the buffers\n",
    "        if self.training:\n",
    "            with torch.no_grad():\n",
    "                self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * xmean\n",
    "                self.running_var = (1 - self.momentum) * self.running_var + self.momentum * xvar\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.gamma, self.beta]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "924700fb-fd48-4dff-9842-119eaf83a6b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tanh:\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        self.out = torch.tanh(x)\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d9ce689e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sequential:\n",
    "    \"\"\"<https://pytorch.org/docs/stable/generated/torch.nn.Sequential.html>\"\"\"\n",
    "    def __init__(self, layers):\n",
    "        self.layers = layers\n",
    "  \n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "        self.out = x\n",
    "        return self.out\n",
    "  \n",
    "    def parameters(self):\n",
    "        # get parameters of all layers and stretch them out into one list\n",
    "        return [p for layer in self.layers for p in layer.parameters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3fa9e7df",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BengioFFNModel:\n",
    "    \n",
    "    def __init__(self, e_dims, n_hidden, context_size, nb_chars, seed):\n",
    "        torch.manual_seed(seed);\n",
    "        self.nb_chars = nb_chars\n",
    "        self.e_dims = e_dims\n",
    "        self.n_hidden = n_hidden\n",
    "        self.context_size = context_size\n",
    "        self.steps = 0\n",
    "        self.create_model()\n",
    "\n",
    "    def _create_model(self):\n",
    "        self.model = Sequential([\n",
    "            Embedding(self.nb_chars, self.e_dims),\n",
    "            FlattenConsecutive(),\n",
    "            Linear(self.e_dims * self.context_size, self.n_hidden, bias=False), BatchNorm1d(self.n_hidden), Tanh(),\n",
    "            Linear(self.n_hidden, self.nb_chars), BatchNorm1d(self.nb_chars),\n",
    "        ])\n",
    "        with torch.no_grad():\n",
    "            self.model.layers[-1].gamma *= 0.1\n",
    "    \n",
    "    def create_model(self):\n",
    "        self._create_model()\n",
    "        self.parameters = self.model.parameters()\n",
    "        for p in self.parameters:\n",
    "            p.requires_grad = True\n",
    "        self.nb_parameters = sum(p.nelement() for p in self.parameters)\n",
    "\n",
    "    def forward(self, X, Y):\n",
    "        logits = self.model(X)\n",
    "        self.loss = F.cross_entropy(logits, Y) # loss function\n",
    "\n",
    "    def backward(self):\n",
    "        for p in self.parameters:\n",
    "            p.grad = None\n",
    "        self.loss.backward()\n",
    "\n",
    "    def train(self, datasets: Datasets, max_steps, mini_batch_size):\n",
    "        lossi = []\n",
    "        for i in range(max_steps):\n",
    "            # minibatch construct\n",
    "            ix = torch.randint(0, datasets.Xtr.shape[0], (mini_batch_size,))\n",
    "            Xb, Yb = datasets.Xtr[ix], datasets.Ytr[ix]\n",
    "            \n",
    "            # forward pass\n",
    "            self.forward(Xb, Yb)\n",
    "        \n",
    "            # backward pass\n",
    "            self.backward()\n",
    "\n",
    "            # update\n",
    "            lr = 0.2 if i < 100000 else 0.02 # step learning rate decay\n",
    "            self.update_grad(lr)\n",
    "        \n",
    "            # track stats\n",
    "            if i % 10000 == 0:\n",
    "                print(f\"{i:7d}/{max_steps:7d}: {self.loss.item():.4f}\")\n",
    "            lossi.append(self.loss.log10().item())\n",
    "        self.steps += max_steps\n",
    "        return lossi\n",
    "\n",
    "    def update_grad(self, lr):\n",
    "        for p in self.parameters:\n",
    "            p.data += -lr * p.grad\n",
    "\n",
    "    @torch.no_grad() # this decorator disables gradient tracking\n",
    "    def compute_loss(self, X, Y):\n",
    "        logits = self.model(X)\n",
    "        loss = F.cross_entropy(logits, Y)\n",
    "        return loss\n",
    "\n",
    "    @torch.no_grad() # this decorator disables gradient tracking\n",
    "    def training_loss(self, datasets:Datasets):\n",
    "        loss = self.compute_loss(datasets.Xtr, datasets.Ytr)\n",
    "        return loss.item()\n",
    "\n",
    "    @torch.no_grad() # this decorator disables gradient tracking\n",
    "    def test_loss(self, datasets:Datasets):\n",
    "        loss = self.compute_loss(datasets.Xte, datasets.Yte)\n",
    "        return loss.item()\n",
    "\n",
    "    @torch.no_grad() # this decorator disables gradient tracking\n",
    "    def dev_loss(self, datasets:Datasets):\n",
    "        loss = self.compute_loss(datasets.Xdev, datasets.Xdev)\n",
    "        return loss.item()\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def generate_word(self, itoc, g):\n",
    "        for layer in self.model.layers:\n",
    "            layer.training = False\n",
    "        out = []\n",
    "        context = [0] * self.context_size\n",
    "        while True:\n",
    "            logits = self.model(torch.tensor([context]))\n",
    "            probs = F.softmax(logits, dim=1)\n",
    "            # Sample from the probability distribution\n",
    "            ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n",
    "            # Shift the context window\n",
    "            context = context[1:] + [ix]\n",
    "            # Store the generated character\n",
    "            if ix != 0:\n",
    "                out.append(ix)\n",
    "            else:\n",
    "                # Stop when encounting '.'\n",
    "                break\n",
    "        return ''.join(itoc[i] for i in out)\n",
    "\n",
    "    def __repr__(self):\n",
    "        l = []\n",
    "        l.append(\"<BengioFFNModel\")\n",
    "        l.append(f'  nb_chars=\"{self.nb_chars}\"')\n",
    "        l.append(f'  e_dims=\"{self.e_dims}\"')\n",
    "        l.append(f'  n_hidden=\"{self.n_hidden}\"')\n",
    "        l.append(f'  context_size=\"{self.context_size}\"')\n",
    "        l.append(f'  nb_parameters=\"{self.nb_parameters}\"/>')\n",
    "        return '\\n'.join(l)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d09cebaf",
   "metadata": {},
   "source": [
    "## Entraînement et sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3b2afd87",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Words\n",
      "  filename=\"civil_mots.txt\"\n",
      "  nb_words=\"7223\"\n",
      "  nb_chars=\"41\"/>\n",
      "<__main__.Datasets object at 0x117c58d70>\n"
     ]
    }
   ],
   "source": [
    "words = Words('civil_mots.txt')\n",
    "print(words)\n",
    "context_size = 3\n",
    "datasets = Datasets(words, context_size)\n",
    "print(datasets)\n",
    "vocab_size = words.nb_chars\n",
    "e_dims = 10 # the dimensionality of the character embedding vectors\n",
    "n_hidden = 200 # the number of neurons in the hidden layer of the FFN\n",
    "seed = 2147483647\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ad5cab62-9b2a-43b2-8337-1e9e84994ebe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BengioFFNModel\n",
      "  nb_chars=\"41\"\n",
      "  e_dims=\"10\"\n",
      "  n_hidden=\"200\"\n",
      "  context_size=\"3\"\n",
      "  nb_parameters=\"15133\"/>\n"
     ]
    }
   ],
   "source": [
    "nn = BengioFFNModel(e_dims, n_hidden, context_size, words.nb_chars, seed)\n",
    "print(nn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "be3a9157",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0/ 200000: 3.7010\n",
      "  10000/ 200000: 1.8070\n",
      "  20000/ 200000: 1.9360\n",
      "  30000/ 200000: 2.0421\n",
      "  40000/ 200000: 1.8004\n",
      "  50000/ 200000: 1.5737\n",
      "  60000/ 200000: 1.7824\n",
      "  70000/ 200000: 1.4953\n",
      "  80000/ 200000: 1.6343\n",
      "  90000/ 200000: 1.7238\n",
      " 100000/ 200000: 1.5417\n",
      " 110000/ 200000: 1.7352\n",
      " 120000/ 200000: 1.2899\n",
      " 130000/ 200000: 1.6082\n",
      " 140000/ 200000: 1.9928\n",
      " 150000/ 200000: 1.5761\n",
      " 160000/ 200000: 2.6016\n",
      " 170000/ 200000: 1.5545\n",
      " 180000/ 200000: 1.5431\n",
      " 190000/ 200000: 1.5238\n",
      "train_loss=1.5285897254943848\n",
      "val_loss=1.6962659358978271\n"
     ]
    }
   ],
   "source": [
    "max_steps = 200000\n",
    "mini_batch_size = 32\n",
    "lossi = nn.train(datasets, max_steps, mini_batch_size)\n",
    "train_loss = nn.training_loss(datasets)\n",
    "val_loss = nn.test_loss(datasets)\n",
    "print(f\"{train_loss=}\")\n",
    "print(f\"{val_loss=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dba3471b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWm1JREFUeJzt3QlYlOXaB/CbHURWERBEATV3QUGIStPc66SmlpqFmalpZmqleTquLZqa+VmmHcsyNTVLrVxzXxEUc1dcETdWZZd9vuu+h5kzg6CgA7Pw/13XXLO98847DPL+fZ77eR4zhUKhIAAAAAAjZ67vAwAAAADQBYQaAAAAMAkINQAAAGASEGoAAADAJCDUAAAAgElAqAEAAACTgFADAAAAJgGhBgAAAEyCJVUTRUVFdOvWLXJwcCAzMzN9Hw4AAACUA88RnJGRQV5eXmRu/uC2mGoTajjQ+Pj46PswAAAA4BFcv36d6tat+8Btqk2o4RYa1Q/F0dFR34cDAAAA5ZCeni6NEqrz+INUm1Cj6nLiQINQAwAAYFzKUzqCQmEAAAAwCQg1AAAAYBIQagAAAMAkINQAAACASUCoAQAAAJOAUAMAAAAmAaEGAAAATAJCDQAAAJgEhBoAAAAwCQg1AAAAYBIQagAAAMAkINQAAACASag2C1pWlqOxd2jTqdvUxNOB+retp+/DAQAAqLbQUvOYLiRk0o8HY2nnuUTdfCMAAADwSBBqHpOlhXIp9PzCosfdFQAAADwGhJrHZG2h/BHmFyoed1cAAADwGBBqHpNVcajJQ0sNAACAXiHUPCar4u6nAoQaAAAAvUKoeUxWluh+AgAAMNpQs3DhQvL19SVbW1sKDQ2lqKiocr1u9erVZGZmRr1799Z6XKFQ0JQpU6hOnTpkZ2dHnTt3posXL2ptc+fOHRo0aBA5OjqSs7MzDR06lDIzM8lwampQKAwAAGBUoWbNmjU0fvx4mjp1Kh07dowCAgKoW7dulJj44CHNsbGx9MEHH1C7du3ue2727Nm0YMECWrx4MUVGRpK9vb3sMycnR70NB5ozZ87Q9u3baePGjbRv3z4aPnw46RtqagAAAIw01MybN4+GDRtGQ4YMoWbNmkkQqVGjBi1durTM1xQWFkoomT59Ovn7+9/XSjN//nz6z3/+Q7169aJWrVrRzz//TLdu3aINGzbINufOnaOtW7fS999/Ly1DzzzzDH399dfS8sPb6ROGdAMAABhhqMnLy6Po6GjpHlLvwNxc7kdERJT5uhkzZpC7u7t0GZV09epVio+P19qnk5OThBfVPvmau5yCg4PV2/D2/N7cslOa3NxcSk9P17pUavdTAYZ0AwAAGE2oSU5OllYXDw8Prcf5PgeT0hw4cIB++OEHWrJkSanPq173oH3yNYciTZaWluTq6lrm+86cOVPCkeri4+NDldn9hJoaAAAAEx79lJGRQa+//roEGjc3N6pKkyZNorS0NPXl+vXrlTqkG6EGAADAiBa05GBiYWFBCQkJWo/zfU9Pz/u2v3z5shQIv/jii+rHioqK1C0tMTEx6tfxPnj0k+Y+AwMD5TZvU7IQuaCgQEZElfa+zMbGRi6V7X8tNeh+AgAAMJqWGmtrawoKCqKdO3dqhRS+HxYWdt/2TZo0oVOnTtHx48fVl549e1LHjh3lNncJ+fn5STDR3CfXv3CtjGqffJ2amir1PCq7du2S9+baG32yVs9TgyHdAAAARtNSw3g49+DBg6VoNyQkREYuZWVlyWgoFh4eTt7e3lLTwvPYtGjRQuv1XPDLNB8fO3Ysffrpp9SoUSMJOZMnTyYvLy/1fDZNmzal7t27y6grHm2Vn59Po0ePpgEDBsh2+qRqqSkoUlBRkYLMzZXdUQAAAGDgoaZ///6UlJQkk+VxkS53EfFwa1Whb1xcnIxKqogJEyZIMOJ5Z7hFhods8z45FKmsXLlSgkynTp1k/3379pW5bfRNNaSb5RcVkY25hV6PBwAAoLoyU/BEMdUAd2nxKCguGuZZiXUlJ7+QmkzeKrdPT+9GNW0qnBMBAABAB+dvrP2ko+4nll+AuhoAAAB9Qah5TBbmZqQqo0GxMAAAgP4g1OhyWHdRtejJAwAAMEgINTpdKgHdTwAAAPqCUKMDVpirBgAAQO8QanTAsrioJg8T8AEAAOgNQo0OYKkEAAAA/UOo0QEslQAAAKB/CDW6XKkbhcIAAAB6g1CjAxjSDQAAoH8INboMNWipAQAA0BuEGl3OU4PRTwAAAHqDUKMDVpYY0g0AAKBvCDU6YGmuaqnBMgkAAAD6glCj03lqsEwCAACAviDU6IB1cfcTQg0AAID+INToAGYUBgAA0D+EGh1A9xMAAID+IdToAOapAQAA0D+EGh2wVi2TgEJhAAAAvUGo0QHL4tFPeRjSDQAAoDcINTqAmhoAAAD9Q6jRAXQ/AQAA6B9CjQ6gpQYAAED/EGp0wMoSyyQAAADoG0KNDqClBgAAQP8QanQANTUAAAD6h1CjyyHdBVilGwAAQF8QanQA3U8AAAD6h1CjA1aYURgAAEDvEGp0wLq4+wnLJAAAABhZqFm4cCH5+vqSra0thYaGUlRUVJnbrlu3joKDg8nZ2Zns7e0pMDCQli9frrWNmZlZqZc5c+aot+H3K/n8rFmzyLC6n1BTAwAAoC+WFX3BmjVraPz48bR48WIJNPPnz6du3bpRTEwMubu737e9q6srffzxx9SkSROytramjRs30pAhQ2Rbfh27ffu21mu2bNlCQ4cOpb59+2o9PmPGDBo2bJj6voODAxnWPDVF+j4UAACAaqvCoWbevHkSLDiYMA43mzZtoqVLl9JHH3103/YdOnTQuv/ee+/RsmXL6MCBA+pQ4+npqbXNH3/8QR07diR/f3+txznElNzWEKCmBgAAwMi6n/Ly8ig6Opo6d+78vx2Ym8v9iIiIh75eoVDQzp07pVWnffv2pW6TkJAgIYlbakri7qZatWpR69atpWuqoKCgzPfKzc2l9PR0rUtlQfcTAACAkbXUJCcnU2FhIXl4eGg9zvfPnz9f5uvS0tLI29tbgoaFhQV9++231KVLl1K35VYcbpHp06eP1uNjxoyhNm3aSHfWoUOHaNKkSdJtxS1HpZk5cyZNnz6dqoIq1OQVoPsJAADAaLqfHgWHlOPHj1NmZqa01HBNDnctleyaYtyNNWjQIClC1sSvUWnVqpXU54wYMULCi42NzX374dCj+RpuqfHx8aHKgO4nAAAAIws1bm5u0tLCXUSa+P6Dal24i6phw4Zym0c/nTt3TsJIyVCzf/9+6ZriYuSH4SJl7n6KjY2lxo0b3/c8B53Swk5lwJBuAAAAI6up4daRoKAgaW1RKSoqkvthYWHl3g+/hruiSvrhhx9k/wEBAQ/dB7f8cFgqbcRVVUNNDQAAgBF2P3GXzuDBg2XumZCQEBnSnZWVpR4NFR4eLvUz3BLD+Jq3bdCggQSZzZs3yzw1ixYt0tovdw+tXbuWvvzyy/vek4uQIyMjZUQUd2Xx/XHjxtFrr71GLi4upG8Y0g0AAGCEoaZ///6UlJREU6ZMofj4eOlO2rp1q7p4OC4uTlpQVDjwjBo1im7cuEF2dnYyX82KFStkP5pWr14to6MGDhx433tyNxI/P23aNAlGfn5+Emo0a2b0CTU1AAAA+mem4CRRDXBLkJOTk4zEcnR01Om+U7PzKHDGdrl9+fPnycLcTKf7BwAAqK7SK3D+xtpPOmBZPKSbYVZhAAAA/UCo0WH3E8vDUgkAAAB6gVCjA1YaNUT5mIAPAABALxBqdPFDNDcjy+I6GqzUDQAAoB8INTqfqwZLJQAAAOgDQo2OYFg3AACAfiHU6Ii1paqlplqMkAcAADA4CDU6YllcLIzuJwAAAP1AqNERK0tloTCGdAMAAOgHQo2uC4UxpBsAAEAvEGp0xFo9+gk1NQAAAPqAUKMjGNINAACgXwg1OoIh3QAAAPqFUKPzlhp0PwEAAOgDQo2OoPsJAABAvxBqdNz9hCHdAAAA+oFQoyNoqQEAANAvhBodsVItk4B5agAAAPQCoUZHME8NAACAfiHU6HpId1GRrnYJAAAAFYBQo/NlEjCkGwAAQB8QanQEhcIAAAD6hVCjI5hRGAAAQL8QanTcUoN5agAAAPQDoUZH0P0EAACgXwg1OmKtnqcGhcIAAAD6gFCjIxjSDQAAoF8INTqCVboBAAD0C6FG5/PUYPI9AAAAfUCo0REM6QYAANAvhBodwZBuAAAAIww1CxcuJF9fX7K1taXQ0FCKiooqc9t169ZRcHAwOTs7k729PQUGBtLy5cu1tnnjjTfIzMxM69K9e3etbe7cuUODBg0iR0dH2dfQoUMpMzOTDAWGdAMAABhZqFmzZg2NHz+epk6dSseOHaOAgADq1q0bJSYmlrq9q6srffzxxxQREUEnT56kIUOGyGXbtm1a23GIuX37tvqyatUqrec50Jw5c4a2b99OGzdupH379tHw4cPJUKBQGAAAwMhCzbx582jYsGESTJo1a0aLFy+mGjVq0NKlS0vdvkOHDvTSSy9R06ZNqUGDBvTee+9Rq1at6MCBA1rb2djYkKenp/ri4uKifu7cuXO0detW+v7776Vl6JlnnqGvv/6aVq9eTbdu3SJDYG1ZvEp3IQqFAQAADD7U5OXlUXR0NHXu3Pl/OzA3l/vcEvMwCoWCdu7cSTExMdS+fXut5/bs2UPu7u7UuHFjGjlyJKWkpKif431zlxN3Y6nwe/J7R0ZGlvpeubm5lJ6ernWpTGipAQAA0C/LimycnJxMhYWF5OHhofU43z9//nyZr0tLSyNvb28JGhYWFvTtt99Sly5dtLqe+vTpQ35+fnT58mX697//TT169JAww9vHx8dL4NE6cEtL6dri50ozc+ZMmj59OlUV1NQAAAAYUah5VA4ODnT8+HEp7OWWGq7J8ff3l64pNmDAAPW2LVu2lO4p7qri1ptOnTo90ntOmjRJ3keFW2p8fHyosmBINwAAgBGFGjc3N2k5SUhI0Hqc73MdTFm4m6hhw4Zym0c/cY0Mt6SoQk1JHHj4vS5duiShhvddshC5oKBARkSV9b5co8OXqoLJ9wAAAIyopsba2pqCgoKktUWlqKhI7oeFhZV7P/wa7ooqy40bN6Smpk6dOnKf952amir1PCq7du2S/XDhsGHNU4MFLQEAAIyi+4m7dAYPHixFuyEhITR//nzKysqS0VAsPDxc6me4JYbxNW/L3UkcZDZv3izz1CxatEie5y4prn3p27evtLpwTc2ECROkZYeHijMeOcV1Nzzqikdb5efn0+jRo6XbysvLiwwBamoAAACMLNT079+fkpKSaMqUKVKky91JPNxaVTwcFxcn3U0qHHhGjRolrS92dnbUpEkTWrFiheyHcXcWz1+zbNkyaY3hkNK1a1f65JNPtLqPVq5cKUGGu6N4/xyCFixYQIbCxlL5mXPyC/V9KAAAANWSmYLHWVcDXCjs5OQkI7F4VmKd7z8nn1pN+1tun/+kO9laWej8PQAAAKqb9Aqcv7H2k4442FiSpblyAr672Xm62i0AAACUE0KNjvB6Vc41rOT23ax8Xe0WAAAAygmhRoeca1jLdSpaagAAAKocQo0OuRS31KTeQ0sNAABAVUOoqYSWGtTUAAAAVD2EmspoqclGSw0AAEBVQ6jRIRdVS00WRj8BAABUNYQaHXJSjX5CSw0AAECVQ6iphJYajH4CAACoegg1lVBTg0JhAACAqodQUxnz1GBINwAAQJVDqKmU7ieMfgIAAKhqCDWVMqQ7j4qKqsU6oQAAAAYDoaYSRj9xnsnIKdDlrgEAAOAhEGp0yMbSgmpYW8htFAsDAABULYSaypqAD4taAgAAVCmEGh1zxlIJAAAAeoFQU1kjoO5hqQQAAICqhFBTSS01d7MwrBsAAKAqIdToGJZKAAAA0A+EmkpbKgEtNQAAAFUJoaaSlkrA6CcAAICqhVCjYxj9BAAAoB8INTqG0U8AAAD6gVCjYxj9BAAAoB8INTqG0U8AAAD6gVBTSaEmK6+Q8gqKdL17AAAAKANCjY452FqSuZnydirWfwIAAKgyCDW6/oGam2kM68ZcNQAAAFUFoaYSJ+BLycqtjN0DAABAKRBqKoFbTRu5Ts7EopYAAAAGHWoWLlxIvr6+ZGtrS6GhoRQVFVXmtuvWraPg4GBydnYme3t7CgwMpOXLl6ufz8/Pp4kTJ1LLli3leS8vLwoPD6dbt25p7Yffz8zMTOsya9YsMkRuDspQk5KJlhoAAACDDTVr1qyh8ePH09SpU+nYsWMUEBBA3bp1o8TExFK3d3V1pY8//pgiIiLo5MmTNGTIELls27ZNns/Ozpb9TJ48Wa45BMXExFDPnj3v29eMGTPo9u3b6su7775Lhqi2uqUGoQYAAKCqWFb0BfPmzaNhw4ZJMGGLFy+mTZs20dKlS+mjjz66b/sOHTpo3X/vvfdo2bJldODAAQlDTk5OtH37dq1tvvnmGwoJCaG4uDiqV6+e+nEHBwfy9PQkQ1fLXlkonJyB7icAAACDbKnJy8uj6Oho6ty58/92YG4u97kl5mEUCgXt3LlTWmLat29f5nZpaWnSvcRdVpq4u6lWrVrUunVrmjNnDhUUFJS5j9zcXEpPT9e6VHX3E1pqAAAADLSlJjk5mQoLC8nDw0Prcb5//vz5B4YUb29vCRoWFhb07bffUpcuXUrdNicnR2psBg4cSI6OjurHx4wZQ23atJHurEOHDtGkSZOkC4pbjkozc+ZMmj59Oum3UBjdTwAAAAbb/fQouNvo+PHjlJmZKS01XJPj7+9/X9cUFw2/8sor0qKzaNEiref4NSqtWrUia2trGjFihIQXGxtliNDEoUfzNdxS4+PjQ1XBrWZx9xNGPwEAABhmqHFzc5OWloSEBK3H+f6Dal24i6phw4Zym0c/nTt3TsKIZqhRBZpr167Rrl27tFppSsOjrrj7KTY2lho3bnzf8xx0Sgs7VdlSk5SZKwGNu9IAAADAgGpquHUkKChIWltUioqK5H5YWFi598Ov4a6okoHm4sWLtGPHDqmbeRhu+eGw5O7uToamdnFNDa/9lJFbdt0PAAAA6LH7ibt0Bg8eLHPP8Ail+fPnU1ZWlno0FM8xw/Uz3BLD+Jq3bdCggQSZzZs3yzw1qu4lDjT9+vWT4dwbN26Ump34+Hh5jutnOEhxEXJkZCR17NhRurL4/rhx4+i1114jFxcXMjS2VhZU08aSMnMLKDkjlxxtlTMMAwAAgAGFmv79+1NSUhJNmTJFwgd3J23dulVdPMzDsLkFRYUDz6hRo+jGjRtkZ2dHTZo0oRUrVsh+2M2bN+nPP/+U27wvTbt375YuKu5GWr16NU2bNk2CkZ+fn4QazZoZQ8N1NRJqMvPIv7a+jwYAAMD0mSm46KMa4EJhnhOHR2I9rF5HF/otOkRHr92lbwe1oedb1qn09wMAAKju52+s/VRJMKwbAACgaiHUVBI3B9WswpirBgAAoCog1FT6sG4slQAAAFAVEGoqCbqfAAAAqhZCTSVBqAEAAKhaCDWVpLaqpgbrPwEAAFQJhJrKbqnJQE0NAABAVUCoqeRQcy+/kLKwVAIAAEClQ6ipJPY2lmRnZSG30QUFAABQ+RBqqmKuGtTVAAAAVDqEmqqYqwZ1NQAAAJUOoaYSYVg3AABA1UGoqUTuDsqWmvi0nMp8GwAAAECoqVx+bvZyfTU5C79sAAAAlQwtNZXIv7Yy1FxOyqzMtwEAAACEmsrl71ZT3VJTVKTALxwAAEAlQktNJarrYkdWFmaUW1BEN1PvVeZbAQAAVHsINZXI0sKcfGspu6CuoK4GAACgUiHUVFFdzRXU1QAAAFQqhJpK5l9bWVeDYmEAAIDKhVBTyfyLh3VfScKwbgAAgMqEUFPJGrgrW2oQagAAACoXQk0la1A8rDs+PYeycgsq++0AAACqLYSaSuZUw4pq2StX68bMwgAAAJUHoaYKYGZhAACAyodQU4UzC19GsTAAAEClQaipAg3cMVcNAABAZUOoqQKN3B3k+tzt9Kp4OwAAgGoJoaYKtKzrpF4qIT0nvyreEgAAoNpBqKkCbjVtyNvZjhQKotM30qriLQEAAKodhJoqEujjLNcnEGoAAAAMJ9QsXLiQfH19ydbWlkJDQykqKqrMbdetW0fBwcHk7OxM9vb2FBgYSMuXL9faRqFQ0JQpU6hOnTpkZ2dHnTt3posXL2ptc+fOHRo0aBA5OjrKvoYOHUqZmZlkLFoVd0GduJ6q70MBAAAwSRUONWvWrKHx48fT1KlT6dixYxQQEEDdunWjxMTEUrd3dXWljz/+mCIiIujkyZM0ZMgQuWzbtk29zezZs2nBggW0ePFiioyMlPDD+8zJyVFvw4HmzJkztH37dtq4cSPt27ePhg8fTsYioLil5uQNhBoAAIBKoaigkJAQxTvvvKO+X1hYqPDy8lLMnDmz3Pto3bq14j//+Y/cLioqUnh6eirmzJmjfj41NVVhY2OjWLVqldw/e/asgg/1yJEj6m22bNmiMDMzU9y8ebNc75mWlib74Gt9yMjJV/h+tFFRf+JGRUL6Pb0cAwAAgLGpyPm7Qi01eXl5FB0dLd1DKubm5nKfW2LKEaBo586dFBMTQ+3bt5fHrl69SvHx8Vr7dHJykm4t1T75mrucuBtLhbfn9+aWndLk5uZSenq61kWfatpYUqPixS1PXkexMAAAgK5VKNQkJydTYWEheXh4aD3O9zmYlCUtLY1q1qxJ1tbW9MILL9DXX39NXbp0kedUr3vQPvna3d1d63lLS0vp2irrfWfOnCnhSHXx8fEhfWtVV1UsjC4oAAAAoxz95ODgQMePH6cjR47QZ599JjU5e/bsqdT3nDRpkoQp1eX69etkKHU1GAEFAACge5YV2djNzY0sLCwoISFB63G+7+npWebruJuoYcOGcptHP507d05aUjp06KB+He+DRz9p7pO3ZbxNyULkgoICGRFV1vva2NjIxZAEaIyA4q44MzMzfR8SAABA9Wyp4e6joKAgqYtRKSoqkvthYWHl3g+/hmtemJ+fnwQTzX1y/QvXyqj2ydepqalSz6Oya9cu2Q/X3hiLJp6OZG1hTmn38ulaSra+DwcAAKD6ttQw7joaPHiwFO2GhITQ/PnzKSsrS4Zps/DwcPL29paWGMbXvG2DBg0kyGzevFnmqVm0aJE8z60VY8eOpU8//ZQaNWokIWfy5Mnk5eVFvXv3lm2aNm1K3bt3p2HDhsmw7/z8fBo9ejQNGDBAtjMW1pbm1NTLUVpquK7G10250CUAAADoIdT079+fkpKSZLI8LtLlLqKtW7eqC33j4uKku0mFA8+oUaPoxo0bMrFekyZNaMWKFbIflQkTJsh2PO8Mt8g888wzsk+e3E9l5cqVEmQ6deok++/bt6/MbWNsAus6KUPN9TTqFeit78MBAAAwGWY8rpuqAe7S4lFQXDTMsxLry+/RN+j9tScouL4L/TbyKb0dBwAAgKmdv7H2k55GQJ2+lUYFhUVV/fYAAAAmC6Gmivm72ZODjSXl5BfRhQTjWbsKAADA0CHUVPUP3NyMWqqGdmMSPgAAAJ1BqNHjzMJY3BIAAEB3EGr0INBH2VJzHGtAAQAA6AxCjR5bai4kZNC9vEJ9HAIAAIDJQajRgzpOtlTbwYYKixR05hZW7AYAANAFhBo94FmUW3oru6DO3k7XxyEAAACYHIQaPWlax0Guz95CqAEAANAFhBo9aVpHOSviObTUAAAA6ARCjZ40Kw41MQkZUlsDAAAAjwehRk/q17InOysLmVn4anKWvg4DAADAZCDU6ImFuRk19lTW1aALCgAA4PEh1OhRMy9lFxRGQAEAADw+hBo9QrEwAACA7iDU6FGz4mHd6H4CAAB4fAg1etTY05HMzIgS0nMpJTNXn4cCAABg9BBq9KimjSXVd60ht8/dztDnoQAAABg9hBoDqas5dRNrQAEAADwOhBo9C/VzleulB69Sek6+vg8HAADAaCHU6NnA0HrkX9uekjJy6cttMfo+HAAAAKOFUKNnNpYW9GmvFnL758PX6OSNVH0fEgAAgFFCqDEATzV0o16BXqRQEM3eitYaAACAR4FQYyBGd2wo10ev3cEClwAAAI8AocZANKhdk+ytlQtcXk7K1PfhAAAAGB2EGgNhbm6mXgvq1A0M7wYAAKgohBoD0sLbSa4xZw0AAEDFIdQYkJbFoebMLbTUAAAAVBRCjUGGmnQUCwMAAFQQQo0B8a9dk+ysLCg7r5CuJqNYGAAAoCIQagyIhWaxMNaCAgAAqPxQs3DhQvL19SVbW1sKDQ2lqKioMrddsmQJtWvXjlxcXOTSuXPn+7Y3MzMr9TJnzhz1Nvx+JZ+fNWsWmWoX1Omb6fo+FAAAANMONWvWrKHx48fT1KlT6dixYxQQEEDdunWjxMTEUrffs2cPDRw4kHbv3k0RERHk4+NDXbt2pZs3b6q3uX37ttZl6dKlElr69u2rta8ZM2Zobffuu++SqcEIKAAAgEdjplDw5Pzlxy0zbdu2pW+++UbuFxUVSVDhgPHRRx899PWFhYXSYsOvDw8PL3Wb3r17U0ZGBu3cuVOrpWbs2LFyeRTp6enk5OREaWlp5Oio7OIxRDHxGdRt/j6qaWNJJ6d2lflrAAAAqqv0Cpy/K9RSk5eXR9HR0dKFpN6Bubnc51aY8sjOzqb8/HxydXUt9fmEhATatGkTDR069L7nuLupVq1a1Lp1a+maKigoKPN9cnNz5QeheTEGDWrbS6DJzC2gY3F39X04AAAARqNCoSY5OVlaWjw8PLQe5/vx8fHl2sfEiRPJy8tLKxhpWrZsGTk4OFCfPn20Hh8zZgytXr1aurFGjBhBn3/+OU2YMKHM95k5c6YkO9WFW5OMgaWFOXVtrvz5/nH8lr4PBwAAwGhU6egnbmnhYLJ+/XopMi4N19MMGjTovue5jqdDhw7UqlUrevvtt+nLL7+kr7/+WlpkSjNp0iRpqlJdrl+/TsaiZ4CXXG8+dZvyC4v0fTgAAACmF2rc3NzIwsJCuog08X1PT88Hvnbu3LkSav7++28JJqXZv38/xcTE0FtvvVWu2h7ufoqNjS31eRsbG+l707wYi6cbulEte2tKycqjg5eS9X04AAAAphdqrK2tKSgoSKuAlwuF+X5YWFiZr5s9ezZ98skntHXrVgoODi5zux9++EH2zyOqHub48eNSz+Pu7k6mxsrCnJ5vWUdu/3kCXVAAAADlYUkVxN1AgwcPlnASEhJC8+fPp6ysLBoyZIg8zyOavL29paaFffHFFzRlyhT65ZdfZASTqvamZs2aclHhQt61a9dKt1JJXIQcGRlJHTt2lHobvj9u3Dh67bXXZCSVKeoZ6EXLD1+jbafjKeelQrK1stD3IQEAAJhWqOnfvz8lJSVJUOGAEhgYKC0wquLhuLg4aUFRWbRokYya6tevn9Z+eJ6badOmqe9zrQ2PLuc5bUrrSuLneXuuofHz85NQwwHLVAXVcyFvZzu6mXqP9l1Ioq7NH9y9BwAAUN1VeJ4aY2Us89Ro+vf6U/RLZBwNa+dHH7/QTN+HAwAAYDrz1EDVt9aw6GuYrwYAAOBhEGoMWLCvi3odqJz8Qn0fDgAAgEFDqDFg9VxrkFtNa8orLKIzt9L0fTgAAAAGDaHGgPGinm2Ku6COxqILCgAA4EEQagxcUH3U1QAAAJQHQo2R1NXw4pbVZKAaAADAI0GoMXDNvZzI2sKckjPz6FpKtr4PBwAAwGAh1Bg4nkm4hbdyXD6GdgMAAJQNocaI6mr2X0zS96EAAAAYLIQaI6Ba3HLzqXhKzMiR27dS79HFhAw9HxkAAIDhQKgxAq3ruVCbes4yX82Kw3EUn5ZDLyzYT93/bz8dvpKi78MDAAAwCAg1RuLNZ/zkeuXhazRm9T90NzufCosU9N7qfyglM1ffhwcAAKB3CDVGontzT/JysqWUrDyKunqHalhbkG+tGpSQnkvvrz1BRUUY7g0AANUbQo2RsLQwp8FP+arvf9KrBS1+PYhsLM1pT0wSbT59W6/HBwAAoG8INUbk1dB61K6RG739bAPq08abmng6UnhYfXlu/4VkfR8eAACAXlnq9+2hIhxsrWj50FCtx570r0VL9l+lI7F38MMEAIBqDS01Ri64viuZmRFdSc6ipAwUDAMAQPWFUGPknGpYUWMPB7l9FK01AABQjSHUmIC2vq5yHYVQAwAA1RhCjQlo66cMNairAQCA6gyhxgSEFLfUnL2VThk5+fo+HAAAAL1AqDEBnk625ONqRzz/3rG4VH0fDgAAgF4g1JiItvWVrTW/R9+grNwCfR8OAABAlUOoMRGdm3nI9Z8nbtHTX+yiNUfi9H1IAAAAVQqhxkT0aOFJc18OID83e0rNzqdJ607RpcRMfR8WAABAlUGoMRFmZmbUL6gu7Rj/LHVq4i71NXO3xej7sAAAAKoMQo2JsTA3o4k9mpC5GdHWM/F0/DoKhwEAoHpAqDFBT3g4UJ82deX2pxvP0rG4u5SOod4AAGDiEGpM1NjOjcjawpyOXrtLfb49RG1mbKdd5xP0fVgAAACVBqHGRNV1qUGLXmtDTzesRW41ramgSEHLI67p+7AAAAAqjWXl7Rr0rVNTD7nwKKjO8/bS/ovJdDcrj1zsrfV9aAAAAIbRUrNw4ULy9fUlW1tbCg0NpaioqDK3XbJkCbVr145cXFzk0rlz5/u2f+ONN2T0juale/fuWtvcuXOHBg0aRI6OjuTs7ExDhw6lzEwMWS6Phu41qVkdR2mt4eJhAAAAU1ThULNmzRoaP348TZ06lY4dO0YBAQHUrVs3SkxMLHX7PXv20MCBA2n37t0UERFBPj4+1LVrV7p586bWdhxibt++rb6sWrVK63kONGfOnKHt27fTxo0bad++fTR8+PCKHn619WKAl1z/deKWvg8FAACgUpgpFApFRV7ALTNt27alb775Ru4XFRVJUHn33Xfpo48+eujrCwsLpcWGXx8eHq5uqUlNTaUNGzaU+ppz585Rs2bN6MiRIxQcHCyPbd26lZ5//nm6ceMGeXkpT9gPkp6eTk5OTpSWliatPdXN9TvZ1G72bjIzI4qc1IncHW31fUgAAAA6PX9XqKUmLy+PoqOjpQtJvQNzc7nPrTDlkZ2dTfn5+eTqqlyrSLNFx93dnRo3bkwjR46klJQU9XO8b+5yUgUaxu/J7x0ZGVnq++Tm5soPQvNSnfm41qA29ZyJI+zGk7f1fTgAAAA6V6FQk5ycLC0tHh7KdYZU+H58fPlqNSZOnCgtK5rBiLuefv75Z9q5cyd98cUXtHfvXurRo4e8F+N9c+DRZGlpKcGorPedOXOmJDvVhVuTqjtVF9SsLefpu72XqZCnHQYAADARVTqke9asWbR69Wpav369FBmrDBgwgHr27EktW7ak3r17S80MdzVx682jmjRpkjRVqS7Xr1+n6m5gSD3q3NSD8gqLaOaW8/TGj1GUW6AMjgAAANUq1Li5uZGFhQUlJGhP4sb3PT09H/jauXPnSqj5+++/qVWrVg/c1t/fX97r0qVLcp/3XbIQuaCgQEZElfW+NjY20vemeanubK0saEl4EM3q05JqWFvIEG9e+LKCZVUAAADGH2qsra0pKChIuolUuFCY74eFhZX5utmzZ9Mnn3wixb2adTFl4eJfrqmpU6eO3Od9cyEx1/Oo7Nq1S96bC5eh/Hi4/ICQevTd60GyTtS6Yzdp0d7L+BECAED1637i4dw898yyZctkVBIX9WZlZdGQIUPkeR7RxF0/KlwjM3nyZFq6dKnMbcM1MHxRzTHD1x9++CEdPnyYYmNjJSD16tWLGjZsKEPFWdOmTaXuZtiwYTLHzcGDB2n06NHSbVWekU9wv3aNatO0F5vJ7dlbYzDUGwAAqt+Mwv3796ekpCSaMmWKhJPAwEBpgVEVD8fFxcmoJJVFixbJqKl+/fpp7YfnuZk2bZp0Z508eVJCErfGcEjheWy4ZYe7kFRWrlwpQaZTp06y/759+9KCBQse79NXc6+H+dLlpCz66VAsjf/1OLnaW1NQfRc6dztdllmo7fC/nz8AAIDJzVNjrKr7PDVl4RFQY1b9Q5tO3SYbS3PiX4a8giKyNDejLs08aMjTfhTipz38HgAAwOjnqQHTw3U1X74SQE/6u1JuQZEEGkdbS1lSYcvpeHrluwia8NsJSruXr+9DBQAAeCC01IC4l1dIu2MS6QkPB2pQ257Ox2fQzxGxtPrIdZmwz8PRhpYPDZXnAQAADLGlBqEGHuho7B2a8NtJupKcRd7OdrT+nafI3QFLLAAAQNVA9xPoTLCvK/0+8inyc7Onm6n3aNiyo9KqAwAAYGhQUwMP5WJvTUvfaEvONazoxI00GrfmOBWVscQCB54l+67IApoAAABVCaEGyoVbav77ejBZW5jT1jPx9MXW86VuN3/HBfps8zka8tMRLMEAAABVCqEGyo2Hds/up1zi4rt9V2hVVJzW83ez8mj54Wty+1JiJi3YeVHr+YsJGbIsw9Q/TtO87RcoNjkLP30AANDf5HtQvfVu7U3XUrLpqx0XaPpfZ6hdIzeZqI/9ePAqZecVUi17a0rJyqPFe69QjxZ1qIW3kzz/0bpTFH3trnpfUVdTaPXwspfXAAAAqAi01ECFjenUkEL9XCknv4g+3XhOHkvPyZeZidmnvVvQC63qyMR+H6w9IXPfRF+7I4GGu6+GtfOT7Y7E3qW0bMx/AwAAuoFQAxXGi2LO6NVCJu7j+hruhpr0+ylKzymghu41qVtzT5rRs7ksu8Dz3Xy75xL9d98Vee1Lrb3p4xeayXYcevZfSpLH+XZOPkZVAQDAo0OogUfS2NOB3njKV25znQwvs8De7/IEmZubUa2aNjStZ3N57Jtdl+jvswlye1h7ZStNx8a15Xr3+STilTpGroimgOl/0+oSdToAAADlhVADj2xs50YyIR/r0Lg2/TIslHq0rKN+/sVWdahrMw9ZcoFnJe7UxJ0auitnJO7YxF2u915IpH0XkyX08DINXHfDtToFhUX4ZgAAoEJQKAyPzMHWijaNeYay8grV4aZkN9WnL7WgyKt3ZO2oEc82UD8XXN+VatpYUnJmnqwtxZp4Okh31Y8HY8ne2pI+6NYY3w4AAJQbWmrgsTjXsC410Kjwkgq/jwyjFUNDtVb7trY0l5FTLCE9l2ytzOnnoSE09+UAeYxrcOJSKj6BH3dlpWTmPtJnAQAA44ZQA5WOu5yeKQ4wmjo2VnZBsfAwXwlAfdt4S9jJKyyizzafrXCg4dmOgz7dQYevpOjk2AEAwHgg1IDedGhSW1poHGwsaUR7f3WX1eR/NZORVdvOJNChS8nl3t+fJ27RhuO35PYfxdcAAFB9INSA3nDLzIZ3nqY/331GRkupPOHhQK+F1pPbY1YfpxPXUx+6r8T0HJryxxn1/X0XlKOqAACg+kChMOhVE0/HUh8f36UxRcXepXO306n/fyPoxVZeUnDMk/z9OiJMgo9moBmz+h8pRm5ax5EuJ2XKiuKXk7JkPhwAAKge0FIDBsmphhWtfTtMhorzzMVro29Q3J1sSs3Op693XZJtuCXmp4NX6bkv99LhK3ek+Hh+/0CZ7ZjtvaCc2A8AAKoHhBowWDzk+/vwYJkPZ1BoPZrRSzmZ36aTt+j6nWxac+Q6TfvrLGXmFlCgjzOtG/mUTArYvlFtdRcUAABUH+h+AoNmaWFOYzs/ob6/41yihJVPN52lAxeVRcTvPteQxnVWzmTM2j9Rmz7bfE5GQPHSC7ZWFmXun1cKr+NsSzaWZW8DAADGAS01YFTeLh4lxSOjeNK/tr4uEnpUgYY94VGTPB1tZYbiqKt3ytzX1tPx1GHuHvpw7ckKHUNyZi4dupyMQmQAAAODUANGJaxBLWrp7SS3a1hbyGR9PPxbEw8Lb/+Ecl6cPTGld0HxAppz/45RDwWPic8o1/tzt9cLC/bTq0si6ds9lx/z0wAAgC4h1IBR4cAy6fkmMovxzD4tqX4t+1K361A8sd/umMRSn+cFOC8lZqrvL9ytLD6+l1dI8Wk5pb4mKSOXXv8hUmZAZhyKULcDAGA4EGrA6DzVwI0OfvQc9Qr0LnMbnpXYysKMriZnyRBvNmndSeoyby/9Fn2DFuy8KI/9q5VyAc6NJ2/JCuHt5+ymZ+fslqHkmkPG/7vvMvVddIhiU7KprosdvRjgJYt08lDyg5eSpeUHAAD0C4XCYLKLbYb61aIDl5Jp57kEyi8solVR1+W5D9YqF9B0srOiz/u0lGJiLkDmFcJVVkXF0YxeLejsrXR6efEhqd9h7g42so4VFxfzEHOeGHDQ95HkVtOGJv+r6QODliYejs6tTgAAoDtoqQGT1ampsguKA8uKw9fkNk/Gx8sysOHt/cnR1ore6dhQ/Zon/V3VyyzkFhRKiw4HmsYeDvT5Sy1px/vPkq+bvYyW4uHmA9r6SDji4uFJ605RVm7BA4+JR1txi0/HuXvK7OYCAIBHY6aoJnPJp6enk5OTE6WlpZGjY+mz2IJp4aLedrN3SyGxjaU5ZecV0i/DQiWgnLmVTs80dFOPmuLWHB4+zo8988Uuup2WQx90fYK+3H5Bupm2j2tPjTRmMdaUV1BEXb7aS9dSsunLlwOob1Dd+7ZJzc6jv88m0IzieXVYpybu9P3gYLTYAADo6PyNlhowWT6uNWR4N9e7cKBpUNuewvxryTpTPJeN5jDwTk096NknaksA6tNG2YU0929loOnc1L3MQMN4JuN+bZRBhut1NEVeSaGuX+2lwBnbacJvJyXQBPg4S73PzvOJMvIKAAB0A6EGTBqHFZXXn6xfrlaRvsUBReXtZxs89DV9guoS7zriSoq0ELGtp2/T60uj6EKCslDZ381eJgn8/e0weve5RvLY1D/P0NjV/9DIFdESgEriIuVFey5TWnZ+OT4tAED1hkJhMGldmnlIKOA5bTh4lId/7ZoUVN+Foq/dpeD6LhTsq6yzeRAeYv5Ug1p08FIKLT14VWYxXrz3srT0dG3mQbP6tiJXe2v19iM7NKAtp+NllNWG48rWGh6ptXVse/U2BYVFNGx5tBQjH429g64qAIDKaKlZuHAh+fr6kq2tLYWGhlJUVFSZ2y5ZsoTatWtHLi4ucuncubPW9vn5+TRx4kRq2bIl2dvbk5eXF4WHh9OtW9rN8vx+/L9szcusWbMe5fChGmlTz0XqXH4Y3FaKgstrQrfG1KaeM015sVm5X9OvODT9eDBWghQHmldD69Gi14K0Ag2zsjCnJeFB9GG3xvTv55uQtYU5nY/PkNFWKt/tuyKBhj1uV1VCeg59uPYEHS/eHwCAKapwqFmzZg2NHz+epk6dSseOHaOAgADq1q0bJSaWPsnZnj17aODAgbR7926KiIggHx8f6tq1K928eVOez87Olv1MnjxZrtetW0cxMTHUs2fP+/Y1Y8YMun37tvry7rvvPspnhmqGC3d5JuKKCPWvRetGPU2t6jqX+zXdm9dRh5dWdZ1o8WtB9FnvFvfNeKxS16WGjLwa3r6BeqTWhuPKfxfn49Np/o4LcjukuKVo+l9nKSVTOfFfSUUPmSfn292XZKXzN386QrdS75X7MwEAmPToJ26Zadu2LX3zzTdyv6ioSIIKB4yPPvrooa8vLCyUFht+PbfIlObIkSMUEhJC165do3r16qlbasaOHSuXR4HRT1AVrqVk0d3sfAqo61ShUU3bzsTTiOXR5OFoQzvGP0svL46QlhsuUv52UBD1/OaA3OeRW6M6NpAi6N3nE+lo7F26kpwpsxx7OdlKQfOI9v70VEPlMhGMC6WfnLlTZkRmvKL5ryPCpMCZcb0Or2XVtblnmQGsrLWz+CN2a+5ZoZ8RAIBBjH7Ky8uj6Oho6UJS78DcXO5zK0x5cMsMdzm5upZdp8AHzicEZ2ft/yVzd1OtWrWodevWNGfOHCooKHtOkNzcXPlBaF4AKhsv28ChoaIT63VoXFvmu+Fw0m+RMtC41bSWuXE4fPAaVzy/TkxCBr23+jj1+fYQfb3rkhQmq5ZtuJWWQ3svJNFbPx+V7iaVyKspEmgcbC3J0dZSuqBmbDwjEwDypIThP0bRyJXHaMn+K+U+3uhrd+jtFdESxFQtSgAARlUonJycLC0tHh7/G1HC+P758+fLtQ+un+G6Gc1gpCknJ0e24S4rzUQ2ZswYatOmjYShQ4cO0aRJk6QLat68eaXuZ+bMmTR9+vSKfDwAveHJ/HjJhpWRcRJcuMbmu9eDyN3RVp5v4e1E+yZ0pOWHr9HPEbGUm19E7RvXpvaN3KR1xsvJjq7fzaZPN56lEzfSaNaW8/RV/0B57caTt+X6+RZ1qFsLD3rzp6O04nAc+dayp7R7+eq6ne/3X6E3nvKVIufSurcSMnKojpOdhKEvtioXA2Xzd1yUcDS+S+MKtfQAAOi1+4mLd729vSVUhIWFqR+fMGEC7d27lyIjIx/4em5pmT17ttTZtGrV6r7nuQWnb9++dOPGDdnmQc1MS5cupREjRlBmZibZ2NiU2lLDFxVuqeFuMky+B4aKRzj1W6xs8ZzdrxW9EuxT5rYcMjTn2VE5eSOVei08KEXKv70dJq1GIZ/vpDtZebR8aAi1a1Rb1rH6fLPyPyG8Cy7HqWljKXPofNKrOfUM9Kb/bDhNtpbmUvNjbmYmS0tExd6R4e7dmnvQ8OXR0oL05tN+MsqL+daqQW8+46cOPlzHxMtVAABUVfdThVpq3NzcyMLCghISErQe5/ueng/uV587d66Emh07dpQZaF555RWpo9m1a9dDD5xre7j7KTY2lho3bnzf8xx0Sgs7AIaKh5F/1KOJdEM9KNCw0gIN48Lm/sE+tPrIdZrw+0mpd+FAU8veWiYeZMPa+dOt1Bz66VCsBBqebDCgrrPMmcMjrn49eoNO3UyTbdf9c1Naje7lK9e++v3YDVr/j3KCwcFh9eV469eqQV9sPS+LfU7544zWMPd1o54ij+LWJsbdYDvOJVCPFp7kXEN7RFhpeBV0Lr7mlirGwWtPTCI93cCNXEqMKAMAqFCosba2pqCgINq5cyf17t1bXSjM90ePHl3m67h15rPPPqNt27ZRcHBwmYHm4sWLMkqK62Ye5vjx41LP4+6uHDUCYOy4Dqc8E/09zAfdGkvh8ZWkLBlazrq38JRlIFTvM/lfzaSriCcKnN6zOVmam8s6Vzfu3pOLKkhwqLhXVEghfq4Slqb9dYYycgqkvmdUB+WaWQND6lHPAC9ac+Q6bT51mwoVCrp+5x7dTL1Hb/x4hNaMeJJqWlvS2ujr9Nmmc5SeUyBFzv8Nv/9vQclWp/ClUWRnZUG7PniWPB1tacyqf2jX+USytTKX4DemUyNZTBQA4JFGP/GQ7sGDB9N3330nI5Tmz59Pv/76q9TUcG0Nj2jiLiquaWFffPEFTZkyhX755Rd6+umn1fupWbOmXDjQ9OvXT4Zzb9y4Uateh+tnOEhxETJ3bXXs2JEcHBzk/rhx46hHjx60bNmych03Rj9BdcJh5dej12nTqduUlJ5Lv74dRk3rPLj1c+HuSzRnW4wUKK9860lq7OlAx+Lu0u3UHAlFHIIuJWZKYTCHGB4tVZa4lGzqs+iQLPTpXMOKsnMLKa+wSP08NzTt/bCjjOJSSczIoWPXUmXEFwcwDjCquXn4/fjCRdCaeHLEtW+HYf0sABOWXoHup0da0JKHY/Poo/j4eAoMDKQFCxZIdxDr0KGDDL/+6aef5D7f5i6lkniem2nTpkn3kZ+fX6nvw602vD8OPKNGjZLgxHUyvP3rr78u8+WUt4sJoQaqI/7nXd6RWFzsu/boDWrXyE0rbDyq0zfTqP93EbLKOeO6nfc6NaI9FxJl5mUeej7p+abyHIen4T8fpeTMPFnOgmdcbj97NxUU8fHz5yDpQkvJypPWLD7GIT8dkcVEVw9/kp4s7loDANNT6aHGGCHUAOinxYi7oeq62En3EbfA7DibIC0uXDt0eFIn2njyFn284bQEFBXVMhVP+rvKshW/RMbJ47yPne8/S/Y2lvTx+lMyWowXIl32Zgi+XgAThVW6AcAgcIsPt6Lw7Mmqmp6OTdzJx9VOhpN3+WovffjbSQk0nZt6yJByxoGGDX3Gnz7s2lgCEPv4haYSaNiI9g2U3VgXkqRVCAAAq3QDQJXi2pzwJ5XhhYuSueiX18DieXn+/XxTmY1ZNUS8UxN3GeW0atiTtPi1NjKXj0q9WjXoxQAvua0aVg4A1RtW6QaAKjcgxIcOXk4mZzsrGa3FLTmqwLNwUBuavTVGtlENXW/m5SiXkri+5o/jt2TUFXdz8TByAKi+UFMDAEbt1SWH6dDlFBrdsaEEJAAwLaipAYBqg0dLsdVH4rSKjQGg+kFNDQAYtc7NPMjdwUaGg/OkgwBQfSHUAIBRs7Iwl1mNGS/4CQDVF0INABg9DjVcZBx19Q6dj09/pH1s+OemLPZZTabuAjBJCDUAYPQ8nWxl9XD2w/6rFX49j5wa/+txWb389M1HC0UAoH8INQBgEniiPsZDvHkdKRVueeHZhz9ce4KKeFnyUvwcoVyxnPFQcwAwTgg1AGASeGmFNvWcZeHMnw/9r7aGF8Xk5RTWRt+g3TGJ973uXl4hrY66rr7Pw8OheruanEXj1xynK0mZej2OnPxC2n42Qa6hfBBqAMBkDGunbK1ZEXmNsvMKJLDM2nJe/fzSg/d3TW04flOWbHAoXn7hyNU7MjT8aOwdajl1m6xersInuUNoyTFKmbkFdDvtXrm2nfrnGVr3z01auFu/M1V/u+cyDfv5KI1aeazMVkbQhlADACaja3NPqudag1Kz82nMqn/o001n6XZaDnk42kghMa8OrllIzF1TPxYHnTGdGslK4PfyC+nEjVT6ascFysgtoC//jpFVxC8lZlKvhQfp1SWRdPASuqiMzbBlR6nDnD10+SGtL+dup9O+C0lym793feJ1zdiu84n0jUa4hrIh1ACAyeDgoppVeMe5ROl2YpP/1Yy6N/eU2z8eiFVvzyetCwmZVMPagl5p60NPNqilLjbmAMT4P8gf/HqC3lp2hDJyCuSxL7aeV4+S4iLj8k76xyepXecTynye9/nGj1HUff4+yspVvtejyC0olPfh5SN0NZqL9/n3mXhKy86/77m4lGz6ds8lupCQQYYgKSOXJm84rQ4lKZm5FHElhXILimjjidsPfO33GoXm3A2VnJlbrvcsKCyiqX+cppEronUyCSR//5oLtXLIVoUcQ5SSmUtv/nSErt/J1utxINQAgEnpGeBFW95rpx4N1a6RG73Qsg69+YxyEc31x2/KH2C26aRysj4OPLwS+FPFoWZr8SR+HRrXllaeK8lZFJuSLWtLcQA6eSONtpyOp6+2X6CnZ+2S1cYf1noTn5Yjf/Tf/OlomauKR169Q3tikuh8fIZ0i1UU117wiTX40x3yPtxt8dfJB5/EyyM1O49e/z6Khi+Pps83n9N6jk9iL393SNbr6vrVPhrw3wg6c6v0z8fdd7wCe+SVlMcKbQ9SWKSgd1cdkzmLpv91Vh7jof4qf5+Nf+B39OcJ5c/d0VbZHXmseMX4hwWacb+eoGUR1+T34sClssPHnaw8afV7GA5k/Fn4d46nLOBsygXv/Jghdu298eMRaVF6b/U/ep0WAaEGAExO0zqO9N3rwXTk4870w+C2ZGZmRm3qucgK4Py/6F8i46RGYctp5Qm/ewtlK85TDdy09vNep0b0Rd9WcpvDzPeDg+mt4rodHgL+fzsvyu1rKdk06PtI6jh3D7Wato2emrmT/jpxS2tffLJUnZDm/h2jfpxrf1T4uFRWHI6r0MmBg0f4D1FyYuUWJTsrC3l83t8xlF/46C0HN+5mU99FhygqVhkM9lxIVB8Xt2KEL42ihPRccqtpLS1lh6/coaE/HZU6JRX+Wb//6wl67su9sq/+/z1Mo385dt978c/i+PXUxzpxf7v7khwDO3E9VUbCHb7yv+LvM7fS5TOV5sdDVym/UEEhfq70QvGK8BzCHoQ/2wdrT2h939vP3l+Qzril69k5u6nrV3tp/8UHt7pEFn+GUD9XmvKvZuRSw0pWtd/+gFD2OBLSc+iVxRH0e/QNrZCs+g9AWXib4T8fpVM308jV3prmvhwg/970BaEGAExWbQcbsrZU/pnjP7RvPuMnt38+fI2OXrsr9Tb21hbU/ona8rhvrRrk5WQrt3kkVet6LtShsTtteOdp2ja2vYSlYe385ASTk19E/Lf7Py80pfCw+nKbuyvScwroVloOvbvqH7lk5ChP7uuO/a/lhVtjuOB42p9nqPnUbTRzyzk5eWw9rTxhcTjg2g4+watw0fOoldGybcmwczcrj/otjpDg4WBrSd+HB1PUx52kRohbmH7TOFHxaz/6/aSM7uEWhgfhY+Kwdjkpi+o42ZK1hbkEGP6cjIMJ3+bWhI3vtqP9EzqSn5s9xafn0PQ/z6jf7+MNp+j3Yzfkc/m42snPandMkrq+JSY+g9755Ri1+WQ79V54kBYUh8WK4tAwv/i1/HNgu84lqkOOTfHvwo6zCaW2oKyIUI6aG97OX0JweULNnL9jaMPxW2RpbkZvPKVsDdx5LkGrsJdbMriFi1u6OHDyUx/9fkoeL4uqdSnU35XsrC3o1VDlrNlLNbpPH0deQZFW8Fx79Lr8/ny2+Zy6+2zE8mgKm7WrzAktuduRQy2PGOR/R8uGhJB/7ZqkTwg1AFBt9GhRR7qTuOaCT+zsuaYeZFvcqsHBp1drbzn5cuGwSqCPM/m41pDbDrZW9PlLLamJpwN9M7CNtNzM6NWCdox/lla+FUp/j2svr+V98P/ex605IQGFu5Q4FKhaAN5YeoR+OhQr3Qrf7b0iJxAejs6tSb0CvdStNZqtCJtPxcu2miOyGO+HuzQ8HW3pt7efkvWw+Djf6dhQnv+/HRfVw4K562z1kesyuue/+6/IYzwqiAuiOVSptuNrHnnDrVB1Xexo3ainqE19Z3mOT2L8fhwW+GS+fGiITIDo5Wwn/1M3NyPZ/ycbz8pJb1XUdXlsfv9A2j/hOerUxF32s/JwnJzYw5dG0qaTtyUosl+i4soMXHwiHbH8qAQfPm4+Me8+n0iDvj8soYFbeV5q7a0eCcdD+WOKa32GFofav0sJNd/tvUxZeYXUwtuROjV1p2BfV+XP62aa1BOVhls1Fu1RjpDizz3p+SZyck/MyJWWCz6WHw5cpfazd9N/9yl/1hx8ONhxLdasLdpdeRxc+efOF1WgDfFTdomGh/nKz5qDx6kbaRJkubWHu/K4q4qD1K9HrtOlxIfXNfHPrOc3B6RFkY+D7b+YrA53O84lyPtzDQ8HHP6eSuJWxW7z90n44lbBJYODqWVdJ9I3ZZQFAKgGuNWGTw5ztsVInQzrUdz1pPJh18b09rMNpMamLD1a1pGLpga1a8qFje/iILU8g5ZEygkiNkX5Xh2b1KbJLzSTlgIuWrW1MqcuzTwl/HDLEeP/kTd0d5CWnY0nb9HkfzWVx1UnTzb37wvyP+LnW9aRFoF1/yhbYj7q0YQaezqot+N9fb//irQccdE0n9S5eFhl/vaL5O9Wk2b8dUa2YXxS5gDHYYO7O7i25KchbamOkx2F+btJkOGiWz5xM27l0vzfOc8XNLx9A1q897Kc0FW4G+/FAGVYe+3J+lLIvTb6ugQGbv2pX6sGfdU/kN5adlRCJ59kOxaHH01T/zwtrTzbziRI8axmoxWHxkFP1qMJ3ZpIC9K87RfULS0cQvu39ZFh0ly7xN11zjWs5bnE9BxaFqFsAXm/S2MJt9xqxy1dKVl5UgMVVF8ZclQ4WExad0puv/tcQ+rd2ltuP9u4toRP/t75+1tSXHjMLViTejSREXpdm3nQq99HSmh1qWEtNTPr/7kpXWf2NpY04tkGEnB5oVY+DubhaEv/alVHWoW4VYu7i/h3qCQrCzOa2L2JfNeqbiAOf19sOU9t6rvQy0E+9M7KYxKy2R/Hb0rQ0hzpxaFX8/ef53r6+IWm6vDPn+vf65WfPcy/lny39YqPU9/QUgMA1QqfQFTdEBwquBhYk7m52QMDTXm19XWlCd2VI7FUhaEvta4rLRqf9G4hRclrRzxFCwYEUu/ilhmeK4dP/Nz1xV1dfNLilo7PNp2Tbgs+Mau6OLimh7tvjsTeoet37lFNG0vqVjzCS4VPQu88p2ytWXqA60WKaGNx4TB3J/GJ8+0V0RJouDWGu5G4tYJPeBxoOCT8NzxYQhZ7qqGy1eDw5RR1DQkXZpc0rksjaW3iny2fYLeObUcvB/uon2/fqLYMvefPpBqhNr1nc+nyUbVSaXaZqURfuyOBhlvBQnxd1YGGT/yDQuvRzvefpakvNpfumqZ1HNRdiexJ/1pUv5a9/Ay5BeXj9aelVYMDDYdcbiXin7vq90HqsOqX3QX1/YEr8vPr3NSDxnV+Qv0432c/R1xTBxquidk+rr0EGuXP0Y0Gh9WX21/vukRPzdolx8A/ew6L3MLFuLZHsz5F1X0adydbfjf4Z+hf216+t5beTtLKxzVBn25SdnWpupEmbzgjYWjKH2co5LMddECjqJ1DLre28Oucayh/77kFSBV+OdRyyw6HNMZdUR+uVbZycnDi1klDCTQMLTUAUK1wMWOfNt7SJdKxsTvVsK68P4NvPu0nTfjc6sBBiVtq2CvBPnJRmdW3lbSOcDeX6ng+f6mFjJbi7iK+MA5JHAguJmbIkPNJv5+SFg72fEtPOZmX1LdNXfry7wvSzTB3W4xcc9HzqmFPShcE1wBxlwvXQ/DP5uztdLqblU9FCgU94eEgIUwloK6zdDVw6wVflC1NypO4JhtLC/q/Aa3L/LlwcOQQMrN4YkQefca1S4xbEn48GCsz6XJBL4cDDj9jOjWkudsuFG9TV35m3FrBAcetps1978FhgLvh+PXsSX9lSwu31HErw6ZTt+Wi6f2uylYazVYnPg6uYzl1M11aJbj1i1uxthWPkBv9XEP5PCr8O8V3VfUqw9v7q8OIJg5f3LXEi6ieuJEmXYfjuz5B647dUNcAhforQ6RKq7rO9GG3xtIFx1MQcAjTPF6FQkErIuMkFPFx88STzzVxl0DCPydnOyv53vgl3LrCLU281pkqWPL3cP1utvxuFSoU8nn5Z8Bz5HDI5ODE3aQ8lxO3RP77+aZan90QmCmqyZK06enp5OTkRGlpaeTo6KjvwwEAPeITDk+6x8GC60AqE4++mbLhjJxg+wXVrdBr+eTFdS1cE8ItE2tGPCknMR5GzcOn+eSismb4k/edBFW4G0az+Ja7Mb55tY3UTfBEc2887UuOtuVrnXr9h0h1/QXXBy18tQ09Cq7deHb2buIT0LZx7aW1QeX5/9sv4Ypb1FRdLKrb3Hq0+8MOWtuXhT8bt3SxY5O7SGhj3Lq1ZN8V2n4ugfiU7FvLnvoG1aVRHRpohQSuherxf/u19sktExymxv96QrqUdr3/7H2jfV75LkJaP/g7WzkslKwsyu4U4VMwt7h5O9eQUMrdcf9Zf1pah1aPeJLcHf4XKstr+9kE+b1hHPh4lBq38PEcTmuOXKf6rjXk91Hzu2RfD2wt3wdPXMm44LyBe00Z1cfZhcMq/85xq95fo58hl+KfpyGdvxFqAAAMGLcK8Kge/h+3qgaEcb2KqpuCC0/3ftCxzP81c43K01/sUndHLH6tDXVvoV0TVF48yR7PScO+ez3ovi6vilBN1KYqwlbhrrIZGp+tdk0bOhanLJzlk/O0ns3LtX/ubpvw20npapvQvUmp4ZbDkqpWpDQ8oSB3x206eUvqeLj7qlZNa2nNGN/lCa2CchWep4drokZ2aFBqK1JVmLn5HH1XXJzMrYR7P+yg9fvDVkXFqeuCWPR/OktNz2vfR8r1j2+0ld+plxcfoiOxd9UtXlwUXdel6rqcKhJq0P0EAGDAuFamT5v7W3j45M4FnDwXS782Pg/sBuCh7S8FetOao9el60nV1fMo2jWsTbMpRk6UJeuRKqpkmFHhrhUuXPV2saMxzzWSY+bV17kbbmyX+0NEWbiFhIuPy1Ke2inuguNL+0ZcJL1HXWDLegcqi4NLau7lJBd9+qBbY/onLlVGS73f9Yn7Ag3jQPqfDaelxqi5lyPVKg5gv418Sms7DoRcp8O1X4PDfA2uy0kTWmoAAIwUzyPDrQd9g7yla+BBrqVkSVdMr0BvaWF4HFwkzIFDNZdLdaHZOsa1Jr+XOPkbmpz8Qmk14u+prAnxVF1QI9r706TnlSPtDA26nx7zhwIAAFASd9/x3Cw8XPyzl1rQoFDlCCZjdikxg5YdukbjujyhrjkyNAg1j/lDAQAAKE1scpYMieapAXhEEVQ+1NQAAABUAl83e7mAYcLkewAAAGASEGoAAADAJCDUAAAAQPUNNQsXLiRfX1+ytbWl0NBQiopSzthYmiVLllC7du3IxcVFLp07d75ve55RccqUKVSnTh2ys7OTbS5e1F56/s6dOzRo0CAp8nV2dqahQ4dSZqZyPRUAAACACoeaNWvW0Pjx42nq1Kl07NgxCggIoG7dulFiYmKp2+/Zs4cGDhxIu3fvpoiICPLx8aGuXbvSzZs31dvMnj2bFixYQIsXL6bIyEiyt7eXfebkKFeNZRxozpw5Q9u3b6eNGzfSvn37aPjw4fgGAQAAQElRQSEhIYp33nlHfb+wsFDh5eWlmDlzZrleX1BQoHBwcFAsW7ZM7hcVFSk8PT0Vc+bMUW+TmpqqsLGxUaxatUrunz17lpejUBw5ckS9zZYtWxRmZmaKmzdvlut909LSZB98DQAAAMahIufvCrXU5OXlUXR0tHQPqZibm8t9boUpj+zsbMrPzydXV+WKqVevXqX4+HitffJ8MtytpdonX3OXU3BwsHob3p7fm1t2SpObmytj2zUvAAAAYLoqFGqSk5OpsLCQPDy0l5rn+xxMymPixInk5eWlDjGq1z1on3zt7q69VomlpaUEo7Led+bMmRKOVBfu9gIAAADTVaWjn2bNmkWrV6+m9evXS5FxZZo0aZLMHqy6XL9+vVLfDwAAAPSrQqt0u7m5kYWFBSUkJGg9zvc9PR+8/PzcuXMl1OzYsYNatWqlflz1Ot4Hj37S3GdgYKB6m5KFyAUFBTIiqqz3tbGxkQsAAABUDxVqqbG2tqagoCDauXOn+rGioiK5HxYWVubreHTTJ598Qlu3btWqi2F+fn4STDT3yfUvXCuj2idfp6amSj2Pyq5du+S9ufYGAAAAoEItNYyHcw8ePFjCSUhICM2fP5+ysrJoyJAh8nx4eDh5e3tLTQv74osvZA6aX375Rea2UdXA1KxZUy68HPrYsWPp008/pUaNGknImTx5stTd9O7dW7Zt2rQpde/enYYNGybDvrnQePTo0TRgwADZDgAAAKDCoaZ///6UlJQkQYUDCncRcQuMqtA3Li5ORiWpLFq0SEZN9evXT2s/PM/NtGnT5PaECRMkGPG8M9wi88wzz8g+NetuVq5cKUGmU6dOsv++ffvK3DYAAAAAzIzHdVeHHwUXC/OwcC4Y5lmJAQAAwPBxSQqPYOZGDx7NrNOWGmOVkZEh1xjaDQAAYJzn8YeFmmrTUsNFxbdu3SIHBwep46mMFGmqrUCm/vkYPqPxw3doGkz9ezT1z1cZn5FjCgcarqHVLG+p1i01/IOoW7dupb4Hf3mm+ktaHT4fw2c0fvgOTYOpf4+m/vl0/Rkf1kKjl8n3AAAAACoLQg0AAACYBIQaHeCZi3mIuqnOYGzqn4/hMxo/fIemwdS/R1P/fPr+jNWmUBgAAABMG1pqAAAAwCQg1AAAAIBJQKgBAAAAk4BQAwAAACYBoeYxLVy4UFYf58U3Q0NDKSoqiowVr6zetm1bmXXZ3d1dVkmPiYnR2qZDhw4yI7Pm5e233yZjwAuoljz2Jk2aqJ/Pycmhd955h2rVqiUryPOiqQkJCWRM+Hex5GfkC38uY/3+9u3bRy+++KLMJsrHu2HDBq3neawDL7Bbp04dsrOzo86dO9PFixe1trlz5w4NGjRIJgLjNeCGDh1KmZmZZOifLz8/nyZOnEgtW7Yke3t72SY8PFxmR3/Y9z5r1iwylu/wjTfeuO/4u3fvbjTfYXk+Y2n/LvkyZ84co/geZ5bj/FCev6G86PULL7xANWrUkP18+OGHVFBQoLPjRKh5DGvWrKHx48fL0LVjx45RQEAAdevWjRITE8kY7d27V34hDx8+TNu3b5c/qF27dpUV1DUNGzaMbt++rb7Mnj2bjEXz5s21jv3AgQPq58aNG0d//fUXrV27Vn4WfOLo06cPGZMjR45ofT7+HtnLL79stN8f//7xvy3+D0Rp+PgXLFhAixcvpsjISDn5879D/gOrwifDM2fOyM9j48aNcgIaPnw4Gfrny87Olr8tkydPlut169bJiaRnz573bTtjxgyt7/Xdd98lY/kOGYcYzeNftWqV1vOG/B2W5zNqfja+LF26VEILn/iN4XvcW47zw8P+hhYWFkqgycvLo0OHDtGyZcvop59+kv+U6AwP6YZHExISonjnnXfU9wsLCxVeXl6KmTNnmsSPNDExkYf7K/bu3at+7Nlnn1W89957CmM0depURUBAQKnPpaamKqysrBRr165VP3bu3Dn5/BEREQpjxd9VgwYNFEVFRUb//TH+PtavX6++z5/L09NTMWfOHK3v0sbGRrFq1Sq5f/bsWXndkSNH1Nts2bJFYWZmprh586bCkD9faaKiomS7a9euqR+rX7++4quvvlIYg9I+4+DBgxW9evUq8zXG9B2W93vkz/vcc89pPWZM32NiifNDef6Gbt68WWFubq6Ij49Xb7No0SKFo6OjIjc3VyfHhZaaR8RJMzo6Wpq6NdeX4vsRERFkCtLS0uTa1dVV6/GVK1eSm5sbtWjRgiZNmiT/mzQW3C3BzcP+/v7yPz9uCmX8XfL/PDS/T+6aqlevntF+n/w7umLFCnrzzTe1FnE15u+vpKtXr1J8fLzW98ZrxHBXsOp742vurggODlZvw9vzv1du2THGf5f8ffJn0sTdFNzs37p1a+nS0GWTflXYs2ePdEc0btyYRo4cSSkpKernTO075C6ZTZs2SRdaScbyPaaVOD+U528oX3NXqoeHh3obblXlBTC5FU4Xqs2ClrqWnJwsTWmaXw7j++fPnydTWNV87Nix9PTTT8vJT+XVV1+l+vXrSzA4efKk9Pdzczg3ixs6PtFxUyf/0eRm3enTp1O7du3o9OnTcmK0tra+70TB3yc/Z4y4Tz81NVXqFUzh+yuN6rsp7d+h6jm+5pOlJktLS/ljbGzfLXep8Xc2cOBArYUCx4wZQ23atJHPxM36HFb5d3zevHlkDLjribsp/Pz86PLly/Tvf/+bevToISdBCwsLk/oOGXe7cG1Kye5tY/kei0o5P5Tnbyhfl/ZvVfWcLiDUQKm475RP9po1J0yzD5sTNxdndurUSf4QNWjQwKB/mvxHUqVVq1YScvgE/+uvv0qBqan54Ycf5DNzgDGF76+64/8Fv/LKK1IYvWjRIq3nuLZP83ebTy4jRoyQ4k5jmI5/wIABWr+X/Bn495Fbb/j309RwPQ23FPMAE2P8Ht8p4/xgCND99Ii4+Z7/B1Gyspvve3p6kjEbPXq0FOLt3r2b6tat+8BtORiwS5cukbHh/1E88cQTcuz8nXF3DbdsmML3ee3aNdqxYwe99dZbJvv9MdV386B/h3xdsnifm/R5NI2xfLeqQMPfKxdparbSlPW98meMjY0lY8Tdw/w3VvV7aQrfocr+/fuldfRh/zYN9XscXcb5oTx/Q/m6tH+rqud0AaHmEXGCDgoKop07d2o1yfH9sLAwMkb8P0D+hV2/fj3t2rVLmoIf5vjx43LN/+M3NjwclFso+Nj5u7SystL6PvkPD9fcGOP3+eOPP0pzPY80MNXvj/HvKP8x1PzeuH+e6yxU3xtf8x9a7vNX4d9v/veqCnXGEGi4HoyDKtdbPAx/r1xvUrLLxljcuHFDampUv5fG/h2WbEHlvzc8UsqYvkfFQ84P5fkbytenTp3SCqiqkN6sWTOdHSg8otWrV8soi59++kmq84cPH65wdnbWquw2JiNHjlQ4OTkp9uzZo7h9+7b6kp2dLc9funRJMWPGDMXRo0cVV69eVfzxxx8Kf39/Rfv27RXG4P3335fPxsd+8OBBRefOnRVubm5Sxc/efvttRb169RS7du2SzxgWFiYXY8Oj8PhzTJw4UetxY/3+MjIyFP/8849c+E/WvHnz5LZq9M+sWbPk3x1/npMnT8qoEj8/P8W9e/fU++jevbuidevWisjISMWBAwcUjRo1UgwcOFBh6J8vLy9P0bNnT0XdunUVx48f1/p3qRotcujQIRkxw89fvnxZsWLFCkXt2rUV4eHhCkPxoM/Iz33wwQcyQoZ/L3fs2KFo06aNfEc5OTlG8R2W5/eUpaWlKWrUqCEjfkoy9O9x5EPOD+X5G1pQUKBo0aKFomvXrvI5t27dKp9x0qRJOjtOhJrH9PXXX8uXaG1tLUO8Dx8+rDBW/A+xtMuPP/4oz8fFxckJ0NXVVcJcw4YNFR9++KH8QzUG/fv3V9SpU0e+K29vb7nPJ3oVPgmOGjVK4eLiIn94XnrpJflHa2y2bdsm31tMTIzW48b6/e3evbvU30seBqwa1j158mSFh4eHfK5OnTrd99lTUlLkBFizZk0ZPjpkyBA5CRn65+OTfFn/Lvl1LDo6WhEaGionHFtbW0XTpk0Vn3/+uVYgMOTPyCdFPsnxyY2HBPOw5mHDht33n0ND/g7L83vKvvvuO4WdnZ0Mfy7J0L9Hesj5obx/Q2NjYxU9evSQnwP/p5L/s5mfn6+z4zQrPlgAAAAAo4aaGgAAADAJCDUAAABgEhBqAAAAwCQg1AAAAIBJQKgBAAAAk4BQAwAAACYBoQYAAABMAkINAAAAmASEGgAAADAJCDUAAABgEhBqAAAAwCQg1AAAAACZgv8HFO8VckS8lkAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.plot(torch.tensor(lossi).view(-1,1000).mean(1));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bafb37c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aveilles\n",
      "saienperçés\n",
      "reconvera\n",
      "prévalue\n",
      "sucultlicieurs\n",
      "déter\n",
      "soire\n",
      "assumet\n",
      "indorelant\n",
      "subsistement\n",
      "accommés\n",
      "solève\n",
      "veules\n",
      "facite\n",
      "éclarenduellotifiquisteprofitechep\n",
      "peunifie\n",
      "prisée\n",
      "didattre\n",
      "libérée\n",
      "rent\n"
     ]
    }
   ],
   "source": [
    "g = torch.Generator().manual_seed(seed + 10)\n",
    "for _ in range(20):\n",
    "    word = nn.generate_word(words.itoc, g)\n",
    "    print(word)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a85901d-31f1-4fa9-8f48-68d3fdfbf287",
   "metadata": {},
   "source": [
    "## Utiliser un contexte plus grand ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ccbafbc9-577d-42c8-9ffc-1ca3a32c426b",
   "metadata": {},
   "outputs": [],
   "source": [
    "context_size = 8\n",
    "datasets = Datasets(words, context_size)\n",
    "vocab_size = words.nb_chars\n",
    "e_dims = 10 # the dimensionality of the character embedding vectors\n",
    "n_hidden = 200 # the number of neurons in the hidden layer of the FFN\n",
    "seed = 2147483647\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e2100dbc-d902-4bb0-9b72-72a0bca86c5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BengioFFNModel\n",
      "  nb_chars=\"41\"\n",
      "  e_dims=\"10\"\n",
      "  n_hidden=\"200\"\n",
      "  context_size=\"8\"\n",
      "  nb_parameters=\"25133\"/>\n"
     ]
    }
   ],
   "source": [
    "nn = BengioFFNModel(e_dims, n_hidden, context_size, words.nb_chars, seed)\n",
    "print(nn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d5df4421-3136-4ef3-9e02-b97a22d29019",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0/ 200000: 3.7017\n",
      "  10000/ 200000: 1.9293\n",
      "  20000/ 200000: 1.5415\n",
      "  30000/ 200000: 1.3004\n",
      "  40000/ 200000: 1.6041\n",
      "  50000/ 200000: 1.4471\n",
      "  60000/ 200000: 1.3692\n",
      "  70000/ 200000: 1.7599\n",
      "  80000/ 200000: 1.2130\n",
      "  90000/ 200000: 2.1057\n",
      " 100000/ 200000: 1.3104\n",
      " 110000/ 200000: 1.5806\n",
      " 120000/ 200000: 1.1999\n",
      " 130000/ 200000: 1.2412\n",
      " 140000/ 200000: 1.2012\n",
      " 150000/ 200000: 1.5974\n",
      " 160000/ 200000: 1.5689\n",
      " 170000/ 200000: 1.3686\n",
      " 180000/ 200000: 1.1966\n",
      " 190000/ 200000: 1.1663\n",
      "train_loss=1.2270640134811401\n",
      "val_loss=1.6161370277404785\n"
     ]
    }
   ],
   "source": [
    "max_steps = 200000\n",
    "mini_batch_size = 32\n",
    "lossi = nn.train(datasets, max_steps, mini_batch_size)\n",
    "train_loss = nn.training_loss(datasets)\n",
    "val_loss = nn.test_loss(datasets)\n",
    "print(f\"{train_loss=}\")\n",
    "print(f\"{val_loss=}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d5bc136-3382-4671-b8cf-186a49664855",
   "metadata": {},
   "source": [
    "## Wavenet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "23e88d02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[19, 19,  7, 20, 17, 16, 15,  6],\n",
      "        [ 0,  0,  0,  0,  0,  0,  0, 20],\n",
      "        [ 5,  7, 20, 20, 11, 23,  7, 20],\n",
      "        [17, 16, 20, 20, 31,  6,  7, 15]])\n"
     ]
    }
   ],
   "source": [
    "ix = torch.randint(0, datasets.Xtr.shape[0], (4,))  # Batch de 4 exemples\n",
    "Xb = datasets.Xtr[ix]\n",
    "print(Xb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c6c1e996-65f7-4cb7-bab9-7362096600cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([6839, 8, 10])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.model.layers[0].out.shape  # Embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "cddb56bb-8aff-400d-a278-147809a82ab5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([6839, 80])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.model.layers[1].out.shape  # Flatten layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "475c86bc-2a20-4d81-a4d2-3ff5d3121fe7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([6839, 200])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.model.layers[2].out.shape  # First Linear layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d9202027-c3fa-47c8-8178-78bb3af30ef8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 200])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.randn(4,80) @ torch.randn(80,200)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b02fee4d-41b3-4abb-afc4-a7d51dc4228f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 200])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.randn(4,80) @ torch.randn(80,200)+torch.randn(200)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "1c657115-a74a-4c4c-88c8-fd343cd89da6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 200])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.randn(4,80) @ torch.randn(80,200)+torch.randn(200)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "99444a44-0806-4d87-aec8-45ccca70f63f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 5, 200])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.randn(4,5,80) @ torch.randn(80,200)+torch.randn(200)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a7112933-e47a-4e0d-9ede-01ce60efab08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 2, 5, 200])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.randn(4,2,5,80) @ torch.randn(80,200)+torch.randn(200)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e4e64f11-c390-404f-86e9-c4d8e79ec29a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0 1 2 3 4 5 6 7\n",
    "# (0 1) (2 3) (4 5) (6 7)\n",
    "# (a b) (c d)\n",
    "# (e f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "bcde6f9f-4294-48b4-bf76-f1c681ad64b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 200])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.randn(4,80) @ torch.randn(80,200)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "07075282-e15b-4559-8e04-82af568999a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 4, 200])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.randn(4,4,20) @ torch.randn(20,200)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b063b0a1-b893-463a-b48b-18b8fc5c9c00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Passer de [4,8,10] à [4,4,20] où les 10-d vecteurs consécutifs sont concaténés"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "15a14910-a23f-4d75-8c45-4f231f86b1a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 3, 5, 7, 9]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(range(10))[1::2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "4e33569b-bcd1-4984-b98e-e3000c1627f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 2, 4, 6, 8]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(range(10))[::2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "8aacd66a-f381-4cae-ac97-1a2c686a9f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "e = torch.randn(4, 8, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c3c0e8de-bb2e-4d18-963e-e33219e6cbb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "explicit = torch.cat([e[:,::2,:], e[:,1::2,:]], dim=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "59f15796-9e1e-4299-bf49-5c7b8e9e5045",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 4, 20])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "explicit.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "803a2e95-8f43-4476-8f0c-021e7ff97e13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 4, 20])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e.view(4,4,20).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "2557e2ba-9da8-46dc-9366-3c5fec9bcfc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(e.view(4,4,20) == explicit).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "d1128fa2-fd63-4e56-89f4-a5e49db1884b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 4, 20])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B = 4\n",
    "T = 8\n",
    "C = 10\n",
    "e = torch.randn(B, T, C)\n",
    "n = 2\n",
    "e.view(B, T//n, C*n).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "3c1471a7-ec4d-4359-bb07-05e081eda814",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 1, 20])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B = 4\n",
    "T = 2\n",
    "C = 10\n",
    "e = torch.randn(B, T, C)\n",
    "n = 2\n",
    "e = e.view(B, T//n, C*n)\n",
    "e.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f6ce2a36-a244-45f9-bbf9-92d22c481c93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 20])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e.squeeze(1).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5e79db5-025d-4ac0-b0f4-543505260370",
   "metadata": {},
   "source": [
    "Il faut que nous reprenions les classes introduites précédemment pour utiliser ces B, T et C."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b57a477-f92e-4842-b2e9-466d0952b61a",
   "metadata": {},
   "source": [
    "### Nouvelles classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "dd72b8ea-d6d7-43ea-b268-9379b777be33",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchNorm1d:\n",
    "    \"\"\"Batch normalization layer.\n",
    "    \n",
    "    Similar to <https://pytorch.org/docs/stable/generated/torch.nn.BatchNorm1d.html#torch.nn.BatchNorm1d>\n",
    "    \"\"\"\n",
    "  \n",
    "    def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "        self.eps = eps\n",
    "        self.momentum = momentum\n",
    "        self.training = True\n",
    "        # parameters (trained with backprop)\n",
    "        self.gamma = torch.ones(dim)\n",
    "        self.beta = torch.zeros(dim)\n",
    "        # buffers (trained with a running 'momentum update')\n",
    "        self.running_mean = torch.zeros(dim)\n",
    "        self.running_var = torch.ones(dim)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        # calculate the forward pass\n",
    "        if self.training:\n",
    "            if x.ndim == 2:\n",
    "                dim = 0\n",
    "            elif x.ndim == 3:\n",
    "                dim = (0, 1)\n",
    "            xmean = x.mean(dim, keepdim=True) # batch mean\n",
    "            xvar = x.var(dim, keepdim=True) # batch variance\n",
    "        else:\n",
    "            xmean = self.running_mean\n",
    "            xvar = self.running_var\n",
    "        xhat = (x - xmean) / torch.sqrt(xvar + self.eps) # normalize to unit variance\n",
    "        self.out = self.gamma * xhat + self.beta\n",
    "        # update the buffers\n",
    "        if self.training:\n",
    "            with torch.no_grad():\n",
    "                self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * xmean\n",
    "                self.running_var = (1 - self.momentum) * self.running_var + self.momentum * xvar\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.gamma, self.beta]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "43c560cf-e266-4e44-9598-384eebc66608",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FlattenConsecutive:\n",
    "  \n",
    "    def __init__(self, n): # n = 2\n",
    "        self.n = n\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        B, T, C = x.shape  # 4, 8, 10\n",
    "        x = x.view(B, T//self.n, C*self.n)\n",
    "        if x.shape[1] == 1:\n",
    "            x = x.squeeze(1)\n",
    "        self.out = x\n",
    "        return self.out\n",
    "  \n",
    "    def parameters(self):\n",
    "        return []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dba8e530",
   "metadata": {},
   "source": [
    "### Modèle Wavenet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "c5ee53a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_wavenet(self):\n",
    "    self.model = Sequential([\n",
    "        Embedding(self.nb_chars, self.e_dims),\n",
    "        FlattenConsecutive(2), Linear(self.e_dims   * 2, self.n_hidden, bias=False), BatchNorm1d(self.n_hidden), Tanh(),\n",
    "        FlattenConsecutive(2), Linear(self.n_hidden * 2, self.n_hidden, bias=False), BatchNorm1d(self.n_hidden), Tanh(),\n",
    "        FlattenConsecutive(2), Linear(self.n_hidden * 2, self.n_hidden, bias=False), BatchNorm1d(self.n_hidden), Tanh(),\n",
    "        Linear(self.n_hidden, self.nb_chars),\n",
    "    ])\n",
    "    with torch.no_grad():\n",
    "        self.model.layers[-1].weight *= 0.1\n",
    "BengioFFNModel._create_model = create_model_wavenet"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "300c4fc0",
   "metadata": {},
   "source": [
    "### Entraînement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "d22250ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "context_size = 8\n",
    "datasets = Datasets(words, context_size)\n",
    "vocab_size = words.nb_chars\n",
    "e_dims = 24 # the dimensionality of the character embedding vectors\n",
    "n_hidden = 128 # the number of neurons in the hidden layer of the FFN\n",
    "seed = 2147483647"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "eafc6c46-126a-4ee2-9b76-43ecd839fb53",
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.set_default_device('mps')\n",
    "#torch.get_default_device()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "8c0980b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = BengioFFNModel(e_dims, n_hidden, context_size, words.nb_chars, seed)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "be203a9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0/ 200000: 3.7377\n",
      "  10000/ 200000: 1.4646\n",
      "  20000/ 200000: 1.5771\n",
      "  30000/ 200000: 1.0538\n",
      "  40000/ 200000: 1.3723\n",
      "  50000/ 200000: 1.2684\n",
      "  60000/ 200000: 1.4154\n",
      "  70000/ 200000: 1.2276\n",
      "  80000/ 200000: 1.2436\n",
      "  90000/ 200000: 0.6402\n",
      " 100000/ 200000: 1.3461\n",
      " 110000/ 200000: 0.9522\n",
      " 120000/ 200000: 1.0788\n",
      " 130000/ 200000: 1.1689\n",
      " 140000/ 200000: 1.1304\n",
      " 150000/ 200000: 1.2926\n",
      " 160000/ 200000: 1.0429\n",
      " 170000/ 200000: 0.9127\n",
      " 180000/ 200000: 1.1666\n",
      " 190000/ 200000: 1.0528\n",
      "train_loss=0.9970558881759644\n",
      "val_loss=1.6977943181991577\n"
     ]
    }
   ],
   "source": [
    "max_steps = 200000\n",
    "mini_batch_size = 32\n",
    "lossi = nn.train(datasets, max_steps, mini_batch_size)\n",
    "train_loss = nn.training_loss(datasets)\n",
    "val_loss = nn.test_loss(datasets)\n",
    "print(f\"{train_loss=}\")\n",
    "print(f\"{val_loss=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78f11639-9d07-44e9-8bc3-d36153c6f99e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11ef25160>]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUJVJREFUeJzt3Ql4jOfaB/A7+77vIWRDrAkJkdbWSi3daLVFnWNpqy2tcnSj/dDtlKo6WpRWq3Sl+6KqKkSpEBIRSxCESGQnu+zzXfc9ZmRISCLJLPn/rmuu2d558868kfl7nvt5HiOFQqEgAAAAAB1mrO0DAAAAALgZBBYAAADQeQgsAAAAoPMQWAAAAEDnIbAAAACAzkNgAQAAAJ2HwAIAAAA6D4EFAAAAdJ4pGYCamhq6cOEC2dnZkZGRkbYPBwAAABqA564tKioib29vMjY2NvzAwmHFx8dH24cBAAAATXD+/Hlq37694QcWbllRvWF7e3ttHw4AAAA0QGFhoTQ4qL7HDT6wqLqBOKwgsAAAAOiXhpRzoOgWAAAAdB4CCwAAAOg8BBYAAADQeQgsAAAAoPMQWAAAAEDnIbAAAACAzkNgAQAAAJ2HwAIAAAA6D4EFAAAAdB4CCwAAAOg8BBYAAADQeQgsAAAAoPMMYvHDllJWWU3vbT1BlyuracF93cnMBPkOAABAG/ANfAO8eOSaXSn05d5UCS0AAACgR4Fl5cqV5OvrS5aWlhQeHk6xsbH1bvvjjz9SWFgYOTo6ko2NDYWEhNAXX3yhsc3kyZNlaenalxEjRpC2mZsYk/GVFa/LKhBYAAAA9KZLaOPGjTR79mxavXq1hJVly5bR8OHD6cSJE+Tu7n7d9s7OzvTqq69SUFAQmZub06ZNm2jKlCmyLb9OhQPKZ599pr5vYWFB2sbBycrMhEoqqtHCAgAAoE8tLEuXLqWpU6dK6OjWrZsEF2tra1q7dm2d2w8ZMoQeeOAB6tq1KwUEBNDMmTOpV69etHv3bo3tOKB4enqqL05OTqQLrMxN5BpdQgAAAHoSWCoqKiguLo4iIyOv7sDYWO7HxMTc9PUKhYKioqKkNWbQoEEaz0VHR0urS5cuXWjatGmUl5dX737Ky8upsLBQ49LSgaUUXUIAAAD60SWUm5tL1dXV5OHhofE43z9+/Hi9rysoKKB27dpJ0DAxMaEPP/yQ7rrrLo3uoAcffJD8/Pzo9OnT9Morr9DIkSMlBPH211q4cCG9/vrr1Bq4S4ihhgUAAMDAhzXb2dlRQkICFRcXSwsL18D4+/tLdxEbN26cetuePXtKlxF3H3Gry9ChQ6/b39y5c2UfKtzC4uPj06KBBV1CAAAAehJYXF1dpcUjKytL43G+z3Un9eFuo8DAQLnNo4SSkpKklUQVWK7FYYZ/1qlTp+oMLFzv0lpFuZYILAAAAPpVw8KjfEJDQ6WVRKWmpkbuR0RENHg//BruHqpPWlqa1LB4eXmRzhTdooYFAABAf7qEuCtm0qRJMrdKv379ZFhzSUmJjBpiEydOlHoVbkFhfM3bchcPh5TNmzfLPCyrVq2S57mbiOtRxowZI600XMPy0ksvSYtM7WHP2mKNUUIAAAD6F1jGjh1LOTk5NH/+fMrMzJQuni1btqgLcVNTU6ULSIXDzPTp06XVxMrKSuZj+fLLL2U/jLuYEhMTaf369ZSfn0/e3t40bNgwevPNN3ViLhZ1lxBaWAAAALTGSMFjjfUcF906ODjIaCR7e/tm3ferPx2mr/al0qzITjQrsnOz7hsAAKAtK2zE9zfWEroJjBICAADQPgSWBhbdYh4WAAAA7UFgaWANC2a6BQAA0B4ElpvAKCEAAADtQ2Bp6NT8ldWtcT4AAACgDggsN4HVmgEAALQPgeUmMA8LAACA9iGwNLBLCEW3AAAA2oPA0sCiW9SwAAAAaA8Cy01gtWYAAADtQ2C5CazWDAAAoH0ILA0e1lzTGucDAAAA6oDA0sDAUlFdQ1XVCC0AAADagMDSwC4hdhmTxwEAAGgFAstNWJgak5GR8jYCCwAAgHYgsNyEkZHR1TqWCnQJAQAAaAMCSwOoAgtaWAAAALQDgaURc7GUVlS19PkAAACAOiCwNAAWQAQAANAuBJYGwPT8AAAA2oXA0qgVm1F0CwAAoA0ILA2AolsAAADtQmBpTGBB0S0AAIBWILA0AIpuAQAAtAuBpVErNqOGBQAAQBsQWBoANSwAAADahcDSAOqp+bH4IQAAgFYgsDSiSwgz3QIAAGgHAktj5mGpRA0LAACANiCwNGpYc3VLnw8AAACoAwJLA2BqfgAAAO1CYGlUlxBaWAAAALQBgaVRRbcILAAAANqAwNIAGNYMAACgXQgsDYCiWwAAAO1CYGkArCUEAACgXQgsDYDAAgAAoF0ILI3oEqqoqqHqGkVLnxMAAABojsCycuVK8vX1JUtLSwoPD6fY2Nh6t/3xxx8pLCyMHB0dycbGhkJCQuiLL77Q2EahUND8+fPJy8uLrKysKDIykpKTk0nXAgvD0GYAAAA9CCwbN26k2bNn04IFCyg+Pp6Cg4Np+PDhlJ2dXef2zs7O9Oqrr1JMTAwlJibSlClT5PLnn3+qt1m8eDF98MEHtHr1atq3b58EG95nWVkZ6QIL06sfE2a7BQAAaH1GCm7eaARuUenbty+tWLFC7tfU1JCPjw/NmDGD5syZ06B99OnTh+655x568803pXXF29ubnn/+eXrhhRfk+YKCAvLw8KB169bRuHHjbrq/wsJCcnBwkNfZ29tTS+g6b4u0rux66Q7ycbZukZ8BAADQlhQ24vu7US0sFRUVFBcXJ1026h0YG8t9bkG5GQ4nUVFRdOLECRo0aJA8lpKSQpmZmRr75IPnYFTfPsvLy+VN1r60NBTeAgAAaE+jAktubi5VV1dL60dtfJ9DR304Odna2pK5ubm0rCxfvpzuuusueU71usbsc+HChRJqVBdu4WmtOhbMdgsAAGCgo4Ts7OwoISGB9u/fT//973+lBiY6OrrJ+5s7d66EINXl/Pnz1NIszZQfFWpYAAAAWp9pYzZ2dXUlExMTysrK0nic73t6etb7Ou42CgwMlNs8SigpKUlaSYYMGaJ+He+DRwnV3idvWxcLCwu5tCZVl1AZFkAEAADQ7RYW7tIJDQ2VOhQVLrrl+xEREQ3eD7+G61CYn5+fhJba++SaFB4t1Jh9tjQbc2W2Kyyr1PahAAAAtDmNamFh3J0zadIkmVulX79+tGzZMiopKZGhymzixInUrl07aUFhfM3bBgQESEjZvHmzzMOyatUqed7IyIhmzZpFb731FnXq1EkCzLx582Tk0OjRo0lXeNhbynV2oTJoAQAAgA4HlrFjx1JOTo5M9MZFsdxts2XLFnXRbGpqqnQBqXCYmT59OqWlpcmkcEFBQfTll1/KflReeukl2e7JJ5+k/Px8GjBggOyTJ6bTFV4OymPJKNCNuWEAAADakkbPw6KLWmMelrW7U+iNTcfo7p6e9OGE0Bb5GQAAAG1JYUvNw9KWqVpYMtHCAgAA0OoQWBrIE4EFAABAaxBYGsjLwUqus4rKsWIzAABAK0NgaSA3OwsyMTaSsJJbjJFCAAAArQmBpYE4rLjbKSerw0ghAACA1oXA0qQ6lsstdT4AAACgDggsjYC5WAAAALQDgaURPO2VhbcY2gwAANC6EFiaMhdLIWa7BQAAaE0ILE2oYUHRLQAAQOtCYGkEzHYLAACgHQgsTVixmWtYDGAJJgAAAL2BwNKEwFJRXUMXSypa6pwAAADANRBYGsHc1JhcbTF5HAAAQGtDYGkk1LEAAAC0PgSWps52i6HNAAAArQaBpZHQwgIAAND6EFgayctBOdtt2qXSljgfAAAAUAcElkbq6GIt1+cuIrAAAAC0FgSWpgaWPAQWAACA1oLA0kgdXWzkmudhKSyrbIlzAgAAANdAYGkkWwtTcrU1l9upaGUBAABoFQgst9DKgm4hAACA1oHA0gQdnZV1LGfzSpr7fAAAAEAdEFhuoYUFXUIAAACtA4HlFkYKoYUFAACgdSCw3EJgScVcLAAAAK0CgeUWuoQyCsqorLK6uc8JAAAAXAOBpQmcrM3IztJUbp9HKwsAAECLQ2BpAiMjo1p1LJjxFgAAoKUhsNzyXCwY2gwAANDSEFhucS4WTB4HAADQ8hBYmshX1cKCGhYAAIAWh8DSRKoaltPZxc15PgAAAKAOCCxN1M3bnoyMiNLzL1NOUXlTdwMAAAANgMDSRHaWZtTJ3VZuJ5zPb+puAAAAoAEQWG5BiI+jXCecv3QruwEAAICbQGC5BSE+TnKNFhYAAAAdDCwrV64kX19fsrS0pPDwcIqNja132zVr1tDAgQPJyclJLpGRkddtP3nyZJmMrfZlxIgRpOt6d1C2sBw6X0DVNQptHw4AAIDBanRg2bhxI82ePZsWLFhA8fHxFBwcTMOHD6fs7Ow6t4+Ojqbx48fTjh07KCYmhnx8fGjYsGGUnp6usR0HlIyMDPXlm2++IV3X2cOOrM1NqLi8ik7nYLQQAACAzgSWpUuX0tSpU2nKlCnUrVs3Wr16NVlbW9PatWvr3P6rr76i6dOnU0hICAUFBdEnn3xCNTU1FBUVpbGdhYUFeXp6qi/cGqPrTIyNqGc7B7mdkIrCWwAAAJ0ILBUVFRQXFyfdOuodGBvLfW49aYjS0lKqrKwkZ2fn61pi3N3dqUuXLjRt2jTKy8urdx/l5eVUWFiocdGWkCvdQgdReAsAAKAbgSU3N5eqq6vJw8ND43G+n5mZ2aB9vPzyy+Tt7a0Rerg76PPPP5dWl3feeYd27txJI0eOlJ9Vl4ULF5KDg4P6wt1M2tL7SuHtQbSwAAAAtBhTakWLFi2iDRs2SGsKF+yqjBs3Tn27Z8+e1KtXLwoICJDthg4det1+5s6dK3U0KtzCoq3Qoiq8PZlVRCXlVWRj0aofKQAAQJvQqBYWV1dXMjExoaysLI3H+T7XndzIkiVLJLBs3bpVAsmN+Pv7y886depUnc9zvYu9vb3GRVs87C3J3c6CeJDQ8cwirR0HAACAIWtUYDE3N6fQ0FCNgllVAW1ERES9r1u8eDG9+eabtGXLFgoLC7vpz0lLS5MaFi8vL9IHXTzt5PoEAgsAAIBujBLirhieW2X9+vWUlJQkBbIlJSUyaohNnDhRumxUuCZl3rx5MoqI527hWhe+FBcrhwHz9Ysvvkh79+6ls2fPSvgZNWoUBQYGynBpfRB0JbBwtxAAAAA0v0YXXIwdO5ZycnJo/vz5Ejx4uDK3nKgKcVNTU2XkkMqqVatkdNFDDz2ksR+ex+W1116TLqbExEQJQPn5+VKQy/O0cIsMd/3ogy6eyi6p45naG60EAABgyIwUCoXeT9HKRbc8WqigoEAr9SxH0gvo3uW7ycnajOLn3SUz9QIAAEDzfX9jLaFmEOhuS8ZGRJdKKymnqLw5dgkAAAC1ILA0A0szE/J1sZHbJ1DHAgAA0OwQWJoJRgoBAAC0HASWZg4smIsFAACg+SGwNPPQZszFAgAA0PwQWJpJZ4+rc7FU87S3AAAA0GwQWJpJRxcbsjQzpvKqGkq9WNpcuwUAAAAEluZjYmxEndyv1LFkYAI5AACA5oQWlmYU7OMg1/tSLjbnbgEAANo8BJZmNLCTm1z/fTKnzf9iAQAANCcElmYUEeAiXUNnckvoPOpYAAAAmg0CSzOytzSj3j6OcntXcm5z7hoAAKBNQ2BpZoM6o1sIAACguSGwNLOBnVzl+p/TuVRVXdPcuwcAAGiTEFiaWa/2juRgZUZFZVV0KC2/uXcPAADQJiGwNDMuuh0QqGxl2XkSdSwAAADNAYGlBQzuoqxj2Xo0syV2DwAA0OYgsLSA4d08yczESFZuxmKIAAAAtw6BpQU4WJvR4M7ucvvXQ+kt8SMAAADaFASWFjIqxFuuf0m4QAoFVm8GAAC4FQgsLSSyqwdZm5tQ2qXLFJ+K0UIAAAC3AoGlhViZm9Cwbh5y+7dDF1rqxwAAALQJCCwt6P4r3UKbD2egWwgAAOAWILC0oNsCXGW0UHZROaViMUQAAIAmQ2BpQZZmJtSjnYPcPnD2Ukv+KAAAAIOGwNLCwjo6yfWBcwgsAAAATYXA0sJCOzrLddy5iy39owAAAAwWAksLC73SwnIyq5gKSitb+scBAAAYJASWFuZmZ0G+LtZyOz4V3UIAAABNgcDSit1CB9AtBAAA0CQILK0gzPdK4S1GCgEAADQJAksrjhQ6lJZPldU1rfEjAQAADAoCSysIcLMlJ2szKqusoR/i0lrjRwIAABgUBJbW+JCNjWjakAC5/dbvSZSef7k1fiwAAIDBQGBpJY8P8Kc+HRypuLyK5vyQiLWFAAAAGgGBpZWYGBvRkoeDycLUmHYl59KmxIzW+tEAAAB6D4GlFfm72dJjA/zk9pajma35owEAAPQaAksri+zqLte7k3OpukbR2j8eAACg7QSWlStXkq+vL1laWlJ4eDjFxsbWu+2aNWto4MCB5OTkJJfIyMjrtlcoFDR//nzy8vIiKysr2SY5OZkMUXB7R7KzNKWCy5UyzBkAAABaILBs3LiRZs+eTQsWLKD4+HgKDg6m4cOHU3Z2dp3bR0dH0/jx42nHjh0UExNDPj4+NGzYMEpPT1dvs3jxYvrggw9o9erVtG/fPrKxsZF9lpWVkaExNTGmAYGucvvvkznaPhwAAAC9YKTg5o1G4BaVvn370ooVK+R+TU2NhJAZM2bQnDlzbvr66upqaWnh10+cOFFaV7y9ven555+nF154QbYpKCggDw8PWrduHY0bN+6m+ywsLCQHBwd5nb29Pem6DbGpNOfHwzJq6Mfpt2v7cAAAALSiMd/fjWphqaiooLi4OOmyUe/A2Fjuc+tJQ5SWllJlZSU5OyvX10lJSaHMzEyNffLBczCqb5/l5eXyJmtf9Mmgzm5ynXA+Hys4AwAANECjAktubq60kHDrR218n0NHQ7z88svSoqIKKKrXNWafCxculFCjunALjz7xdrSiQHdb4prb3adytX04AAAAOq9VRwktWrSINmzYQD/99JMU7DbV3LlzpflIdTl//jzpm0GdlK0sO0/WXfsDAAAATQwsrq6uZGJiQllZWRqP831PT88bvnbJkiUSWLZu3Uq9evVSP656XWP2aWFhIX1dtS/65o4gZWDZfjwbw5sBAACaM7CYm5tTaGgoRUVFqR/jolu+HxERUe/reBTQm2++SVu2bKGwsDCN5/z8/CSY1N4n16TwaKEb7VPfhfu5yPDm3OIKOph6SduHAwAAYFhdQjykmedWWb9+PSUlJdG0adOopKSEpkyZIs/zyB/uslF55513aN68ebR27VqZu4XrUvhSXFwszxsZGdGsWbPorbfeol9//ZUOHz4s++A6l9GjR5OhMjc1pqFByknk/sSstwAAADdkSo00duxYysnJkYneOHiEhIRIy4mqaDY1NVVGDqmsWrVKRhc99NBDGvvheVxee+01uf3SSy9J6HnyyScpPz+fBgwYIPu8lToXfTCsuyf9nHCB/jyaRa/c3VXCGwAAADTDPCy6SN/mYVEpKa+i3m/+RRVVNbRl1kAK8tSfYwcAANDZeVigedlYmNKgTspZb/88oll0DAAAAFchsGjZsG7KkVCoYwEAAKgfAouWRXbzIDMTIzqWUUhH0gu0fTgAAAA6CYFFy5xtzGlkDy+5/UXMOW0fDgAAgE5CYNEBk27rKNc/J6TTpZIKbR8OAACAzkFg0QF9OjhRd297Kq+qoW8P6N8yAwAAAC0NgUUH8PwrkyJ85fYXe89hqn4AAIBrILDoiPtDvMnR2ozSLl2mTYkXtH04AAAAOgWBRUdYmpnQ47f7ye13/zxB5VXV2j4kAAAAnYHAokMeH+hH7nYW0sqCEUMAAABXIbDoEGtzU5p9V2e5vXz7KSoordT2IQEAAOgEBBYd81Boe+rsYUsFlyvpsz0p2j4cAAAAnYDAomNMTYxp+pBAuf19XBrV1Oj92pQAAAC3DIFFBw3v7kl2FqZSy7Iv5aK2DwcAAEDrEFh0kJW5Cd0b7KVuZQEAAGjrEFh0uJaF/XEkg0rKq7R9OAAAAFqFwKLD0/X7udpQaUU1/XEkU9uHAwAAoFUILDo8Xb+qlWXh5iT6JSGdFAoU4AIAQNuEwKLDHu3XQYY455VU0MwNCfT0l3FUVV2j7cMCAABodQgsOszJxpx+mzFAJpMzNzGmP49m0eqdp7V9WAAAAK0OgUXHWZia0HNDO9GiMT3l/rJtyZSYlq/twwIAAGhVCCx64oHe7eienl5UVaOgWRsTqLQCI4cAAKDtQGDRoyLct0b3kMURz+SU0OyNhzALLgAAtBkILHpW0/LhhD5Sz7LlaCYt/vOEtg8JAACgVSCw6JkwX2da/FAvuc0FuL8nZmj7kAAAAFocAoseGt27HT0xwE9ufx93XtuHAwAA0OIQWPQ4tLD9Zy9hbhYAADB4CCx6qquXPdlbmlJxeRUdvVCo7cMBAABoUQgsesrE2Ij6+bnI7b1n8rR9OAAAAC0KgUWP9fd3lmsEFgAAMHQILHqsv7+LRh0Ldw/ll1Zo+7AAAACaHQKLgdSx/JZ4ge5cEk1DlkTTpRKEFgAAMCwILHpexxJ+pZXlPxsPUXZROeWXVtIfRzK1fWgAAADNCoHFQLqFmLmp8nT+eihdi0cEAADQ/BBY9NzQIHcyMzGiQHdb+vapCHlsX8pFyios0/ahAQAANBvT5tsVaIOvqw39M+dOcrAyIwtTEwrt6ERx5y7RpsQMevzKbLgAAAD6Di0sBsDdzlLCCruvl5dc/3bogpaPCgAAoPkgsBiYu3t5kbERUcL5fNqdnEsKhULbhwQAAKCdwLJy5Ury9fUlS0tLCg8Pp9jY2Hq3PXr0KI0ZM0a2NzIyomXLll23zWuvvSbP1b4EBQU15dDaPG5tGdTZTT6Hf326j0Z/uIcS0/Lb/OcCAABtLLBs3LiRZs+eTQsWLKD4+HgKDg6m4cOHU3Z2dp3bl5aWkr+/Py1atIg8PT3r3W/37t0pIyNDfdm9e3djDw2uePehYBrX10dGDR06n0///jSWkrOK8PkAAEDbCSxLly6lqVOn0pQpU6hbt260evVqsra2prVr19a5fd++fendd9+lcePGkYWFRb37NTU1lUCjuri6ujb20OAKNzsLWjSmF+2Zcyf16eBIBZcraeLaWMoouIzPCAAADD+wVFRUUFxcHEVGRl7dgbGx3I+JibmlA0lOTiZvb29pjZkwYQKlpqbWu215eTkVFhZqXOB6rrYW9OmkvhTgZkMZBWX06Jp9dCanGB8VAAAYdmDJzc2l6upq8vDw0Hic72dmNn12Va6DWbduHW3ZsoVWrVpFKSkpNHDgQCoqqrsbY+HCheTg4KC++Pj4NPlnGzonG3P6/PFwaudoRSm5JTR65T/0z6lcbR8WAACA/o0SGjlyJD388MPUq1cvqYfZvHkz5efn07ffflvn9nPnzqWCggL15fz5861+zPqEw8rPz9wu3UOFZVU0Zd1+upCP7iEAADDQwMJ1JSYmJpSVlaXxON+/UUFtYzk6OlLnzp3p1KlTdT7PtTD29vYaF7h5XcvXU/tTj3b2VFFVQ1HH6y6SBgAA0PvAYm5uTqGhoRQVFaV+rKamRu5HRCinhW8OxcXFdPr0afLyUk6CBs3D0syERvZQfqY7TyCwAACAAXcJ8ZDmNWvW0Pr16ykpKYmmTZtGJSUlMmqITZw4UbpsahfqJiQkyIVvp6eny+3arScvvPAC7dy5k86ePUt79uyhBx54QFpyxo8f31zvE64YfGWOlj2n86i8qppyisrp2a/jaQdaXAAAwJDWEho7dizl5OTQ/PnzpdA2JCREimVVhbg8uodHDqlcuHCBevfurb6/ZMkSuQwePJiio6PlsbS0NAkneXl55ObmRgMGDKC9e/fKbWhe3b3tpXuIg8qBs5dkCn9edygqKZt+mzFAFlEEAADQNUYKA5i7nYc182ghLsBFPcvNPf/tIfohPo1G9vCkv45lUVWN8lcgyNNOinO56wgAAECXvr91YpQQtK7BXZQtV38cyZSwwis8u9iY0/HMInp7cxJOBwAA6BwEljZoYKCrLJCo8srdXWnp2BC5/eXec5gRFwAAdA4CSxudTC7Yx1Fu80KJ3MLCxbj9/JyJe4e+P5Cm7UMEAADQgMDSRj03tBOF+znT/Hu7qh8bG6acMfjbuPNUc6WuBQAAQBcgsLRRd3Rxp41PRVCgu536sbt7epGdhSmdv3iZ9p7J0+rxAQAA1IbAAmpW5iZ0X4i33N54AMsdAACA7kBgAQ2qbiEeQXT6Bis7H88spO8OoOsIAAB0dOI4MGy92jtQcHsHOpRWQPct300L7utG7RytZeRQB2drKdblkUTvbDlOldUKcrE1pzuDNFfvBgAAaG4ILKDByMiI1kwMo+c2HKS9Zy7Syz8c1vyFMTZSTzTHeBsEFgAAaGnoEoLruNtb0ldP9Kf/RHYmT3tLma7/tgAXcrU1l7BiYWpMw7opW1UOnL2ITxAAAFocWligTibGRjQzspNcVHgVhzO5JeRsbU6FZZW09VgWHUkvpLLKakznDwAALQotLNCo7qIAN1uZeI7rWVxtLaiiuoYOpxfgUwQAgBaFwAJNDi9hHZ3kNq/6DAAA0JIQWKDJeEp/Fnfuoka3Ede17MPEcwAA0IxQwwJNFuqrCiyXJKj8nZxLH0Qly302485AKdw1rr3SIgAAQBMgsECT9fB2kBFDl0oraerncbQtKUseNzcxltqW5dtP0bELhdSjnQOZmxrTQ6HtycPeEp84AAA0GgILNBmHkOD2jhR79qKEFSMjosm3+dK0wQEUfTKHXv3pMEUdz5YLO3qhgD6cEIpPHAAAGg2BBW5JRICLBBY3Owt6f2wI3RboKo8/EuZDndxt6ZeEC1RUVkU/xKdRVFI2lZRXkY0Ffu0AAKBx8M0Bt2TqIH/ycbamIV3cZJhzbb07OMmF61u4MPdsXqm0ttwfrFxgUYWf51FHAAAA9cEoIbglthamUptybVipjcPIPb285PbviRc0nluxPZnC346ibceU9S8AAAB1QWCBVnFPT2Wryo4TOVRcXiW3f4hLoyVbT1J2UbmsXZSUUYizAQAAdUJggVbR1cuO/F1tqKKqRlpT/jmVS3N/VC6syK0zpRXV9MT6A5RXXI4zAgAA10FggVZRu1voP98m0IRP9snQ57u6edDW/wwiXxdrSs+/TLM2JkhNCwAAQG0ILNBqRoV4y6KKnEcszYzp7p6etGxsCDnbmNOaiWEyp8uu5Fz6al8qzgoAAGgwUhjAf2cLCwvJwcGBCgoKyN7eXtuHAzfAE8lxy0o3L3uZx6W2T3en0JubjpG1uQltmTmIOrhY47MEADBghY34/kYLC7Sqbt72FOLjeF1YYVNu86VwP2epZ5n6+QHaeTIH3UMAACDQwgI65fzFUrp3+W4quFwp94N9HOm5OwNpSBd3+utYFm3Yn0oXSyqkeLe/vwstuK8b5nABAGgDLSwILKBzsgvL6KO/z9BX+85RWWWNPGZnaSoz5l4r+oUh5Otqo4WjBACAW4UuIdBr7vaWNO/ebrT75TvpqcH+UtPCYYVDy/QhAbR2chj1au8g2+5KztF4bXlVNX0YfYpiUy5q6egBAKAlYGp+0Fk8P8vckV3p6UEBdORCgdS+2FmayXNJGUWUmFZAO0/m0r8jfNWveXfLCfpkdwp52FvQnjlDZVQSAADoPxTdgs5zsjGngZ3c1GGFDerkJtcxp3OpslrZbbQ7OVfCCssqLKd9KXlaOmIAAGhuCCygl7p728v8LSUV1XQwNZ8ulVTQ898lyHM25iZy/duhq+sW1dQo6Ot9qTRq5T/090nNbiQAANB9CCygl4yNjWhAoKvcjj6RLTPkcqtKgJsNvT+utzy++XCmjCbikUePfrKXXvnpMB06n0/v/XVSy0cPAACNhRoW0FsDO7nSr4cu0JpdZ6iyWiGz5y4f34e6eNpJ/UtucTl9ve8crd55hjILy+T58qoaCS28DEA7RyttvwUAAGggtLCA3hrUWVnHwmGFvTOml0xMx4W2915Zt+i1345JWAl0t6U/Zw2ivh2d5fEtRzK1eOQAANBYCCygtzzsLWWKfzZ1oB+NCmmnfu6+YG/1bT9XG/r6iXDq6GJDI3t6ymNbjmRo4YgBAKCp0CUEeu2D8b2li4cXVqytTwdH6TLKKSqntZP7ytwubEQPT3r9t2N04NwlOpNTTO9HJcvMuav/FUo2FvjnAACgqzDTLbQ5o1f+Qwnn88nWwpSKy5Wz5/7fPV3piYH+6m24/mXpXyfJ39VG43EAANCjmW5XrlxJvr6+ZGlpSeHh4RQbG1vvtkePHqUxY8bI9kZGRrRs2bJb3ifArRjZQ9ktxGHF3ET5T+CTXSkyoogdTL1E9y3fLcOg3/o9iVJyS/CBAwBoWaMDy8aNG2n27Nm0YMECio+Pp+DgYBo+fDhlZ2fXuX1paSn5+/vTokWLyNPTs1n2CXAruL6Fp/v3d7OhzTMHkJudhRTm/pKQTt/HpdEjH8VQRkGZevsv95674f4UCmXRLwAA6FCXELd+9O3bl1asWCH3a2pqyMfHh2bMmEFz5sy54Wu5BWXWrFlyaa59NrZJCUB+Z8oqycrMhMxMjGn1ztO06I/jGgssDu/uQff08qbnvjlI9pamtO+VSLK6MiGdyp5TuTTvlyMyRNrb0Yp6+zjR/Pu6kYPV1Rl5AQBAC11CFRUVFBcXR5GRkVd3YGws92NiYhqzq1vaZ3l5ubzJ2heAxrC3NJOwwh4N70B2FlfDyjN3BNCqCaF0b08v6uBsTYVlVdL6onK5oprm/3KEHv1kH53OKZEVpc/klNAP8Wn06k+H1S0uZZXVVF2D1hcAgObQqMCSm5tL1dXV5OHhofE438/MbNq8Fk3Z58KFCyWRqS7cGgNwK+Flzt1B5OVgSe8+1IteHB4kM+ny5V/9O8g262POSSHu8cxCun/Fbvo8RtlNNCG8A22bPZhWPNpb5n/ZlJgh3UortidTz9f+lBYaAAC4dXo5jnPu3LlS86LCLSwILXArJoR3lMu1Hgnzofe2nqSkjEIKe2ubhBJuNeG6l/ceDlZPXscT06XklMi0/y9+n6h+/e+HM+hfp/MoIsAFJwgAoLVaWFxdXcnExISysrI0Huf79RXUtsQ+LSwspK+r9gWgJTham9Obo3pQFw87MjIiCStDurjRHzMHqsOKyvQ7Aqmvr5Pc5iHT/XyVs+ou2XoChbkAAK3ZwmJubk6hoaEUFRVFo0ePVhfI8v1nn322SQfQEvsEaE6P9PWRS0FpJeWVlMvMuTxE/1rc+vLRv8Pop4PpNKybB5mbGtOgxTso7twlij6RQ3cEuePEAAC0VpcQd8VMmjSJwsLCqF+/fjKvSklJCU2ZMkWenzhxIrVr107qTFRFtceOHVPfTk9Pp4SEBLK1taXAwMAG7RNAFzhYm8nlRpxtzOnxAX7q+5Nu86WP/z4jK0UP6uRGPdo70Li+PuqC39q4WJdn3v3uQBp99O9Q6tHOoUXeBwBAmwgsY8eOpZycHJo/f74UxYaEhNCWLVvURbOpqakyykflwoUL1Lt3b/X9JUuWyGXw4MEUHR3doH0C6KunBwfQhthUmddl44Hzckm7VEpzR3aVUUT/23aSHK3MaWxfH1l1elX0aXnde1tP0GdT+mn78AEAdAam5gdoYefySmj/2Ut0PKOQPtmdIrUw30ztT5/uTqG/jilrt8xMjNSrTvPzPDJ6y6yBFORpL8GGt/v5YDrVKBS0ckIfsjbXy3p5AIAmz8OCv3oALYxXieYLu1RaKfO1/PvTfRJQuM4l0M2WjmUo5xJacF83OnD2kowu+mjnGZoY0ZGe+iKOsovK1fvbfDiTHgptj/MGAG0KAgtAK1pwfzeKOZ1LFwrKyNiIaPn43lKgG5+aT5XVNdTf34XCOjpLYPn10AXafDiDyqtqyNPeUiaxiz17kTYlXpDAwmsffb3vHA3o5CbDqgEADFmTFj8EgKZPUrf80T7Up4Mj/W9sCA3v7ikjjkI7OklYYT3bO9DtgS4yhJrDyp1B7hT1/GB6+8Ge8vzu5FzKL62gj3aeptd+O0bPfh1f77BpfvxC/mUqr6rGKQMAvYYWFoBWxuHkx+m333CbOSO60oxv4mlYd096eUSQDJnmVpQgTzs6nlkkI4k+3nVGtuX7CefzqXcHJ1q545QsI8BDr3lyu13JuXQur5SGBrnTp5P7qvfPrTm7T+VS/LlLNK5fB2rnaNWgY+dlCdb+k0Il5VX0/LAuclwAAK0BgQVAB3ErS/SLd1z3+L29vCSgvLPlOFXVWqfom9hUsjQzkdFF/PDJrGKN10Udz6aDqZck1PwYn0Zvbjom9TRs75k8+vapiDrnlqltx4lsmvfzEUq7dFnu9/Vzpju6YG4ZAGgd6BIC0CN39/SSa1VYmTpQOefLb4cyZEFGfnhwZzead283emKAH304oQ+NCvGWbXhV6uSsIprzw2EJK6625lL0yyOYopKyZZu/T+ZIC821XUynsovoifUH1GGF7TyR02rvGwAALSwAesTfzZa6ednLqKKe7RxkPpcdJ3LoVHaxBA9zE2N6a3QP8nG2Vr+ms4ct/ZJwgf48miWrS1dU19AdXdxozcQwWfuI537hFpszucX09ubj8hou6H047OqiotwFxTU1/f2dZc6Y/2w8RDtPKgNLdmEZvfrzEXo4tL10Yanw9rW7jE7nFJOHvaUsWwAA0FhoYQHQM88N7ST1LK+P6i4rSj/aT7miNJsywFcjrLBAdzu6q5tyEkYONnYWplLAa2piLBPbOVqbUXL21bDCFvx6lFJyS+R2VXWNLDcg+7/djyK7epCpsZE8z3PMfLA9WeaJWfrXSfXrX/4+kcLe+oviUy/JfV7Beuh7O+mZr+Jb+NMBAEOFwAKgZ0b08KRtswdTnw7KhRYf7NNOlgTwdrCkZ+5QLndxrWlDAtS3X72nK3k5KItsHazM6Nlar5l9V2dpRSmtqKbnvjkoLS27TuXKPDD8M7hmxc7STAqHGQcZDiOMa2sulVRIYS4/zt1OT35+QIZm89IEjFtleKZfAIDGQtssgAGsKB01e7C0tvCw6bpwuOHRRpcrq6VLp7aJEb5UWFYlrTb3B3vTw2HtaeT7u+hwegE9/WUcGV8pxuXnuOaFDeniTvtSLtKK7ac0in/3n71IFmYm0u3EcosraPo1rSo8v8z0IXUHKwCA+qCFBcAAONmYS2vJjXArC7egXDsaiEMIP86BhHHrC09oZ2FqTNuPZ9O2JOXyAbVn1x3SxU2uVWHFy8FSrjnE/HMqV25znYyHvYXc9ne1of+7p6vc5iUG6ps3BgCgPggsAHCdgZ3c6MsnwsnOUtkIy/O/dPe+us4H3+fZdxlfvzCsi9zel5Inc7+w0b3b0VdP9JeRTOsf6ydFvFwUzEOukzKK8KkDQKMgsABAnfr6OtPGJyNk6QAeJl27ZYZvq4ZYTx3kTwM6ucrtoxcKKenKuki3B7pKN9Or93STQmBuARraVTlvC88Fk1dcTlmFZWhtAYAGwWrNANAkvIo0BxReZoADzJB3d9DZPGVBbVcve/pj5sDrXrPlSKbUxdTmamsh++AuK57YjruLeAh24eVKGhPaHrPpAhiwQqzWDAAtjWfWVY0WYuF+LurAMvBKi8u17ghyk0UcUy8qt+NpWnKLy2nrsSyZjXfm0E50JL1A7rPv49NkzaWGLh0AAIYLLSwA0Cy4m2f2t4fkNtes8Iy7deGh0rx4IxcK8+RyHFDW7TlLmxIz1NuYmRiRmYmxDK+2tzSlP2YNQmgBaOMtLKhhAYBmERHgIt03NuYm1M/Xud7teFSSu72lBBJupQnzdZZRSYvH9JLXcnHvr88OkC4lHl3EQ66jroxUAoC2C/OwAECz4OHQnz/Wj6zMTeTSGFwD80hfHxrV21tGEqkKfMP9XehMbgnlFVfgLAG0cQgsANBseGTQrbAw1Qw6Ljbmcn2xBIEFoK1DlxAA6CxeDoAhsAAAAgsA6CwXW2VgySsp1/ahAICWIbAAgM5CCwsAqCCwAIDOQmABABUEFgDQWS42ysUTL5VWUk2tVaEBoO1BYAEAneVko1yBmieYK7hcqe3DAQAtQmABAJ3Fw5ztLJSzL+RhaDNAm4bAAgA6zfnKSCEMbQZo2xBYAECnXZ08DkObAdoyBBYA0GnOVwpv0SUE0LYhsACAfrSwYD0hgDYNgQUA9KKGBS0sAG0bAgsA6DQsgAgADIEFAHQaZrsFAIbAAgB6EVjQJQTQtiGwAIBeTM+PYc0AbRsCCwDozcRxCgXWEwJoqxBYAEAvim4rqxVUVF6l7cMBAC1BYAEAnWZpZkLW5iZyOw9zsQC0WU0KLCtXriRfX1+ytLSk8PBwio2NveH23333HQUFBcn2PXv2pM2bN2s8P3nyZDIyMtK4jBgxoimHBgAGPVII0/MDtFWNDiwbN26k2bNn04IFCyg+Pp6Cg4Np+PDhlJ2dXef2e/bsofHjx9Pjjz9OBw8epNGjR8vlyJEjGttxQMnIyFBfvvnmm6a/KwAwyG4htLAAtF2NDixLly6lqVOn0pQpU6hbt260evVqsra2prVr19a5/fvvvy9h5MUXX6SuXbvSm2++SX369KEVK1ZobGdhYUGenp7qi5OTU9PfFQAYFMzFAgCNCiwVFRUUFxdHkZGR6seMjY3lfkxMTJ2v4cdrb8+4Reba7aOjo8nd3Z26dOlC06ZNo7y8vHqPo7y8nAoLCzUuAGC4sAAiADQqsOTm5lJ1dTV5eHhoPM73MzMz63wNP36z7bkF5vPPP6eoqCh65513aOfOnTRy5Ej5WXVZuHAhOTg4qC8+Pj44kwAGzKXW0GYAaJtMSQeMGzdOfZuLcnv16kUBAQHS6jJ06NDrtp87d67U0ahwCwtCC4Dh17BkFpZp+1AAQB9aWFxdXcnExISysrI0Huf7XHdSF368Mdszf39/+VmnTp2q83mud7G3t9e4AIDh6tXeUa73ns6jmhpMHgfQFjUqsJibm1NoaKh03ajU1NTI/YiIiDpfw4/X3p799ddf9W7P0tLSpIbFy8urMYcHAAYqzNeJbC1MZT2hw+kF2j4cANCHUULcFbNmzRpav349JSUlSYFsSUmJjBpiEydOlC4blZkzZ9KWLVvovffeo+PHj9Nrr71GBw4coGeffVaeLy4ulhFEe/fupbNnz0q4GTVqFAUGBkpxLgCAmYkxDezkKh/EjhN1T6EAAIat0YFl7NixtGTJEpo/fz6FhIRQQkKCBBJVYW1qaqrMo6Jy22230ddff00ff/yxzNny/fff088//0w9evSQ57mLKTExke6//37q3LmzzNfCrTi7du2Srh8AAHZHF3e53nEiBx8IQBtkpDCA1cS46JZHCxUUFKCeBcBAZRWWUfjbUWRkRLT/1UhytcV/aADa0vc31hICAL3gYW9J3b3tif+L9fdJtLIAtDUILACgd91CG/efp+gT2ZRXjLWFANoKBBYA0Bt3BCkDy76UizT5s/10+zvb0doC0EYgsACA3ujTwZH+756uNKybB3Vwtqayyhqa+vkB2p2cq+1DA4AWhqJbANBLFVU1NP2rONqWlE2WZsa04ckICvFRTjAHAPoBRbcAYPDMTY1p5YQ+NKSLm7S0vPT9IQkxPPDx98QMijld/wKqAKB/dGItIQCAprAwNaH/PRJCkUt30smsYlq98zSVVFTRRzvPkIWpMe17ZSg5WivXIQIA/YYaFgDQa0425jT/vm5ye+lfJyWssPKqGvot8eoklgCg3xBYAEDv3R/sLV1DKhH+LnL9Q1yaFo8KAJoTAgsA6D0jIyN6Z0wverBPO1o1oQ8tf7Q3mRobUcL5fDqVXXTd9tU1CtqUeAHrEgHoEdSwAIDBzIS79JEQ9f0hXdxpW1IWfR+XTv+5qxMdzygiYyMjKiqrpHe2HKdDaQUyzf+vzwygnu0dtHrsAHBzGNYMAAZpy5EMevrLeLK1MJVgUlRWVed2YR2d6LunI6SVhvEoo/1nL5GvqzW521m28lEDtC2FWEsIANq6O4M8yMnajIrLqySsONuYk5eDJbnamkvX0W/PDiArMxM6cO6Suji3qrqGXvnpMD3yUQyN+2iv3Gc8+uj2RdspPvXSdT8nv7SCDpy9SCcyi+hSSYXGYo13Ld1Jkz+LpZoavV9jFkDr0CUEAAY7T8uKR/vQ7lO5sgYRt6QYGytbUVSmDwmg9/46Sf/9/Ril5JTQwfOXKPqEcmHFM7kl9PvhDAr3c5HRRzzHyzNfxdPvzw2U8MM4iDy0OoZOZRfLfd79s3d2kv0+9UUcJWcXy+WvpCwa3t3zlt9TeVU1ZReWk4+ztUYwsrEwlZYkAEOGLiEAaLPKKqtp6Hs7KT3/svoxnr/l9kBX2n48m4I87SjM14m+3Juqfn5wZzf6bHJfCT88Od34NXulwNfeyowuXmlh4ZacjIIy9Wt6tLOXFh1Vt1NTPf/tIfrxYBqtndxXQhgHpfuW76Yunnb00/Tbbnn/2saLWc7amEBudhYa9UhguNAlBADQAJZmJrT+sb703J2BNK6vDz3Yux1teLI/LX0kmKzNTeh4ZpE6rLx+f3dZAmDnyRxas0s518sP8cph0w+Htaf4eXfR4od6kZmJkYQVbm15f1yI7OdIeqG65abJf9jLKum3QxdIoSBasf2UPLbm7zN0ubJaRkOdyLp+NNSNghqvdq3q8tIF2YVlNO7jvbQrOZd+jE+n7KKrgQ+AYVgzALRpge52NHtYF1o0phctHRtCvTs4yey4j/broN7m9kAXmnSbL712X3e5/35UMqXkltAfh5W1L2P6tJfrR8J86Oup/am/vzO9+1AwjQppR//q31Ge+2B7shT0NtVfR7Oo4krAiDt3ibYezaSfDqarn+flCOpzNrdEHQC4G4u7q3i167d+TyJdUFBaKWGFu89Ujl0obPXjOH+xVOqOPt2d0uo/G24OgQUAoA6PD/QjcxPln8jZd3WR67F9fSi0oxOVVlTThDV7qaSiWlaN5sdU+vo6y0KMY0KVIeaJgX7SzXQwNZ+e/+6QtG40xW+JF+SaW2zYcxsOSoDhwmFVYKkrEB1JL6Bh//ubhrwbLfPOfPT3GWklYuv2nKV9Z/KotKKKFv1xnFbuOKXeB7d4fBCVTGdylCGCW2OWRyXTa78epaMXCpr0Hk7nFNfZcvJr4gWpGeKutH5+zvLYsYxbCyzn8koafZzfHjgvoentzUnyuYFuQZUWAEAdvBysaP1j/WTeFlUg4RoR7hq6b8VuunClRoVHHN2odoSHRi+4rzvN++WIdHXwfDCfTAojb0creZ7rXrjFhL/MeS/j+nYghyujm747cJ56tXcgf1db2p2cK9sveTiYpn8VLws+Mj6e//vliHzhcxdWVy97jQnyXv3psASbimqiJ9YfUD/H9Tm8/YvfJ6q7v1h7Jyu6p6cXPflFnHQ18QipV+/pKt1Re89cVAedfr7OtOpffcjF1kLj/XLgqevz4BFWj6yOkW64zx/vR306XA15B88pR189HOYjxxKbcpGO3kILCxcnczE0j9r6Y+ZA6uRh16DX/X3lM+bPbc6PifTz9NvJ9EpovRZvM3PDQakj4hqi3j6O9O8IXzK5prAbmg9aWAAA6hER4ELDrhnd06Odg9S7qDzYW9mSciOPhnegLx7vRy425tJyMHFtrAyH5taNwYt30NTPD0gLx8I/jtPQpdEyKinyvZ30+m/H6JGP9tKL3x+iqhqFhIy7e3rRoM7KZQh8nK0kMN1xZVmCa7uFvo5NlQny7CxM6Z5eXvIly5dRId707dMR0qKRerFUwoqqNYlbUN7efFzCCuPWpFd/OiJhxcbchO7q5iF1OrFnL9KH0ac1fh6PpPrXp/to6HvREvRqPz73h8PyHjiITfo0lg7WGiKuGi7ep4MjdbsSuJIaEVg48HEY+ueUMnDsOplLOUXl8vNWX1lb6mY43CSmKd8zj7jiuqPP/jlb7/Y/H0ynTYkZ8tn9knCBXvvtmARMXVVcXiW/c/oMgQUAoJFeGNaFerZzoPH9OlAHl6tDjG/ktgBX+uXZ2yUk8P/KuWaDg0tReRV1dLGm+4K9KcDNhnKLK6QrJrOwjOwsTSVgbEvKln3wNuzlEV1k5NGCe7tLCwCHGMbDsFVdOpkFZbR4y3Hl8Q7vQivG96b/u6erhK3/PtCT7C3NpEiYu6u4BSnq+cHSOnOptJLW/pOibs15bmgnmXjPz9WGfn7mdlozMYw+nhgmz2+ITaWCy1eDCf+8f07l0emcEvrjSKb68Y//Pi1FwTwcnFtm+D3ze+fuIR4ZdDavVLbr7eNE3byVgSUlr4RKyuue7O/arp8n1u+XAPX6b0fl/fOyCyq/JKRrjAKrDw9/54+us4ctzbu3qzz27tYTEoauxd167209Ibf/3b+jFGurWp5upU6ppaTnX5YQGf52lHwe+gpdQgAAjcTdIL/NGNDoz629k7V0Mz20ao+6C2ZokDutnNBHukq4JYJHIPHoo/t6edO0IQHSJbNsW7KEBn6Mdfd2oE0zBqr3O7SrhwQPLgTmrqenBwfQvz+NlQnzuEuJC3+5m+aJgf4axzOwk5uMbuJuGH7+3Yd60aiV/0hIGtnDk8Zc6e76V3gHKUTmuW3YkM5u1MXDTkLI1/tS5Th3HM+mT2oVq/IXIxch8zF9cGVU0/x7u9Gw7h704IfK9/9rwgXydbGR5wLdbaUrjHnYW1BWYTkdzyyk0I7KmhZVVw+39PDP4m4u7lbiz4dDFjuZVUxbj2XRX8ey5H47Ryv5subRVBzK+FhH9/amsX2vFlSr/H2lrmdQJzc5bg6JvJ8nPz8gQc3R2ozO5ZVKuOSRY9wlyPvn7rLyyhoJaPye+Pi4Za6huH7o/W3JMp/O4oeC5TPmz+yxdfvpodD29MwdgTfdx/dxaTJajAvFr+2S4tDHXYH8ebKZGxIoKaOIZkV2kt85fYJ5WAAAWtn+sxdp1oYEGhDoSm890IPM6qmTUOH6lYrqapm9tz7fxKbKLL38H3z+0uPww1+oPEy79kRzN8OtJn8n59Bbo3uqJ8ir70vyhe8OkbudBf3nrs6yPlN+aaUEHf7y5oAVM2cozf/liISIgZ1c6fPH+kkA+jzmLM3/5SgFt3eQOW+4a+mRsPbyhc2mfBZLO07k0JujutOo3u3ok7/P0L6Ui5SYViBfzNdSFety1wwHCz4Ofu9vP9iTJq2NvW77f/XvIHVFqs+dW0X6L4ySL3U+Ru5y4y/6h1fHSBced6mVVFQRT1jM78vEyEi6m3j4+4NXRohxrdBX+1JpeHcPaYHjwl0OeVwPZGVuIl1IfPx8fO72luTvaiPH+GH0KXULE8/vc0eQO737JxdAK7vbuBWMA1RNjYIqa2rIwtTkuqDFrVWMP+P3x/VWn7fKauVkh/z58wzP9/byllYgxnPdTLndlyK7ekjrmQTOXSnyMzhUcwscfy48EzR3RdpZKsOkNudhQWABANCC+opTb8XmwxkShLj1gVsCeIg1fym2BA5EAxdvV//PnXFrDq/L9OiafdKVogov/J/+P2cNUhe/5haXS/cEt+TwcXLLxcIHe0oXG1vy5wlaseOUdF9x11jtOWw4IA3t6i5fqNwNxEOieXVuvj/43R0SKhi3MnHX2b3Ld0sBL4+m4tcpu82UX+5c/MwB4GRWkYyk4laqQwuGqVseMgou0+iV/6jfo4OVmboLjLsEf3nmdvXsyclZRXTX//6W98pBqLyq8XPcTOah8/d3p1ErdkvtEeN6IQ4sUUnZEqLWTAqj/v4u6hanEct2SYuMiqe9JT1zRwD183Ohl35IpEPn86U+6Zsn+0srExdPc5iqPbEht8rwuVCZOzKInhocICF47o+HaUR3T1r971DSdmBBlxAAgBa0xKy0XMvi6WBJfx7NpMdu95MVrFsKt+I8NSiA3th0TP73zgGBu544AIwO8ZbAoqpj4S6Y2iN1XG0t6LYAF5kkjsMKqz1qSFXHwi0m3KLCX7gL7u8m9S8BbrbXLbGgMqKHJ20+rPyZ9/byks949b9C6Y8jGXR/cDv5bEYfy5LRPfyzZ397iN4fG0Lf7lcWy4b7u2h0k/BIsV+eGSDDozmgcMsIhy1eN4qLg2sfB78/DkG8Xw4rPCOyv5sN/XE4U4LFyJ5edGcXdyqtrKbMgsuUnFUsI8O4e4/3zeFiV3KOFMYmXhlSzS093ILCLTcq3L3zzdT+ssI4t4hwWOHWklUT+siIL2W34FH19vaWpvS/sSHqkW5cB8XLRHBw2XjgPB1NL5Dh+dyKFOrrJOGQW2H4XHItFdt6LFPCG38e2oQWFgAAaHIrEbde8BeztfnV///yUO1+/90m3SZcHxP94pDrVr7mOU9e+j5RbvOXJbdsqAIAT3Q3ZEm0etvn7+pMM4Z2uunx8MimMav2SH3N78/VvxQCd7FNWRdLldUKabHJLlK2oLz9QE8Z0dVUPNndG5uOSnDkYtyGhlJuten9xlZpHeLCaJ7Qr5O7rdRJvfLjYSqrqpb6pfUxZ6VGhruVOLgdTiuQ1rRlY0NodO92Ug/D4WvtP2dl9Bevn7VsXIjUTtWHu5q4zoe7kbilZcA726XwmydL5AJqldl3dZYC7OaGLiEAANAqLlbl2gku7pwV2bnOL+m+b22TL1xumfji8XCNL9Fer2+VobgcPviLW1XwezPcvcPDx6+dH+ZaXBTMBaiqwMRfxo8P8Ku39aalPfjhPxSfmq/uduL6Eq6zqa2orJImfLJPamFU7gxyp08nhWmEI+7e4ZFoXMjc2Hlhlm07KUXeKhH+LhRzJk+6Fne9dEezfz7oEgIAAK3imhT+X399q1TzF/MdQW7059EsmR24Nv5SjOzqLoFn4ZieDQ4rrHMDJ4njZROMjYxkJNLk2/ykW0WbuPuHA4uqRoZD3LXsLM2kO2hbUpbUyXCXH09Yd21LDocUnsyuKbgriIuguUbJ28GSPpoYSgMWbZdWGB76rZoDSBvQJQQAAFrB0///EJ9OEyM6ko2FZkklt7JwvQdP4tYWcM0Pd2epCm0T5g+77jNpLTyfDU+a997DwbLEBE8myHUtd/f0pA8nNG/xLVZrBgAAncdFrDyHS11fzNzK0lbCCuMh3lwgqypA1lZYYa/e3VW6f1TrYfEaWoznpeGiY23BTLcAAABaxjMWD+7iLreHXLnW5rH41Jq7h2dA5lYwnvmYZ1/WFnQJAQAA6ABe/2jL0UyZRO/aCeIMFYpuAQAA9AwX/vJwaKgbuoQAAABA5yGwAAAAgM5DYAEAAACdh8ACAAAAOg+BBQAAAAwzsKxcuZJ8fX3J0tKSwsPDKTY29obbf/fddxQUFCTb9+zZkzZv3nzdAlrz588nLy8vsrKyosjISEpOvrqWAQAAALRtjQ4sGzdupNmzZ9OCBQsoPj6egoODafjw4ZSdnV3n9nv27KHx48fT448/TgcPHqTRo0fL5ciRI+ptFi9eTB988AGtXr2a9u3bRzY2NrLPsrKyW3t3AAAAYBAaPXEct6j07duXVqxYIfdramrIx8eHZsyYQXPmzLlu+7Fjx1JJSQlt2rRJ/Vj//v0pJCREAgr/eG9vb3r++efphRdekOcLCgrIw8OD1q1bR+PGjWvWiWcAAABAN7TYWkIVFRUUFxcnXTbqHRgby/2YmJg6X8OP196eceuJavuUlBTKzMzU2IYPnoNRffssLy+XN1n7AgAAAIarUYElNzeXqqurpfWjNr7PoaMu/PiNtlddN2afCxculFCjunALDwAAABguvRwlNHfuXGk+Ul3Onz+v7UMCAAAAXQksrq6uZGJiQllZWRqP831PT886X8OP32h71XVj9mlhYSF9XbUvAAAAYLgaFVjMzc0pNDSUoqKi1I9x0S3fj4iIqPM1/Hjt7dlff/2l3t7Pz0+CSe1tuCaFRwvVt08AAABoW0wb+wIe0jxp0iQKCwujfv360bJly2QU0JQpU+T5iRMnUrt27aTOhM2cOZMGDx5M7733Ht1zzz20YcMGOnDgAH388cfyvJGREc2aNYveeust6tSpkwSYefPmycghHv7cEKqBTii+BQAA0B+q7+0GDVhWNMHy5csVHTp0UJibmyv69eun2Lt3r/q5wYMHKyZNmqSx/bfffqvo3LmzbN+9e3fF77//rvF8TU2NYt68eQoPDw+FhYWFYujQoYoTJ040+HjOnz/P7xQXfAb4HcDvAH4H8DuA3wHSv8+Av8dvptHzsOgi7pa6cOEC2dnZSYtNc6c/HoXEhb2GWitj6O/R0N8fw3vUfziHhsHQz2NhM78/jiBFRUXSq8LTpDRrl5Au4jfZvn37Fv0ZbaG419Dfo6G/P4b3qP9wDg2DoZ9H+2Z8fzw9icEOawYAAIC2BYEFAAAAdB4Cy03wnC+80CNfGypDf4+G/v4Y3qP+wzk0DIZ+Hi20+P4MougWAAAADBtaWAAAAEDnIbAAAACAzkNgAQAAAJ2HwAIAAAA6D4HlJlauXEm+vr5kaWlJ4eHhFBsbS/qI13bq27evzAbs7u4u6zSdOHFCY5shQ4bITMG1L08//TTpi9dee+264w8KClI/X1ZWRs888wy5uLiQra0tjRkz5rpVwnUZ/x5e+/74wu9JX8/f33//Tffdd5/McsnH+/PPP2s8z2MC5s+fT15eXmRlZUWRkZGUnJyssc3FixdpwoQJMomVo6MjPf7441RcXEz68B4rKyvp5Zdfpp49e5KNjY1sw+ux8czdNzv3ixYtIn04h5MnT77u2EeMGGEw55DV9e+SL++++65enMOFDfh+aMjfz9TUVFkz0NraWvbz4osvUlVVVbMdJwLLDWzcuFEWe+QhXPHx8RQcHEzDhw+n7Oxs0jc7d+6UX7a9e/fKatn8h3LYsGGycGVtU6dOpYyMDPVl8eLFpE+6d++ucfy7d+9WP/ef//yHfvvtN/ruu+/k8+AvhQcffJD0xf79+zXeG59H9vDDD+vt+ePfP/53xf8xqAsf/wcffECrV6+WFdz5S53/DfIfTxX+ojt69Kh8Hps2bZIvlyeffJL04T2WlpbK3xZe8JWvf/zxR/miuP/++6/b9o033tA4tzNmzCB9OIeMA0rtY//mm280ntfnc8hqvze+rF27VgIJf6nrwznc2YDvh5v9/ayurpawUlFRQXv27KH169fTunXr5D8czabBKwy2Qbyw4zPPPKO+X11drfD29lYsXLhQoe+ys7NlwamdO3dqLFw5c+ZMhb5asGCBIjg4uM7n8vPzFWZmZorvvvtO/VhSUpJ8BjExMQp9xOcqICBAFg81hPPH5+Knn35S3+f35enpqXj33Xc1ziMvkPrNN9/I/WPHjsnr9u/fr97mjz/+UBgZGSnS09MVuv4e6xIbGyvbnTt3Tv1Yx44dFf/73/8Uuq6u98eL4Y4aNare1xjiOeT3e+edd2o8pi/nsK7vh4b8/dy8ebPC2NhYkZmZqd5m1apVCnt7e0V5ebmiOaCFpR6cEuPi4qQJuvaaRXw/JiaG9F1BQYFcOzs7azz+1VdfkaurK/Xo0YPmzp0r/wPUJ9xdwM22/v7+8r82bqJkfC75fw21zyd3F3Xo0EEvzyf/fn755Zf02GOPaSz4qe/nr7aUlBTKzMzUOGe85gh3zarOGV9zF0JYWJh6G96e/61yi4y+/tvkc8rvqzbuPuDm+N69e0tXQ3M2tbe06Oho6SLo0qULTZs2jfLy8tTPGdo55G6S33//Xbq1rqUv57Dgmu+Hhvz95Gvu2vTw8FBvw62hvFgit541B4NY/LAl5ObmShNX7Q+f8f3jx4+Tvq9uPWvWLLr99tvli03l0UcfpY4dO8oXfmJiovStc/M0N1PrA/4i4yZI/qPIza2vv/46DRw4kI4cOSJffObm5td9CfD55Of0Dfeh5+fnS32AoZy/a6nOS13/BlXP8TV/EdZmamoqf2j18bxyVxeft/Hjx2ssLPfcc89Rnz595H1xczuHUf4dX7p0Kek67g7irgM/Pz86ffo0vfLKKzRy5Ej5gjMxMTG4c8hdIVwLcm13s76cw5o6vh8a8veTr+v6t6p6rjkgsLRB3FfJX+K16ztY7T5jTspc6Dh06FD5IxMQEEC6jv8IqvTq1UsCDH+Bf/vtt1KwaUg+/fRTeb8cTgzl/LV1/D/YRx55RAqNV61apfEc19LV/t3mL4+nnnpKiiV1fQr4cePGafxe8vHz7yO3uvDvp6Hh+hVu3eWBGvp4Dp+p5/tBF6BLqB7crM7p/9oqaL7v6elJ+urZZ5+VorYdO3ZQ+/btb7gtf+GzU6dOkT7i/w107txZjp/PGXejcKuEvp/Pc+fO0bZt2+iJJ54w6POnOi83+jfI19cWwXMzO4860afzqgorfG656LF260p955bf59mzZ0nfcHct/31V/V4ayjlku3btklbNm/3b1NVz+Gw93w8N+fvJ13X9W1U91xwQWOrB6Tc0NJSioqI0msr4fkREBOkb/l8b/zL+9NNPtH37dmmevZmEhAS55v+p6yMeFsmtC3z8fC7NzMw0zif/YeEaF307n5999pk0oXNFviGfP/4d5T90tc8Z94dzXYPqnPE1/xHlPnYV/v3mf6uqwKYvYYXrrziIco3DzfC55RqPa7tS9EFaWprUsKh+Lw3hHNZu+eS/NTyiSJ/OoeIm3w8N+fvJ14cPH9YIn6rw3a1bt2Y7UKjHhg0bZETCunXrpJL9ySefVDg6OmpUQeuLadOmKRwcHBTR0dGKjIwM9aW0tFSeP3XqlOKNN95QHDhwQJGSkqL45ZdfFP7+/opBgwYp9MXzzz8v74+P/59//lFERkYqXF1dpeKdPf3004oOHTootm/fLu8zIiJCLvqER6rxe3j55Zc1HtfX81dUVKQ4ePCgXPjP0dKlS+W2aoTMokWL5N8cv5/ExEQZfeHn56e4fPmyeh8jRoxQ9O7dW7Fv3z7F7t27FZ06dVKMHz9eoQ/vsaKiQnH//fcr2rdvr0hISND4t6kaWbFnzx4ZXcLPnz59WvHll18q3NzcFBMnTlTo+vvj51544QUZScK/l9u2bVP06dNHzlFZWZlBnEOVgoIChbW1tYyMuZaun8NpN/l+aMjfz6qqKkWPHj0Uw4YNk/e5ZcsWeY9z585ttuNEYLmJ5cuXy0kyNzeXYc579+5V6CP+R1bX5bPPPpPnU1NT5cvN2dlZQlpgYKDixRdflH+E+mLs2LEKLy8vOVft2rWT+/xFrsJfctOnT1c4OTnJH5YHHnhA/lHqkz///FPO24kTJzQe19fzt2PHjjp/L3korGpo87x58xQeHh7yvoYOHXrde8/Ly5MvN1tbWxlCOWXKFPmC0Yf3yF/i9f3b5NexuLg4RXh4uHyhWFpaKrp27ap4++23Nb7wdfX98Rcef4HxFxcPi+WhvVOnTr3uP336fA5VPvroI4WVlZUMAb6Wrp9Dusn3Q0P/fp49e1YxcuRI+Rz4P4v8n8jKyspmO06jKwcLAAAAoLNQwwIAAAA6D4EFAAAAdB4CCwAAAOg8BBYAAADQeQgsAAAAoPMQWAAAAEDnIbAAAACAzkNgAQAAAJ2HwAIAAAA6D4EFAAAAdB4CCwAAAOg8BBYAAAAgXff/5IE2ZVbUTagAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t_lossi = torch.tensor(lossi).cpu()\n",
    "plt.plot(t_lossi.view(-1, 1000).mean(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e44944",
   "metadata": {},
   "source": [
    "### Génération"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "5b5844e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#g = torch.Generator(device='mps').manual_seed(seed + 10)\n",
    "g = torch.manual_seed(seed + 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "83d34e89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avenue\n",
      "qu'avec\n",
      "impératives\n",
      "sport\n",
      "présumés\n",
      "cultif\n",
      "leurs\n",
      "sciences\n",
      "liquidation\n",
      "citre\n",
      "représentement\n",
      "judiciaire\n",
      "soumettant\n",
      "exceptionnés\n",
      "bâti\n",
      "mollaté\n",
      "typoser\n",
      "bordereaux\n",
      "peutiennent\n",
      "sécurisée\n"
     ]
    }
   ],
   "source": [
    "for _ in range(20):\n",
    "    word = nn.generate_word(words.itoc, g)\n",
    "    print(word)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cours_nlp_mines (3.14.2)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.14.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}