{
"cells": [
{
"cell_type": "markdown",
"id": "aac3243f-7d89-48fe-8c14-d58e5bb43636",
"metadata": {},
"source": [
"# `micrograd`: construction d'une bibliothèque de rétropropagation du gradient (partie 2)\n",
"\n",
"Ce notebook est dérivé du travail d'[Andrej Karpathy](https://karpathy.ai/) et reprend pas à pas les étapes exposées dans sa première séance de cours sur la construction d'un outil en Python pour le calcul du gradient et sa propagation arrière:\n",
"\n",
"[The spelled-out intro to neural networks and backpropagation: building micrograd](https://www.youtube.com/watch?v=VMj-3S1tku0&list=PLAqhIrjkxbuWI23v9cThsA9GvCAUhRvKZ)\n",
"\n",
"Voici les ressources originales associées à la vidéo youtube d'Andrej:\n",
"\n",
"- micrograd on github: https://github.com/karpathy/micrograd\n",
"- notebook original: https://github.com/karpathy/nn-zero-to-hero/tree/master/lectures/micrograd\n",
"- exercices: https://colab.research.google.com/drive/1FPTx1RXtBfc4MaTkf7viZZD4U2F9gtKN?usp=sharing"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "9b4fcf50-1497-420d-bf83-509712e79037",
"metadata": {},
"outputs": [],
"source": [
"# Imports de la librairie standard Python\n",
"import math\n",
"# Imports spécifiques (doivent être présent dans l'environnement Python de ce notebook)\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"from graphviz import Digraph"
]
},
{
"cell_type": "markdown",
"id": "c9f63afd-ea76-4abc-8b63-c8221cb9d0af",
"metadata": {},
"source": [
"## Amélioration de classe `Value`"
]
},
{
"cell_type": "markdown",
"id": "7708f490-1a9d-4aed-b797-59f1429794c0",
"metadata": {},
"source": [
"### Rappels"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "fc9472e1-f18b-49c6-a482-4bcfb3f6177e",
"metadata": {},
"outputs": [],
"source": [
"class Value:\n",
" \n",
" def __init__(self, data, children=(), op='', label=''):\n",
" self.data = data\n",
" self._prev = set(children)\n",
" self._op = op\n",
" self.label = label\n",
" self.grad = 0.0\n",
" self._backward = lambda: None\n",
"\n",
" def __repr__(self):\n",
" return f\"Value(data={self.data}, label={self.label}, grad={self.grad})\"\n",
"\n",
" def __add__(self, other):\n",
" out = self.__class__(self.data + other.data, children=(self, other), op='+')\n",
" def _backward():\n",
" self.grad = out.grad \n",
" other.grad = out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def __mul__(self, other):\n",
" out = self.__class__(self.data * other.data, children=(self, other), op='*')\n",
" def _backward():\n",
" self.grad = other.data * out.grad \n",
" other.grad = self.data * out.grad\n",
" out._backward = _backward\n",
" return out\n",
" \n",
" def tanh(self):\n",
" x = self.data\n",
" t = (math.exp(2*x) - 1) / (math.exp(2*x) + 1)\n",
" out = Value(t, children=(self,), op='tanh')\n",
" def _backward():\n",
" self.grad = (1 - t**2) * out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def backward(self):\n",
" topo = []\n",
" visited = set()\n",
" def build_topo(v):\n",
" if v not in visited:\n",
" visited.add(v)\n",
" for child in v._prev:\n",
" build_topo(child)\n",
" topo.append(v)\n",
" build_topo(self)\n",
" self.grad = 1.0\n",
" for node in reversed(topo):\n",
" node._backward()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "4c76d359-a934-4614-9e9f-d6ec723cd1f0",
"metadata": {},
"outputs": [],
"source": [
"a = Value(2.0, label='a')\n",
"b = Value(-3.0, label='b')\n",
"c = Value(10.0, label='c')\n",
"e = a * b\n",
"e.label = 'e'\n",
"d = e + c\n",
"d.label = 'd'\n",
"f = Value(-2.0, label='f')\n",
"L = d * f\n",
"L.label = 'L'"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "18501ff1-1ad0-4b52-be7b-c69cac8b3640",
"metadata": {},
"outputs": [],
"source": [
"def trace(root):\n",
" # builds a set of all nodes and edges in a graph\n",
" nodes, edges = set(), set()\n",
" def build(v):\n",
" if v not in nodes:\n",
" nodes.add(v)\n",
" for child in v._prev:\n",
" edges.add((child, v))\n",
" build(child)\n",
" build(root)\n",
" return nodes, edges\n",
"\n",
"def draw_dot(root):\n",
" dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) # LR = left to right\n",
" \n",
" nodes, edges = trace(root)\n",
" for n in nodes:\n",
" uid = str(id(n))\n",
" # for any value in the graph, create a rectangular ('record') node for it\n",
" dot.node(name = uid, label = \"{ %s | data %.4f | grad %.4f }\" % (n.label, n.data, n.grad), shape='record')\n",
" if n._op:\n",
" # if this value is a result of some operation, create an op node for it\n",
" dot.node(name = uid + n._op, label = n._op)\n",
" # and connect this node to it\n",
" dot.edge(uid + n._op, uid)\n",
"\n",
" for n1, n2 in edges:\n",
" # connect n1 to the op node of n2\n",
" dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
"\n",
" return dot"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "9276f95c-5c4f-4a38-820b-2520cd14d82c",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"draw_dot(L)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "e883cef2-cd0d-42b6-bc0e-3a924f844853",
"metadata": {},
"outputs": [],
"source": [
"L.backward()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "b231c653-ec5a-430e-919b-adff3eb1c154",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"draw_dot(L)"
]
},
{
"cell_type": "markdown",
"id": "f159f11b-9fc3-45b0-9dcd-2dcdf175e52d",
"metadata": {},
"source": [
"### Problème de non-cumul des gradients"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "6a97bb87-1bb2-4bcd-bb31-9ccda55dae56",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"a = Value(-2.0, label='a')\n",
"b = Value(3.0, label='b')\n",
"d = a * b ; d.label = 'd'\n",
"e = a + b ; e.label = 'e'\n",
"f = d * e ; f.label = 'f'\n",
"\n",
"f.backward()\n",
"\n",
"draw_dot(f)"
]
},
{
"cell_type": "markdown",
"id": "cfb483e7-0d21-4486-9c2b-ddc6eb47a9f9",
"metadata": {},
"source": [
"Dans ce graphe un peu particulier, les valeurs des gradients pour $a$ et $b$ sont fausses. En effet, il faut cumuler ces gradients lors du calcul, voir [multivariable case (chain rule)](https://en.wikipedia.org/wiki/Chain_rule#Multivariable_case)."
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "8026b006-9757-47e4-8dba-5135868f022f",
"metadata": {},
"outputs": [],
"source": [
"class Value:\n",
" \n",
" def __init__(self, data, children=(), op='', label=''):\n",
" self.data = data\n",
" self._prev = set(children)\n",
" self._op = op\n",
" self.label = label\n",
" self.grad = 0.0\n",
" self._backward = lambda: None\n",
"\n",
" def __repr__(self):\n",
" return f\"Value(data={self.data}, label={self.label}, grad={self.grad})\"\n",
"\n",
" def __add__(self, other):\n",
" out = self.__class__(self.data + other.data, children=(self, other), op='+')\n",
" def _backward():\n",
" self.grad += out.grad \n",
" other.grad += out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def __mul__(self, other):\n",
" out = self.__class__(self.data * other.data, children=(self, other), op='*')\n",
" def _backward():\n",
" self.grad += other.data * out.grad \n",
" other.grad += self.data * out.grad\n",
" out._backward = _backward\n",
" return out\n",
" \n",
" def tanh(self):\n",
" x = self.data\n",
" t = (math.exp(2*x) - 1) / (math.exp(2*x) + 1)\n",
" out = Value(t, children=(self,), op='tanh')\n",
" def _backward():\n",
" self.grad += (1 - t**2) * out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def backward(self):\n",
" topo = []\n",
" visited = set()\n",
" def build_topo(v):\n",
" if v not in visited:\n",
" visited.add(v)\n",
" for child in v._prev:\n",
" build_topo(child)\n",
" topo.append(v)\n",
" build_topo(self)\n",
" self.grad = 1.0\n",
" for node in reversed(topo):\n",
" node._backward()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "24e15b37-7ff3-47ff-a8a8-b5c5eb46f702",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"a = Value(-2.0, label='a')\n",
"b = Value(3.0, label='b')\n",
"d = a * b ; d.label = 'd'\n",
"e = a + b ; e.label = 'e'\n",
"f = d * e ; f.label = 'f'\n",
"f.backward()\n",
"\n",
"draw_dot(f)"
]
},
{
"cell_type": "markdown",
"id": "74651552-874a-49bb-94cc-78df095fdfe0",
"metadata": {},
"source": [
"### Ajout de nouvelles fonctions"
]
},
{
"cell_type": "markdown",
"id": "b0765900-28eb-4a15-bdc8-6f57cd6f0595",
"metadata": {},
"source": [
"#### Support des constantes\n",
"\n",
"Pour `__add__` et `__mul__`, on ajoute une instruction permettant d'utiliser des constructions comme `b = a + 1`, en transformant `1` en `Value(1.0)`."
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "dee8092c-3f47-4e8c-af94-a939a4160fa3",
"metadata": {},
"outputs": [],
"source": [
"class Value:\n",
" \n",
" def __init__(self, data, children=(), op='', label=''):\n",
" self.data = data\n",
" self._prev = set(children)\n",
" self._op = op\n",
" self.label = label\n",
" self.grad = 0.0\n",
" self._backward = lambda: None\n",
"\n",
" def __repr__(self):\n",
" return f\"Value(data={self.data}, label={self.label}, grad={self.grad})\"\n",
"\n",
" def __add__(self, other):\n",
" other = other if isinstance(other, Value) else Value(other) # a + 1\n",
" out = self.__class__(self.data + other.data, children=(self, other), op='+')\n",
" def _backward():\n",
" self.grad += out.grad \n",
" other.grad += out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def __mul__(self, other):\n",
" other = other if isinstance(other, Value) else Value(other) # a * 1\n",
" out = self.__class__(self.data * other.data, children=(self, other), op='*')\n",
" def _backward():\n",
" self.grad += other.data * out.grad \n",
" other.grad += self.data * out.grad\n",
" out._backward = _backward\n",
" return out\n",
" \n",
" def tanh(self):\n",
" x = self.data\n",
" t = (math.exp(2*x) - 1) / (math.exp(2*x) + 1)\n",
" out = Value(t, children=(self,), op='tanh')\n",
" def _backward():\n",
" self.grad += (1 - t**2) * out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def backward(self):\n",
" topo = []\n",
" visited = set()\n",
" def build_topo(v):\n",
" if v not in visited:\n",
" visited.add(v)\n",
" for child in v._prev:\n",
" build_topo(child)\n",
" topo.append(v)\n",
" build_topo(self)\n",
" self.grad = 1.0\n",
" for node in reversed(topo):\n",
" node._backward()"
]
},
{
"cell_type": "markdown",
"id": "d6d96b18-a1b1-4253-ba49-b19ef08dee15",
"metadata": {},
"source": [
"#### Commutativité\n",
"\n",
"Pour `__add__` et `__mul__`, support indiférencié de `a + 1` ou `1 + a`, en déclarant les méthodes [`__radd__`](https://docs.python.org/3.12/reference/datamodel.html#object.__radd__) et [`__rmul__`](https://docs.python.org/3.12/reference/datamodel.html#object.__rmul__)."
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "7702d34b-d9f1-4515-adea-6f07eec841bf",
"metadata": {},
"outputs": [],
"source": [
"def __rmul__(self, other): # other * self\n",
" return self * other\n",
"Value.__rmul__ = __rmul__"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "efc1800f-0ff1-4158-9da3-c871aec99f81",
"metadata": {},
"outputs": [],
"source": [
"def __radd__(self, other): # other * self\n",
" return self + other\n",
"Value.__radd__ = __radd__"
]
},
{
"cell_type": "markdown",
"id": "f71f7daa-816a-4eb6-8267-da096aa73a53",
"metadata": {},
"source": [
"#### Implémentation de l'exponentielle, de la négation, de la puissance et de la division"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "a1dcf23d-9f83-47e3-a612-96a2df00f01d",
"metadata": {},
"outputs": [],
"source": [
"class Value:\n",
" \n",
" def __init__(self, data, children=(), op='', label=''):\n",
" self.data = data\n",
" self._prev = set(children)\n",
" self._op = op\n",
" self.label = label\n",
" self.grad = 0.0\n",
" self._backward = lambda: None\n",
"\n",
" def __repr__(self):\n",
" return f\"Value(data={self.data}, label={self.label}, grad={self.grad})\"\n",
"\n",
" def __add__(self, other):\n",
" other = other if isinstance(other, Value) else Value(other) # a + 1\n",
" out = self.__class__(self.data + other.data, children=(self, other), op='+')\n",
" def _backward():\n",
" self.grad += out.grad \n",
" other.grad += out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def __mul__(self, other):\n",
" other = other if isinstance(other, Value) else Value(other) # a * 1\n",
" out = self.__class__(self.data * other.data, children=(self, other), op='*')\n",
" def _backward():\n",
" self.grad += other.data * out.grad \n",
" other.grad += self.data * out.grad\n",
" out._backward = _backward\n",
" return out\n",
" \n",
" def tanh(self):\n",
" x = self.data\n",
" t = (math.exp(2*x) - 1) / (math.exp(2*x) + 1)\n",
" out = Value(t, children=(self,), op='tanh')\n",
" def _backward():\n",
" self.grad += (1 - t**2) * out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def __rmul__(self, other): # other * self\n",
" return self * other\n",
"\n",
" def __radd__(self, other): # other * self\n",
" return self + other\n",
"\n",
" def __pow__(self, other):\n",
" assert isinstance(other, (int, float)), \"only supporting int/float powers for now\"\n",
" out = Value(self.data**other, (self,), f'**{other}')\n",
" def _backward():\n",
" self.grad += other * (self.data ** (other - 1)) * out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def __truediv__(self, other): # self / other\n",
" return self * other**-1\n",
"\n",
" def __neg__(self): # -self\n",
" return self * -1\n",
"\n",
" def __sub__(self, other): # self - other\n",
" return self + (-other)\n",
" \n",
" def exp(self):\n",
" x = self.data\n",
" out = Value(math.exp(x), (self, ), 'exp')\n",
" def _backward():\n",
" self.grad += out.data * out.grad\n",
" out._backward = _backward\n",
" return out\n",
"\n",
" def backward(self):\n",
" topo = []\n",
" visited = set()\n",
" def build_topo(v):\n",
" if v not in visited:\n",
" visited.add(v)\n",
" for child in v._prev:\n",
" build_topo(child)\n",
" topo.append(v)\n",
" build_topo(self)\n",
" self.grad = 1.0\n",
" for node in reversed(topo):\n",
" node._backward()"
]
},
{
"cell_type": "markdown",
"id": "4054b683-b1a3-4ccc-8b6b-02e3a511fb21",
"metadata": {},
"source": [
"#### Exemples d'utilisation"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "05d20ec2-5ef3-4d88-a0cd-7226eedcbf23",
"metadata": {},
"outputs": [],
"source": [
"# inputs x1,x2\n",
"x1 = Value(2.0, label='x1')\n",
"x2 = Value(0.0, label='x2')\n",
"# weights w1,w2\n",
"w1 = Value(-3.0, label='w1')\n",
"w2 = Value(1.0, label='w2')\n",
"# bias of the neuron\n",
"b = Value(6.8813735870195432, label='b')\n",
"# x1*w1 + x2*w2 + b\n",
"x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
"x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
"x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n",
"n = x1w1x2w2 + b; n.label = 'n'\n",
"o = n.tanh(); o.label = 'o'\n",
"o.backward()"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "a802d359-094a-44d3-90bb-7e4b4ee76330",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"draw_dot(o)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "d8ff9996-090a-44c8-bfeb-4994da955169",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# inputs x1,x2\n",
"x1 = Value(2.0, label='x1')\n",
"x2 = Value(0.0, label='x2')\n",
"# weights w1,w2\n",
"w1 = Value(-3.0, label='w1')\n",
"w2 = Value(1.0, label='w2')\n",
"# bias of the neuron\n",
"b = Value(6.8813735870195432, label='b')\n",
"# x1*w1 + x2*w2 + b\n",
"x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
"x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
"x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n",
"n = x1w1x2w2 + b; n.label = 'n'\n",
"# ----\n",
"e = (2*n).exp()\n",
"o = (e - 1) / (e + 1)\n",
"# ----\n",
"o.label = 'o'\n",
"o.backward()\n",
"draw_dot(o)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "344d845f-858b-4011-95e0-d8bfb2c691c6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.7071067811865477\n"
]
}
],
"source": [
"print(o.data)"
]
},
{
"cell_type": "markdown",
"id": "71999e0a-66f4-4541-b59a-df1e922f7c2a",
"metadata": {},
"source": [
"#### Expression équivalente avec Pytorch\n",
"\n",
"Pour que cette section fonctionne, il est nécessaire que PyTorch soit installé sur la machine faisant tourner Jupyter (`pip install torch`)."
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "d1f3395d-c610-45f7-9d58-2eb5688cf21c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.7071066904050358\n",
"---\n",
"x2 0.5000001283844369\n",
"w2 0.0\n",
"x1 -1.5000003851533106\n",
"w1 1.0000002567688737\n"
]
}
],
"source": [
"import torch\n",
"x1 = torch.Tensor([2.0]).double() ; x1.requires_grad = True\n",
"x2 = torch.Tensor([0.0]).double() ; x2.requires_grad = True\n",
"w1 = torch.Tensor([-3.0]).double() ; w1.requires_grad = True\n",
"w2 = torch.Tensor([1.0]).double() ; w2.requires_grad = True\n",
"b = torch.Tensor([6.8813735870195432]).double() ; b.requires_grad = True\n",
"n = x1*w1 + x2*w2 + b\n",
"o = torch.tanh(n)\n",
"\n",
"print(o.data.item())\n",
"o.backward()\n",
"\n",
"print('---')\n",
"print('x2', x2.grad.item())\n",
"print('w2', w2.grad.item())\n",
"print('x1', x1.grad.item())\n",
"print('w1', w1.grad.item())"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "5b8c8cca-1367-4893-9347-4fd3831009b7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor([2.], dtype=torch.float64, requires_grad=True)\n"
]
}
],
"source": [
"print(x1)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "d76976b2-a905-4adb-a27f-1b83f4443759",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor([-1.5000], dtype=torch.float64)\n"
]
}
],
"source": [
"print(x1.grad)"
]
},
{
"cell_type": "markdown",
"id": "61c56de4-b7cb-4287-ab7c-b9279a4eeea8",
"metadata": {},
"source": [
"# MLP: Multi Layer Perceptron\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "ab91f4c8-2c6f-42d9-8f91-48c402811f06",
"metadata": {},
"source": [
"## Modélisation d'un neurone\n",
"\n",
"Dessin issu de [\"_neural networks: representation_\"](https://www.jeremyjordan.me/intro-to-neural-networks/) de Jeremy Jordan.\n",
"\n",
"\n",
"\n",
"Source: \n",
"\n",
"Par défaut, dans la classe `Neuron` ci-dessous, on initialise les poids avec une valeur aléatoire (distribution de probabilité uniforme) comprise entre -1 et 1."
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "0039461a-a52c-4599-84dd-f0103abdd35e",
"metadata": {},
"outputs": [],
"source": [
"import random\n",
"\n",
"class Neuron:\n",
" \n",
" def __init__(self, nin):\n",
" self.w = [Value(random.uniform(-1,1), label=f'w{i}') for i in range(nin)]\n",
" self.b = Value(random.uniform(-1,1), label='b')\n",
" \n",
" def __call__(self, x):\n",
" # tanh(w * x + b)\n",
" act = sum((wi*xi for wi, xi in zip(self.w, x)), self.b)\n",
" out = act.tanh() # Fonction d'activation\n",
" out.label = 'out'\n",
" return out\n",
"\n",
" def parameters(self):\n",
" return self.w + [self.b]"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "03a75eb4-8fdb-49d6-9365-9de36f56f232",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Value(data=-0.8514542857879747, label=out, grad=0.0)"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"n = Neuron(5)\n",
"n((0.2,0.2,0.3,0.4,0.5))"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "9445d1c1-6394-4245-adf3-08bc5fa10ec4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Value(data=-0.8039598110691475, label=w0, grad=0.0),\n",
" Value(data=0.5874188380995722, label=w1, grad=0.0),\n",
" Value(data=-0.2589112549460806, label=w2, grad=0.0),\n",
" Value(data=-0.9122108656376093, label=w3, grad=0.0),\n",
" Value(data=0.24446900172649078, label=w4, grad=0.0),\n",
" Value(data=-0.8977855630313485, label=b, grad=0.0)]"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"n.parameters()"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "bd63319c-97df-44f4-ac10-178d84350b72",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"draw_dot(n((0.2,0.2,0.3,0.4,0.5)))"
]
},
{
"cell_type": "markdown",
"id": "5aeff752-f9bf-4df0-84ad-aebbc2bb0327",
"metadata": {},
"source": [
"## Couche de neurones"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "2224689f-12e3-4065-92e3-7d9e3924b780",
"metadata": {},
"outputs": [],
"source": [
"class Layer:\n",
" \n",
" def __init__(self, nin, nout):\n",
" self.neurons = [Neuron(nin) for _ in range(nout)]\n",
" \n",
" def __call__(self, x):\n",
" outs = [n(x) for n in self.neurons]\n",
" return outs[0] if len(outs) == 1 else outs\n",
" \n",
" def parameters(self):\n",
" return [p for neuron in self.neurons for p in neuron.parameters()]"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "85b278ad-0e4c-4205-a316-89c9ed69493f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Value(data=0.3213930604635117, label=out, grad=0.0),\n",
" Value(data=0.31464203078000524, label=out, grad=0.0),\n",
" Value(data=-0.858850222206831, label=out, grad=0.0)]"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"l = Layer(2,3)\n",
"l((0.2,0.3))\n"
]
},
{
"cell_type": "markdown",
"id": "2f015bf5-7fc3-40b1-9fe8-3b710e74c9f1",
"metadata": {},
"source": [
"## Multicouches"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "3b6b332f-7d42-4dc1-8dd7-65ba6af003dd",
"metadata": {},
"outputs": [],
"source": [
"class MLP:\n",
" \n",
" def __init__(self, nin, nouts):\n",
" sz = [nin] + nouts\n",
" self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n",
" \n",
" def __call__(self, x):\n",
" for layer in self.layers:\n",
" x = layer(x)\n",
" return x\n",
" \n",
" def parameters(self):\n",
" return [p for layer in self.layers for p in layer.parameters()]"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "578dc4dc-2fe7-4a60-8c14-3f97ebe2ead2",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Value(data=0.5428321376017807, label=out, grad=0.0)"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"x = [2.0, 3.0, -1.0]\n",
"n = MLP(3, [4, 4, 1])\n",
"n(x)"
]
},
{
"cell_type": "markdown",
"id": "854a13e3-27f8-4f55-a64d-ba7f9acc31e4",
"metadata": {},
"source": [
"https://cs231n.github.io/neural-networks-1/\n",
"\n",
"\n",
"\n",
"Source: "
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "1567f2cc-fc90-45c7-88cb-8749006cad89",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"draw_dot(n(x))"
]
},
{
"cell_type": "markdown",
"id": "7312b83a-a979-4d3a-a735-014ac0830795",
"metadata": {},
"source": [
"# Apprentissage"
]
},
{
"cell_type": "markdown",
"id": "22c839fe-3908-4039-a07f-207bf44e52b0",
"metadata": {},
"source": [
"## Définition d'une fonction de perte"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "90937f75-36ae-4760-b66e-898f2f2e02b0",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Value(data=0.5428321376017807, label=out, grad=0.0),\n",
" Value(data=0.8825476224698201, label=out, grad=0.0),\n",
" Value(data=0.24590509211930503, label=out, grad=0.0),\n",
" Value(data=0.4050940086029426, label=out, grad=0.0)]"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Jeu d'entrainement\n",
"xs = [\n",
" [2.0, 3.0, -1.0], # exemple 1\n",
" [3.0, -1.0, 0.5], # exemple 2\n",
" [0.5, 1.0, 1.0], # exemple 3\n",
" [1.0, 1.0, -1.0], # exemple 4\n",
"]\n",
"ys = [1.0, -1.0, -1.0, 1.0] # desired targets\n",
"ypred = [n(x) for x in xs] # sortie \n",
"ypred"
]
},
{
"cell_type": "markdown",
"id": "ebc29b6f-9a64-431c-8c0d-7a78271e94d8",
"metadata": {},
"source": [
"À ce stade, les valeurs de sortie du réseau ne sont pas bonnes, ce qui est normal car nous n'avons réglé aucun paramètre.\n",
"Nous allons définir une fonction que nous allons chercher à optimiser, ici à minimiser: une fonction de perte utilisant l'erreur quadratique moyenne."
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "c53ebd96-9de4-4f1c-a431-a277160f3d80",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Value(data=0.20900245440975715, label=, grad=0.0),\n",
" Value(data=3.5439855508667724, label=, grad=0.0),\n",
" Value(data=1.5522794985688142, label=, grad=0.0),\n",
" Value(data=0.3539131386001157, label=, grad=0.0)]"
]
},
"execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"[(yout - ygt)**2 for ygt, yout in zip(ys, ypred)]"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "504d1e45-2c88-4b7d-a016-651cca2afb1d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Value(data=5.65918064244546, label=, grad=0.0)"
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n",
"loss"
]
},
{
"cell_type": "markdown",
"id": "3dd67929-0ab4-4307-a105-8c4802268e8c",
"metadata": {},
"source": [
"C'est cette valeur que l'on va chercher à minimiser."
]
},
{
"cell_type": "markdown",
"id": "d42f3ea1-c6a4-498e-942d-4b79f818c47f",
"metadata": {},
"source": [
"## Reprise des étapes: apprentissage manuel"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "6aed4665-edf3-4994-948d-681828b3f584",
"metadata": {},
"outputs": [],
"source": [
"# Architecture de notre réseau\n",
"n = MLP(3, [4, 4, 1])\n",
"\n",
"# Données exemple\n",
"xs = [\n",
" [2.0, 3.0, -1.0], # exemple 1\n",
" [3.0, -1.0, 0.5], # exemple 2\n",
" [0.5, 1.0, 1.0], # exemple 3\n",
" [1.0, 1.0, -1.0], # exemple 4\n",
"]\n",
"\n",
"# Cible\n",
"ys = [1.0, -1.0, -1.0, 1.0] # desired targets"
]
},
{
"cell_type": "code",
"execution_count": 46,
"id": "6a6e52cf-7127-46f6-a3e2-af55bc8fe0c7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0.4020920928583902, -0.1894861802920144, -0.05922087190814421, 0.2550894154311267]\n",
"2.4543836642161017\n"
]
}
],
"source": [
"# forward pass\n",
"ypred = [n(x) for x in xs]\n",
"loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n",
"\n",
"# backward pass\n",
"for p in n.parameters():\n",
" p.grad = 0.0\n",
"loss.backward()\n",
"\n",
"# update\n",
"for p in n.parameters():\n",
" p.data += -0.1 * p.grad\n",
"\n",
"print(list(map(lambda x: x.data, ypred)))\n",
"print(loss.data)"
]
},
{
"cell_type": "markdown",
"id": "ceda5b12-4a89-4937-98f8-b1be79a26d1b",
"metadata": {},
"source": [
"## Automatisation de l'apprentissage"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "ae841e0b-493c-4765-a0ff-dfc7cc611081",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 loss=0.00027815740115110454 [0.9947032216496835, -0.9913560575993751, -0.9900334740555171, 0.9912792110086672]\n",
"1 loss=0.0002778910086773501 [0.9947058577753756, -0.9913601868709156, -0.9900382219334123, 0.9912833679881876]\n",
"2 loss=0.0002776251162239342 [0.9947084901418856, -0.9913643103734111, -0.9900429631794665, 0.991287519178031]\n",
"3 loss=0.0002773597223959083 [0.9947111187580135, -0.9913684281201739, -0.9900476978089977, 0.9912916645915285]\n",
"4 loss=0.00027709482580351365 [0.9947137436325303, -0.9913725401244734, -0.990052425837274, 0.9912958042419678]\n",
"5 loss=0.0002768304250621039 [0.9947163647741786, -0.9913766463995367, -0.9900571472795149, 0.9912999381425945]\n",
"6 loss=0.0002765665187921489 [0.9947189821916727, -0.9913807469585485, -0.9900618621508906, 0.9913040663066116]\n",
"7 loss=0.00027630310561922066 [0.9947215958936985, -0.9913848418146511, -0.9900665704665226, 0.9913081887471796]\n",
"8 loss=0.00027604018417393545 [0.9947242058889137, -0.9913889309809447, -0.9900712722414844, 0.9913123054774174]\n",
"9 loss=0.00027577775309196634 [0.9947268121859485, -0.9913930144704878, -0.9900759674908006, 0.9913164165104016]\n",
"10 loss=0.00027551581101400513 [0.9947294147934046, -0.9913970922962964, -0.9900806562294483, 0.9913205218591671]\n",
"11 loss=0.0002752543565857338 [0.9947320137198563, -0.991401164471346, -0.9900853384723558, 0.9913246215367074]\n",
"12 loss=0.0002749933884578092 [0.9947346089738499, -0.99140523100857, -0.9900900142344047, 0.9913287155559745]\n",
"13 loss=0.00027473290528583504 [0.9947372005639045, -0.991409291920861, -0.9900946835304283, 0.9913328039298793]\n",
"14 loss=0.0002744729057303457 [0.9947397884985115, -0.99141334722107, -0.9900993463752135, 0.9913368866712914]\n",
"15 loss=0.0002742133884567916 [0.9947423727861351, -0.9914173969220075, -0.9901040027834986, 0.9913409637930397]\n",
"16 loss=0.00027395435213548193 [0.9947449534352122, -0.9914214410364433, -0.9901086527699767, 0.9913450353079123]\n",
"17 loss=0.000273695795441608 [0.9947475304541527, -0.9914254795771064, -0.9901132963492928, 0.9913491012286568]\n",
"18 loss=0.00027343771705518716 [0.9947501038513393, -0.9914295125566853, -0.9901179335360464, 0.9913531615679803]\n",
"19 loss=0.00027318011566106204 [0.9947526736351281, -0.9914335399878288, -0.9901225643447896, 0.9913572163385497]\n",
"20 loss=0.0002729229899488548 [0.9947552398138483, -0.9914375618831455, -0.9901271887900291, 0.991361265552992]\n",
"21 loss=0.00027266633861298164 [0.9947578023958024, -0.9914415782552033, -0.9901318068862255, 0.991365309223894]\n",
"22 loss=0.00027241016035259026 [0.9947603613892664, -0.9914455891165311, -0.9901364186477934, 0.9913693473638031]\n",
"23 loss=0.0002721544538715773 [0.9947629168024897, -0.9914495944796184, -0.9901410240891017, 0.9913733799852267]\n",
"24 loss=0.00027189921787852537 [0.9947654686436956, -0.9914535943569147, -0.9901456232244743, 0.9913774071006332]\n",
"25 loss=0.000271644451086726 [0.9947680169210811, -0.9914575887608305, -0.9901502160681893, 0.9913814287224514]\n",
"26 loss=0.0002713901522141172 [0.9947705616428171, -0.9914615777037371, -0.9901548026344803, 0.9913854448630712]\n",
"27 loss=0.0002711363199832938 [0.9947731028170481, -0.9914655611979671, -0.9901593829375357, 0.9913894555348436]\n",
"28 loss=0.00027088295312146973 [0.9947756404518935, -0.9914695392558139, -0.9901639569914994, 0.9913934607500805]\n",
"29 loss=0.0002706300503604721 [0.9947781745554459, -0.9914735118895325, -0.9901685248104705, 0.9913974605210554]\n",
"30 loss=0.00027037761043668615 [0.9947807051357732, -0.9914774791113397, -0.9901730864085039, 0.9914014548600034]\n",
"31 loss=0.00027012563209108295 [0.9947832322009169, -0.9914814409334132, -0.9901776417996107, 0.9914054437791211]\n",
"32 loss=0.0002698741140691602 [0.9947857557588934, -0.9914853973678934, -0.9901821909977574, 0.991409427290567]\n",
"33 loss=0.0002696230551209477 [0.9947882758176937, -0.9914893484268822, -0.990186734016867, 0.9914134054064616]\n",
"34 loss=0.0002693724540009601 [0.9947907923852835, -0.9914932941224436, -0.9901912708708193, 0.9914173781388874]\n",
"35 loss=0.00026912230946821435 [0.994793305469603, -0.991497234466604, -0.9901958015734496, 0.9914213454998895]\n",
"36 loss=0.00026887262028616176 [0.994795815078568, -0.9915011694713524, -0.990200326138551, 0.991425307501475]\n",
"37 loss=0.0002686233852227112 [0.9947983212200685, -0.9915050991486403, -0.9902048445798729, 0.9914292641556139]\n",
"38 loss=0.0002683746030501911 [0.9948008239019703, -0.9915090235103813, -0.9902093569111221, 0.991433215474239]\n",
"39 loss=0.0002681262725453214 [0.9948033231321142, -0.9915129425684531, -0.9902138631459623, 0.9914371614692457]\n",
"40 loss=0.0002678783924892086 [0.9948058189183161, -0.9915168563346958, -0.9902183632980147, 0.9914411021524925]\n",
"41 loss=0.0002676309616673282 [0.9948083112683676, -0.991520764820912, -0.9902228573808582, 0.9914450375358014]\n",
"42 loss=0.00026738397886947555 [0.9948108001900355, -0.9915246680388691, -0.9902273454080298, 0.9914489676309572]\n",
"43 loss=0.0002671374428897925 [0.9948132856910629, -0.9915285660002966, -0.9902318273930236, 0.9914528924497088]\n",
"44 loss=0.00026689135252670835 [0.9948157677791677, -0.9915324587168886, -0.9902363033492926, 0.9914568120037681]\n",
"45 loss=0.00026664570658295 [0.9948182464620443, -0.9915363462003022, -0.9902407732902473, 0.9914607263048112]\n",
"46 loss=0.00026640050386550387 [0.9948207217473627, -0.9915402284621586, -0.9902452372292573, 0.9914646353644778]\n",
"47 loss=0.00026615574318559365 [0.9948231936427689, -0.9915441055140438, -0.9902496951796503, 0.991468539194372]\n",
"48 loss=0.0002659114233586779 [0.9948256621558853, -0.9915479773675072, -0.9902541471547133, 0.9914724378060616]\n",
"49 loss=0.0002656675432044268 [0.9948281272943102, -0.9915518440340627, -0.9902585931676916, 0.9914763312110793]\n",
"50 loss=0.0002654241015467078 [0.9948305890656182, -0.9915557055251883, -0.9902630332317898, 0.9914802194209219]\n",
"51 loss=0.0002651810972135345 [0.9948330474773605, -0.9915595618523279, -0.9902674673601719, 0.9914841024470509]\n",
"52 loss=0.0002649385290371021 [0.9948355025370644, -0.9915634130268884, -0.9902718955659612, 0.9914879803008926]\n",
"53 loss=0.00026469639585372163 [0.9948379542522342, -0.9915672590602432, -0.9902763178622406, 0.991491852993838]\n",
"54 loss=0.0002644546965038287 [0.9948404026303508, -0.9915710999637297, -0.9902807342620525, 0.9914957205372436]\n",
"55 loss=0.00026421342983196637 [0.9948428476788715, -0.9915749357486506, -0.9902851447783994, 0.9914995829424305]\n",
"56 loss=0.00026397259468673195 [0.9948452894052309, -0.9915787664262744, -0.9902895494242441, 0.9915034402206855]\n",
"57 loss=0.0002637321899208108 [0.9948477278168403, -0.9915825920078347, -0.9902939482125089, 0.9915072923832606]\n",
"58 loss=0.00026349221439092536 [0.994850162921088, -0.9915864125045307, -0.9902983411560771, 0.9915111394413736]\n",
"59 loss=0.0002632526669578241 [0.9948525947253397, -0.9915902279275273, -0.9903027282677919, 0.9915149814062079]\n",
"60 loss=0.0002630135464862588 [0.994855023236938, -0.9915940382879552, -0.990307109560458, 0.9915188182889126]\n",
"61 loss=0.0002627748518449845 [0.9948574484632029, -0.9915978435969114, -0.9903114850468401, 0.9915226501006031]\n",
"62 loss=0.0002625365819067164 [0.994859870411432, -0.9916016438654587, -0.9903158547396645, 0.9915264768523607]\n",
"63 loss=0.0002622987355481437 [0.9948622890889002, -0.9916054391046263, -0.990320218651618, 0.991530298555233]\n",
"64 loss=0.0002620613116498816 [0.9948647045028599, -0.9916092293254094, -0.9903245767953495, 0.9915341152202339]\n",
"65 loss=0.00026182430909647543 [0.9948671166605414, -0.9916130145387706, -0.9903289291834682, 0.991537926858344]\n",
"66 loss=0.0002615877267763755 [0.9948695255691526, -0.991616794755638, -0.9903332758285457, 0.9915417334805103]\n",
"67 loss=0.00026135156358190526 [0.9948719312358792, -0.9916205699869077, -0.9903376167431156, 0.9915455350976468]\n",
"68 loss=0.0002611158184092889 [0.994874333667885, -0.9916243402434415, -0.9903419519396724, 0.9915493317206341]\n",
"69 loss=0.00026088049015857975 [0.9948767328723116, -0.991628105536069, -0.9903462814306735, 0.9915531233603202]\n",
"70 loss=0.0002606455777336732 [0.9948791288562789, -0.9916318658755867, -0.9903506052285381, 0.99155691002752]\n",
"71 loss=0.00026041108004229614 [0.9948815216268849, -0.9916356212727587, -0.9903549233456476, 0.9915606917330158]\n",
"72 loss=0.0002601769959959707 [0.9948839111912059, -0.9916393717383161, -0.990359235794346, 0.9915644684875572]\n",
"73 loss=0.00025994332451001576 [0.9948862975562964, -0.9916431172829575, -0.99036354258694, 0.9915682403018614]\n",
"74 loss=0.00025971006450350737 [0.9948886807291896, -0.9916468579173495, -0.9903678437356993, 0.9915720071866134]\n",
"75 loss=0.0002594772148992945 [0.9948910607168969, -0.9916505936521265, -0.9903721392528556, 0.9915757691524658]\n",
"76 loss=0.00025924477462394707 [0.9948934375264088, -0.9916543244978904, -0.990376429150605, 0.9915795262100391]\n",
"77 loss=0.0002590127426077744 [0.994895811164694, -0.9916580504652116, -0.9903807134411053, 0.9915832783699219]\n",
"78 loss=0.0002587811177847791 [0.9948981816387003, -0.9916617715646288, -0.9903849921364788, 0.991587025642671]\n",
"79 loss=0.0002585498990926639 [0.9949005489553541, -0.991665487806648, -0.9903892652488109, 0.9915907680388114]\n",
"80 loss=0.0002583190854728064 [0.994902913121561, -0.9916691992017448, -0.9903935327901501, 0.9915945055688365]\n",
"81 loss=0.00025808867587022986 [0.9949052741442054, -0.9916729057603626, -0.9903977947725098, 0.9915982382432085]\n",
"82 loss=0.00025785866923361744 [0.994907632030151, -0.9916766074929136, -0.9904020512078662, 0.9916019660723578]\n",
"83 loss=0.00025762906451527224 [0.9949099867862405, -0.991680304409779, -0.99040630210816, 0.991605689066684]\n",
"84 loss=0.0002573998606711118 [0.994912338419296, -0.9916839965213087, -0.9904105474852961, 0.9916094072365551]\n",
"85 loss=0.0002571710566606487 [0.994914686936119, -0.9916876838378215, -0.9904147873511436, 0.9916131205923086]\n",
"86 loss=0.00025694265144697635 [0.9949170323434902, -0.9916913663696055, -0.9904190217175362, 0.9916168291442506]\n",
"87 loss=0.00025671464399674884 [0.99491937464817, -0.9916950441269184, -0.9904232505962718, 0.9916205329026572]\n",
"88 loss=0.00025648703328018396 [0.9949217138568983, -0.9916987171199864, -0.9904274739991137, 0.991624231877773]\n",
"89 loss=0.00025625981827102136 [0.9949240499763948, -0.9917023853590058, -0.9904316919377895, 0.9916279260798128]\n",
"90 loss=0.000256032997946528 [0.9949263830133587, -0.9917060488541427, -0.9904359044239917, 0.9916316155189605]\n",
"91 loss=0.00025580657128748024 [0.9949287129744692, -0.991709707615532, -0.9904401114693784, 0.9916353002053699]\n",
"92 loss=0.0002555805372781285 [0.9949310398663853, -0.9917133616532797, -0.9904443130855728, 0.9916389801491646]\n",
"93 loss=0.0002553548949062144 [0.9949333636957461, -0.9917170109774607, -0.9904485092841633, 0.9916426553604384]\n",
"94 loss=0.000255129643162935 [0.9949356844691708, -0.9917206555981201, -0.9904527000767042, 0.9916463258492546]\n",
"95 loss=0.0002549047810429273 [0.9949380021932585, -0.9917242955252741, -0.9904568854747148, 0.9916499916256472]\n",
"96 loss=0.0002546803075442763 [0.9949403168745884, -0.9917279307689078, -0.9904610654896808, 0.99165365269962]\n",
"97 loss=0.00025445622166846534 [0.9949426285197206, -0.9917315613389777, -0.9904652401330535, 0.9916573090811478]\n",
"98 loss=0.0002542325224203795 [0.9949449371351949, -0.9917351872454109, -0.9904694094162505, 0.9916609607801753]\n",
"99 loss=0.0002540092088083135 [0.9949472427275319, -0.991738808498104, -0.9904735733506552, 0.9916646078066182]\n"
]
}
],
"source": [
"for k in range(100):\n",
" # forward pass\n",
" ypred = [n(x) for x in xs]\n",
" loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n",
" \n",
" # backward pass\n",
" for p in n.parameters():\n",
" p.grad = 0.0\n",
" loss.backward()\n",
" \n",
" # update\n",
" for p in n.parameters():\n",
" p.data += -0.1 * p.grad\n",
" \n",
" print(f\"{k} loss={loss.data} {list(map(lambda x: x.data, ypred))}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d6ed699c-9cf8-4e54-b97f-ed2ddec25d09",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.14.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}