{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NETS implementation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 0. Set Up\n",
    "\n",
    "For this tutorial, we will be using ``numpy`` to generate data and all operations, ``matplotlib`` to render and plot images and of course ``nets`` for ``Tensor`` operations and computational graph creation. We will set the ``SEED`` for random numbers from ``numpy``, so each time we reload the notebook the results will be the same as numbers will be generated in the same way. Finally, we set an autoreload with ``%load_ext autoreload`` and ``%autoreload 2`` for debug mode purposes. This enable the notebook to reload the modules each time we run a cell, so it keeps everything up-to-date. More info [on StackOverflow](http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython).\n",
    "\n",
    "Let's load the packages we need to run our tests:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random as rd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# This is a bit of magic to make matplotlib figures appear inline in the notebook\n",
    "# rather than in a new window.\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.style.use('seaborn-darkgrid')\n",
    "# Change the font ?\n",
    "font = {'size'   : 15}\n",
    "plt.rc('font', **font)\n",
    "\n",
    "# Some more magic so that the notebook will reload external python modules;\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's fix the seed\n",
    "\n",
    "SEED = 42\n",
    "rd.seed(SEED)\n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Import the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset MNIST(\n",
       "   (data): Tensor(shape=(60000, 784), dtype=float64)\n",
       "   (labels): Tensor(shape=(60000,), dtype=uint8)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import random as rd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import nets\n",
    "import nets.datasets as datasets\n",
    "\n",
    "# IMPORTANT: normalize the data !\n",
    "reshape = lambda x: x.reshape(-1, 784)\n",
    "normalize = lambda x: (x - np.mean(x, axis=1, keepdims=True)) / np.std(x, axis=1, keepdims=True)\n",
    "transform = lambda x: normalize(reshape(x))\n",
    "\n",
    "# Download the training and testing data\n",
    "train_data, test_data = datasets.MNIST.splits(transform = transform)\n",
    "\n",
    "\n",
    "# Let's check the CIFAR10 dataset structure\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 34024 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 32 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# CIFAR10 classes\n",
    "# classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "# Configure the mosaic to display images\n",
    "fig = plt.figure(figsize = (20, 10))\n",
    "columns = 8\n",
    "rows = 4\n",
    "\n",
    "# Create the mosaic\n",
    "for i in range(1, columns * rows + 1):\n",
    "    # Choose randomly an image\n",
    "    image_index = rd.randint(0, len(train_data))\n",
    "    image, label = train_data[image_index]\n",
    "    # Plot it\n",
    "    fig.add_subplot(rows, columns, i)\n",
    "    plt.imshow(image.data.reshape(28, 28))\n",
    "    # Remove axis and display image's label\n",
    "    # plt.title(classes[label.data])\n",
    "    plt.axis('off')\n",
    "    \n",
    "# Render\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nets.data import Iterator\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "iterator = Iterator(train_data, \n",
    "                    batch_size = BATCH_SIZE, \n",
    "                    shuffle = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Build a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\home\\Arthur\\Documents\\Informatique\\GitHub\\nets\\nets\\nn\\module\\linear.py:33: DeprecationWarning: Call to deprecated function init_params (weights and biases will be initialized in another function soon. This method will be then deprecated in the next update).\n",
      "  self.init_params()\n",
      "D:\\home\\Arthur\\Documents\\Informatique\\GitHub\\nets\\nets\\nn\\module\\linear.py:33: DeprecationWarning: Call to deprecated function init_params (weights and biases will be initialized in another function soon. This method will be then deprecated in the next update).\n",
      "  self.init_params()\n",
      "D:\\home\\Arthur\\Documents\\Informatique\\GitHub\\nets\\nets\\nn\\module\\linear.py:33: DeprecationWarning: Call to deprecated function init_params (weights and biases will be initialized in another function soon. This method will be then deprecated in the next update).\n",
      "  self.init_params()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Classifier(\n",
       "   (layer1): Linear(input_dim=784, output_dim=100, bias=True)\n",
       "   (layer2): Linear(input_dim=100, output_dim=100, bias=True)\n",
       "   (layer3): Linear(input_dim=100, output_dim=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nets.nn as nn\n",
    "\n",
    "# Just a simple model\n",
    "class Classifier(nn.Module):\n",
    "\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        super().__init__()\n",
    "        self.layer1 = nn.Linear(input_dim, hidden_dim)\n",
    "        self.layer2 = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.layer3 = nn.Linear(hidden_dim, output_dim)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        out1 = nets.relu(self.layer1(inputs))\n",
    "        out2 = nets.relu(self.layer2(out1))\n",
    "        return self.layer3(out2)\n",
    "    \n",
    "model = Classifier(784, 100, 10)\n",
    "\n",
    "# Let's check the architecture\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Criterion & Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nets.optim import SGD\n",
    "from nets.nn import CrossEntropyLoss, MSE\n",
    "from nets.nn.utils import one_hot\n",
    "\n",
    "# How much do we update the parameters\n",
    "LEARNING_RATE = 0.01\n",
    "\n",
    "optimizer = SGD(model.parameters(), lr=LEARNING_RATE)\n",
    "criterion = CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 0/50 | loss: 4.349E-01\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 1/50 | loss: 2.194E-01\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 2/50 | loss: 1.765E-01\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 3/50 | loss: 1.501E-01\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 4/50 | loss: 1.313E-01\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 5/50 | loss: 1.169E-01\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 6/50 | loss: 1.054E-01\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 7/50 | loss: 9.594E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 8/50 | loss: 8.794E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 9/50 | loss: 8.107E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 10/50 | loss: 7.507E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 11/50 | loss: 6.980E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 12/50 | loss: 6.515E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 13/50 | loss: 6.094E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 14/50 | loss: 5.715E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 15/50 | loss: 5.368E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 16/50 | loss: 5.050E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 17/50 | loss: 4.757E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 18/50 | loss: 4.483E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 19/50 | loss: 4.230E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 20/50 | loss: 3.996E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 21/50 | loss: 3.779E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 22/50 | loss: 3.574E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 23/50 | loss: 3.385E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 24/50 | loss: 3.205E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 25/50 | loss: 3.036E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 26/50 | loss: 2.879E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 27/50 | loss: 2.729E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 28/50 | loss: 2.589E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 29/50 | loss: 2.456E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 30/50 | loss: 2.330E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 31/50 | loss: 2.211E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 32/50 | loss: 2.103E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 33/50 | loss: 1.999E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 34/50 | loss: 1.900E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 35/50 | loss: 1.809E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 36/50 | loss: 1.720E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 37/50 | loss: 1.638E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 38/50 | loss: 1.560E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 39/50 | loss: 1.485E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 40/50 | loss: 1.416E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 41/50 | loss: 1.350E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 42/50 | loss: 1.287E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 43/50 | loss: 1.229E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 44/50 | loss: 1.174E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 45/50 | loss: 1.122E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 46/50 | loss: 1.073E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 47/50 | loss: 1.027E-02\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 48/50 | loss: 9.832E-03\n",
      "Training: 100% | [==================================================] | Done !\n",
      "epoch: 49/50 | loss: 9.419E-03\n",
      "\n",
      "time: 612.8995003700256\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from nets.utils import progress_bar\n",
    "\n",
    "# Training iterations\n",
    "EPOCHS = 50\n",
    "\n",
    "#Display the time during training\n",
    "start = time.time()\n",
    "# Record the loss\n",
    "history = []\n",
    "\n",
    "# Run the simulation EPOCHS times\n",
    "for epoch in range(EPOCHS):\n",
    "    \n",
    "    # Keep track of the loss at each epoch\n",
    "    epoch_loss = 0.0\n",
    "    epoch_start = time.time()\n",
    "    \n",
    "    # Number of batches\n",
    "    N  = len(iterator)\n",
    "\n",
    "    # Predict for all batches\n",
    "    for idx, batch in enumerate(iterator):\n",
    "        \n",
    "        # Gradients accumulates, therefore we need to set them to zero at each iteration\n",
    "        model.zero_grad()\n",
    "        \n",
    "        # Get the data from the batch\n",
    "        image, label = batch        \n",
    "        # Run the forward pass on the model\n",
    "        predictions = model(image)\n",
    "        # Get how far are the predictions from the truth (aka gold labels)\n",
    "        label = one_hot(label, 10).astype(int)\n",
    "        loss = criterion(predictions, label)\n",
    "        \n",
    "        # Compute the gradient\n",
    "        loss.backward()\n",
    "        # Update the parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        # Record the loss for plotting\n",
    "        epoch_loss += loss.item() \n",
    "        # .item() is really important, it will free the memory from the computational graph\n",
    "        \n",
    "        # Let's plot a progress bar in the console\n",
    "        progress_bar(idx, N, \"Training:\")\n",
    "    \n",
    "    # Update the history of all previous loss\n",
    "    history.append(epoch_loss / N)\n",
    "    \n",
    "    # Check the mean loss for this loop\n",
    "    print(f\"epoch: {epoch}/{EPOCHS} | loss: {epoch_loss / N:1.3E}\")\n",
    "\n",
    "# Duration of the total training loop\n",
    "print('\\ntime:', time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'loss')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 29056 characters]\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history)\n",
    "plt.title(\"Loss on MNIST dataset\")\n",
    "plt.xlabel(\"epochs\")\n",
    "plt.ylabel(\"loss\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
