DIGITS-CNN/cars/lr-investigations/lr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3c568ab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ecc547f",
   "metadata": {},
   "source": [
    "# Fixed Learning Rate\n",
    "80/10/10 Split, 100/200 epochs\n",
    "\n",
    "## Index\n",
    "0. learning rate\n",
    "1. top-1 accuracy\n",
    "2. top-5 accuracy\n",
    "3. last val loss\n",
    "4. last val accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1b2471d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "fixed_results_200e = np.array([\n",
    "    [1e-6, 0.31,  2.84,  5.28, 0.67],\n",
    "    [1e-5, 0.8,   2.59,  5.28, 0.55],\n",
    "    [1e-4, 6.98,  17.23, 4.6,  7.41],\n",
    "    [1e-3, 21.56, 44.72, 4.97, 26.9],\n",
    "    [5e-3, 39.35, 66.83, 3.34, 43.5],\n",
    "    [1e-2, 13.65, 30.02, 4.15, 17.46],\n",
    "    [5e-2, 1.79,  6.73,  5.13, 1.78],\n",
    "    [1e-1, 0.8,   2.78,  5.29, 0.55]\n",
    "])\n",
    "\n",
    "fixed_results_100e = np.array([\n",
    "    [1e-6, 0.31, 2.9, 5.28, 0.67],\n",
    "    [1e-5, 0.8, 2.1, 5.28, 0.55],\n",
    "    [1e-4, 2.35, 8.28, 5.00, 2.63],\n",
    "    [1e-3, 18.47, 40.09, 4.55, 23.41],\n",
    "    [5e-3, 35.52, 63.19, 3.33, 40.93],\n",
    "    [1e-2, 22.42, 47.19, 3.59, 27.02],\n",
    "    [5e-2, 2.47, 9.02, 5.07, 2.14],\n",
    "    [1e-1, 0.8, 2.53, 5.28, 0.55]\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca34155e",
   "metadata": {},
   "source": [
    "## 100 Epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c664a31c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEaCAYAAAAWvzywAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABnjklEQVR4nO2dd1yV1f/A3x+WTEFAcKCCeyBuVDS37cxZqZW2zaz8lpXfsrLht2XTUn8NS8s0zSyz6Z4lbsW9Jw4UBETWvef3x3NBQIQL3MtlnPfr9XDvc+bnPM/l85znc875HFFKodFoNJrKg5OjBdBoNBpN6aIVv0aj0VQytOLXaDSaSoZW/BqNRlPJ0Ipfo9FoKhla8Ws0Gk0lQyv+coaIhIqIEhEXK9KOFJG1pSFXWUYMvhaReBGJtmM9N4jIPjuUO1FEvrN1ubZERKaLyMuOlkNjHVrx2xEROSoi6SISmCd8q0V5hzpItJyyeItIsoj84WhZ7EhXoC8QopSKLGlhOR6+yTmO7UqpNUqpJiUXt0iy9BCRk6VZZ34opUYppd6wdbn5XOujIjK+CPm/EZE3bS1XeUcrfvtzBBiadSIiLQFPx4lzDYOANKCviNQozYqteWuxEfWAo0qpy0XNWIiMfkopb8vRqvjilW1K8T4VhJ9SyhsYDLwsIn0dLVB5Rit++/MtcH+O8xHArJwJRMRXRGaJyHkROSYiE0TEyRLnLCKTRSRORA4Dt+WT9ysRiRWRUyLypog4F0G+EcB0YAdwb56yu4rIehFJEJETIjLSEu4hIu9bZL0kImstYdf0Pi09tD6W7xNF5EcR+U5EEoGRIhIpIv9Y6ogVkU9FxC1H/hYiskRELorIWRF5UURqiEiKiATkSNfWcv1c89T/EPAl0NnSY3zNEv6IiBy0lLtIRGrlyKNE5AkROQAcsPZC5my/iDSwlN3Wcl7LIl8Py3mnHNd2e1a4JS5MRFaJSJKILAECr6nMOnlqicgCS71HROSpHHGFXfdc1yCrbSLyrIics+R5IEf67J61FWkDRORXEUkUkY2W36xVJkml1CZgF9A6R3nzReSM5be4WkRaWMIfBYYDz1vu/a9WXpdNFtnOisgHRb3u5QKllD7sdABHgT7APqAZ4AycxOiBKiDUkm4W8AvgA4QC+4GHLHGjgL1AHcAfWGHJ62KJXwj8H+AFBAHRwGOWuJHA2gLkqweYgebAs8COPHFJGG8rrkAA0NoS9xmwEqhtaVMUUAXoAZzM7xpYvk8EMoD+GJ0OD6Ad0AlwsbR9DzDWkt4HiLXI5m4572iJ+x14PEc9HwJTrtPOXNcB6AXEAW0tck8BVueIV8ASy/X2yKe80Jz3IEd4rvYDjwC7Md7w/gImW8JrAxeAWy3Xoa/lvLol/h/gA4ts3Sz34bvrtO2aa24JdwI2A68AbkB94DBwkyX+utc9v2tgqScTeN3ye7gVSAGqWdJ/A7yZQ6aC0s61HJ4Yv70TXOd3mvdaW2ROAQbkSPOg5bdRBfgI2JYjLlsuK6/LP8B9lu/eQCdH6xG76CZHC1CRD64q/gnAW8DNln8mF8uPORRDcaYDzXPkewxYafm+HBiVI+7GrH8EIBjDTOORI34osMLyfeT1/qEs8ROy/kkwlJEJaGM5/y+wMJ88TsAVoFU+cT0oXPGvvp48ljRjs+q1tGXrddLdDayzfHcGzgCR10mb6zoAXwHv5jj3xngghVrOFdCrABmzlFFCjmPcddq/CNiJ8UZVxRL2AvBtnnR/Ybx91cVQml454r6n6Iq/I3A8T9h/ga8Lu+75XQNLPVfI8bADzmFRjFyr+PNNa7lXGUCTHHFvXu93mudaX7F8nwzIddL7WdL45pXLmusCrAZeAwIL+p2W96Ms2O4qA99i/KDCyGPmwXiNdwWO5Qg7hqGIAWph9IhyxmVRz5I3VkSywpzypC+I+4EvAJRSp0RkFYby2YrxhnEonzyBGL3v/OKsIZdsItIYo3fbHqMH6ILRI6MAGcB4Q5ouImFAE+CSUsraGTu1gC1ZJ0qpZBG5gHHNj+Yn53UIVEpl5mhLj3zSfIGh/B9VSqVZwuoBQ0TkjhzpXDHe5moB8Sr3eMQxjGtRFOoBtUQkIUeYM7DGImtB1z2LvNfgQs72YvS8va9T//XSVrfUlbNsq641hkJ/GhiGcb3SxTBrTgKGWMo250h/KZ9yCrwuwEMYbyp7ReQI8JpSarEV8pUrtI2/FFBKHcMY5L0V+ClPdBxGD6hejrC6wCnL91hy/9PXzfH9BEaPP1Ap5Wc5qiqlWhQmk4hEAY2A/1rso2cwekPDxBjMOwE0yCdrHJB6nbjL5Bi4tvxTVs+TJq872GkYpqxGSqmqwItA1lPsBMar+DUopVKBeRjjEvdhPFyt5TQ5rreIeGGYsk7lSFNit7Ui4o1hevgKmCgi/paoExg9fr8ch5dS6m2M+13NIlMWdSk6J4AjeerwUUrdaokv6LpnYQ/Xvecx3mhCcoRZ9VBTSpmUUh9g/P5GW4KHAXdivFn7YrwhwNW25G1DgddFKXVAKTUUw2z6DvBjnntRIdCKv/R4COPVOdfMEqWUCUOBTRIRHxGpBzwDZM3bngc8JSIhIlINGJ8jbyzwN/C+iFQVESfLoGJ3K+QZgWF2ao4xUNYaCMew594CzAb6iMhdIuJiGZBrrZQyAzOADyyDZM4i0llEqmCMTbiLyG2WQdYJGHbXgvABEoFkEWkKPJ4jbjFQU0TGikgVy/XpmCN+FoYZpx9FU/xzgAdEpLVF7v8BG5RSR4tQhjV8DGxSSj0M/IYxiA7Gvb1DRG6yXD93y4BoiKWTsAl4TUTcRKQrcEf+xV/FUkb2gTHWkyQiL4gx8O4sIuEi0sGSpaDrbjcsv/efMB6Enpa67y8kW17exhiwzRr3ScMYI/HEuJc5OUvuzkOB10VE7hWR6pbfeYIlj5kKhlb8pYRS6pAyZiTkx5MYveXDwFoMm+4MS9wXGPbf7RjmibxvDPdjDFLtBuKBH4GaBcli+Ye5C2Mw9EyO4wiGAh2hlDqO8YbyLHAR2AZkTVkch2G33miJewdwUkpdwuiJfYnRe76MMZhdEOMwem1Jlrb+kBWhlErCGPi8A8OGfwDomSN+HcY/5RaLwrQKpdRS4GVgAUYPuwFwj7X5rUFE7sQY08lSqM8AbUVkuFLqBEYv9UWMHvAJ4Dmu/j8Ow3j7ugi8yrXmwbzUxrB/5zzCgNsxHuhHMN7UvsToFUMB170UGGOR4wzG720OhvK2lt8wfuuPYFybYxi/t93Av3nSfgU0t8xe+tny4CnoutwM7BKRZIwH9z1KqStFbWBZRywDGhpNuURElgPfK6W+dLQsmuIhIu8ANZRSIxwtS2VB9/g15RbL63lbSre3qikhItJURCLEIBLDDLrQ0XJVJvSsHk25RERmYqwHeNpiEtKUH3wwzDu1MGzw72PM0tKUEtrUo9FoNJUMberRaDSaSoZW/BqNRlPJKBc2/sDAQBUaGlqsvJcvX8bLq8KtvygQ3ebKgW5zxaek7d28eXOcUirvIsryofhDQ0PZtOl6U+ALZuXKlfTo0cO2ApVxdJsrB7rNFZ+StldE8l3fok09Go1GU8nQil+j0WgqGVrxazQaTSWjXNj48yMjI4OTJ0+SmppaYDpfX1/27NlTSlKVDSpDm93d3QkJCcHV1bXwxBqNJhflVvGfPHkSHx8fQkNDyeGL/hqSkpLw8fEpRckcT0Vvs1KKCxcucPLkScLCwhwtjkZT7ii3pp7U1FQCAgIKVPqaiomIEBAQUOjbnsaBrP0IjqzOHXZktRGucTjlVvEDWulXYvS9L+PUbgvzR15V/kdWG+e12zpSKo2FcmvqcTQXLlygd+/eAJw5cwZnZ2eqVzfWSURHR+Pm5lbkMl966SVmzZpFfHw8ycnJBabt378/Z86c4d9/87of12jKAGHdYMg3MGcoeAdDaoJxHtbNwYJpoJz3+K1l+qpDrD8Ulyts/aE4pq8q7raxEBAQwLZt29i2bRujRo3iP//5T/Z5cZQ+wB133EF0dOHbxiYkJLB582YuXbrE4cOHi1WXNWR
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fixed_results_100e[:, 0], fixed_results_100e[:, 1], 'x-', label=\"Top-1 Accuracy\")\n",
    "plt.plot(fixed_results_100e[:, 0], fixed_results_100e[:, 2], 'x-', label=\"Top-5 Accuracy\")\n",
    "plt.plot(fixed_results_100e[:, 0], fixed_results_100e[:, 4], 'x-', label=\"Final Val. Accuracy\")\n",
    "\n",
    "plt.ylim(0)\n",
    "\n",
    "plt.title('Model Accuracy for Fixed Learning Rates')\n",
    "plt.ylabel('% Accuracy')\n",
    "plt.xlabel('Learning Rate')\n",
    "\n",
    "plt.legend()\n",
    "plt.xscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bf5fa35b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEaCAYAAAAL7cBuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8NklEQVR4nO3deXwU9f348dc7IQkQIIQEQsJNuAUCBhXxAKy3BP328my1VdG2Hl+t1uNrtfXbfmvbb4+vWq9aq9YiWn9qAa9SzUY0yiWESxMJoIQrCXc4c7x/f8xEl7hJNmFnN7v7fj4e88juzGdm3p+Zzb53PvOZGVFVjDHGmKYSIh2AMcaYjskShDHGmIAsQRhjjAnIEoQxxpiALEEYY4wJyBKEMcaYgCxBhImI1IjI0BAs52ci8lwoYmqy3KtE5D2/983G27RsO9b1hohc2d75OyIR+YWIVIvINg/XMdDdL4khXu4x7c9wEJG7ReTJSMcRbzpFOoBYIyIbgSyg3m/0CFXt5vF6+wGfASNVtbzJtFeAclW9LdjlhSpeEfkZMExVr/Bb9nmhWHaAdT0NVKjqPV4sv4X1DgR+DAxS1coQLVOBA0DjhUp1qtoT8PRzFCCOwcAGIElV68K5bn+q+j9eLbvJtt4DvADcrqr1Lc5I4M93LLEjCG8UqGo3v2GL1ytU1c3A28B3/MeLSC/gfOAZr2OIYwOBHe1JDiLS0o+0PL/PUM92R9fBtbINwiXP/VE0FbgY+H6E4+kQLEGEiYioiAxzXz8tIn8SkddEZJ+ILBKRXL+y/ycim0Rkr4gsE5HTglzNMzRJEMAlwFpVXSUid4pIubvOtSLyH0HGmyEic914FgO5TcoGjFdEzgXuBi52m0ZK3PE+EbnGfZ0gIveIyGciUikiz4pImjttsBvHlSLyuduE819Bboum9blWRNaJyE63LjnueBGRP7jr3isiq0RkrDvtfHc77RORzSLylSMwETkTWADkuHV82h0/U0TWiMhut76j/ebZKCJ3iMhKYH+wX5B+26OTiPQSkQoRKXCndXPr9133/SgRWeDWt1REvu23nBb3Zxu2aZqI/EVEtrrb5xeNzV8ikisi74jIDne//V1EerawDYa1tK/Fr2m1tc+FiHQRkWdEZJeIfCwiPxGRimDqpKrrgPeBCX7La+vnu6XtMkxEikRkjxv3C+3Z9mGjqjaEcAA2AmcGGK84h6IATwM7gBNxmvn+DszxK3sFkOFO+zGwDejsTvsZ8Fwz6+6Cc4h8qt+4D4D/dF9/C8jB+WFwMbAfyHanXQW810y8c4AXgVRgLLC5Sdk2xQv4gGvc198H1gFDcZpPXgb+5k4b7MbxZ7duecBhYHQz9X8a+EWA8WcA1cDxQArwEPCuO+0cYBnQExBgtN822Qqc5r5OB45vZr3TcJq2Gt+PcLftWUAS8BO3jsl+n5EVwACgSzPL/GL7+41r3B6d3Pdnu9u6j7uNXnLHpwKbgO+5+2SiW/8xwezPltbZZNorwOPucvoAi4Hr3GnD3PqnAL2Bd4E/Nvk/+WIbtLav8fscBVH2AaDI3Wf9gZX++6eV/81R7n6/5Rg+3y1tl+eB/8L5H+yM3/9qRxwiHkCsDe4HvwbY7Q6vuuObJogn/eY5H/ikhWXuwjkEDviBbFL2SeAJ9/Vw4AjQp5myK4AL3ddXESBBAIlALTDKb9r/0MwXSjDxcnSCeBv4od+0ke76Ovl9EfT3m74YuKSZ9T5N4ATxF+A3fu+7uesYjJM8yoDJQEKT+T4HrgN6tLLPp3F0gvgp8KLf+wScL+Fpfp+R77eyTAX2+n2OHiTAlzVOslvlLj/DHXcxsLDJ8h4H7mvr/gy0Tnd8Fs6Xche/cZcChc0s5yJgeZP/k+8HWE/AfU3gBNFc2fXAOX7TrqH1BLEXJ6krzpd4Sns+361tF+BZ4An/2DvyYE1M3rhIVXu6w0XNlPHv7XIAv5OPInKbe2i8R0R2A2lAZpDrfgb4loh0xmluekvdtnER+a6IrHCbPXbj/Hpsbbm9cb6sN/mN+8y/wDHGm9NkeZ+568vyG9fstmrPOlS1BucIrp+qvgM8DPwJqBSRJ0Skh1v0GzjJ+zO3WeDkdq6vAWf79fMrs6npTAEc7/c5uqmZMk/g7MenVXWHO24QcFLjfnb3yeVAX4LYn0EahHN0tNVvHY/j/GJGRLJEZI7bxLIXeI6vfiYCbYO27OvmyuY0WXZQ29qd/2LgJJxf/0CbP98tbheco0kBFrtNkB36XIcliA7Gbd/8CfBtIF2dk5N7cD5UwXgP2AlciHNo/Iy73EE4h+Q34PzS7AmsDmK5VUAdTlNAo4FtiFdbWf4WnH8q/2XXAdtbma8tjlqHiKTiNBlsBlDVB1U1HxiD0zx0uzt+iapeiPPP/SpOs0x71ic422+zX5nWtkur3HbtJ3B+lf5Q3HNGOF+IRX7Jpac6J7p/QCv7sw024fxSzvRbRw9VPc6d/j84dRynqj1wPotNP2vHvA2asRWnaanRgOYK+lPHizjNsvdCuz7fLW4XVd2mqteqag7O0ekjfvutw7EE0fF0x/kHrgI6ici9QI+WZ/mSOsexzwK/xmlXn+dOSsX5MFcBiMj3cH55tra8epzzAj8Tka4iMga4sg3xbgcGi0hzn7XngVtEZIiIdMP5YnlB29+lMlFEOvsNye46viciE0QkxV3HIlXdKCIniMhJIpKE08RwCGgQkWQRuVxE0lS1FqcJoiHIGF4ELhCRr7nL/THOl0ZxO+vUnLtx9un3gd8Cz7pJYz4wQkS+IyJJ7nCCiIwOYn82J8V/u+Ls138BvxORHuJ0NsgVkalu+e44Ta17xOmCfXtIa96yF4G7RCTdXfcNbZz/AeBaEelLGz/fqrqVFraLiHxLRBqT1y6c/Rfs5yrsLEF0PG8Bb+K0i3+G84UVzCGyv2dxfhW+oKqHAVR1LfA7nF9H24FxOL01gnEDzuH3Npx2/r+2Id5/uH93iMhHAZb9FPA3nJOYG9z5bwwyrkDuBA76De+o6r9xzgv8P5xfl7k4vbvA+Wf/M84/62c4TU+/dad9B9joNpFcj9NM0ypVLcX5xfwQzsnhApyuz0eOoV5HEZF84Fbgu+6X/q9xvmzuVNV9OCewL8E5mtnmTk9xZ29pfzanhqO36xnAd4FkYC3O9nsJyHbL/xyn2WYP8BpOUgqX+4EKnM/Tv924Dgc7s6quwvk83k77Pt8tbZcTgEUiUgPMBW5W1fVtrF/YiHvixBhjYpKI/ADnBPbUVgubo9gRhDEmpohItoic4jbvjMRp4nsl0nFFo45wBaMxxoRSMk7PoSE4XYTnAI9EMqBoZU1MxhhjArImJmOMMQFZgjDGGBNQTJ2DyMzM1MGDB7dr3v3795Oamtp6wRhidY598VZfsDq31bJly6pVtXegaTGVIAYPHszSpUvbNa/P52PatGmhDaiDszrHvnirL1id20pEmr3VijUxGWOMCcgShDHGmIAsQRhjjAnIEoQxxpiA4jpBPFZUTnF59VHjisureayoPEIReS8e62xMLArH/3JcJ4jx/dO4YfbyLzZycXk1N8xezvj+aRGOzDvxWGdjYlE4/pdjqptrW03JzeThyyYy69lljElX1hYu47qpQ9l/uJ4Fa0P5vJqO5XunDGbWs8s4LRsWLVzOw5dNZEpusA+AM8Z0BI3fXz987iNG92yg1IP/ZU8ThIhsBPYB9UCdqk5qMv1y4A6cpzPtA36gqiXBzBsqU3IzOVRbz+Jtzj2pfvevMi9W0yG9sRHyB6YzrE9bn+BpjOkIxvZLIyEBPthaz/VTB4X8h144jiCmq2p1M9M2AFNVdZeInIfz+MSTgpw3JIrLq+manMiEDGXFDuHO80Yxvn9PL1cZcSsrdvOrNz6he2I9yz7fxZRfvcO3Jg3gutOHMjgzvq5ANSZa1dY3cPmfP2Tn/lpOzknkxaUVnD6id/QcQbRGVf0fwfghRz9H1nONbXaPfSefI5tWkzxgLDfMju0ml+Lyav73X2U87tZ5V49h3PnyKv6xdBMvLPmc88Zmc/3UXMbZOQljOixVZdazS1m
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fixed_results_100e[:, 0], fixed_results_100e[:, 3], 'x-', label=\"Final Validation Loss\")\n",
    "\n",
    "# plt.ylim(0)\n",
    "\n",
    "plt.title('Final Validation Loss for Fixed Learning Rates')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Learning Rate')\n",
    "\n",
    "# plt.legend()\n",
    "plt.xscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff7746a4",
   "metadata": {},
   "source": [
    "## 200 Epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9f4a799f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEaCAYAAAAWvzywAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABmU0lEQVR4nO2dZ3hURReA35NCEkhCgIQaSAJShEACofcuRbEg0lRAPlFUQBErFuyogAoqVqoI2BUElRa6lNB7TeglIZAESNud78fdxAApm7K7KfM+z31279wpZ+7unjt75swZUUqh0Wg0mpKDk6MF0Gg0Go190Ypfo9FoShha8Ws0Gk0JQyt+jUajKWFoxa/RaDQlDK34NRqNpoShFX8RQ0QCRUSJiIsVeYeKyDp7yFWYEYOZIhIrIptt2E47ETlog3oniMh3BV1vQSIiX4jIq46WQ2MdWvHbEBGJFJFkEfG9KX27RXkHOki0jLJ4ikiCiCx1tCw2pC3QDfBXSjXPb2UZHr4JGY6dSqm1Sqm6+Rc3V7J0FJFT9mwzM5RSjyul3iroejO515Ei8mIuys8SkbcLWq6ijlb8tuc4MDDtREQaAqUdJ84t9AWSgG4iUtmeDVvzr6WACAAilVJXc1swBxl9lFKeliMk7+IVbuz4OWWHj1LKE7gfeFVEujlaoKKMVvy2Zy7wcIbzIcCcjBlEpKyIzBGRiyISJSKviIiT5ZqziEwSkWgROQb0zqTstyJyVkROi8jbIuKcC/mGAF8Au4AHb6q7rYhsEJHLInJSRIZa0j1EZLJF1isiss6Sdsvo0zJC62p5P0FEfhKR70QkDhgqIs1FZKOljbMi8qmIlMpQvoGILBORSyJyXkReFpHKInJNRCpkyNfEcv9cb2p/OPAN0MoyYnzDkv6oiByx1PuHiFTNUEaJyJMichg4bO2NzNh/EallqbuJ5byqRb6OlvOWGe7tzrR0y7UgEVktIvEisgzwvaUx6+SpKiI/W9o9LiKjM1zL6b7fcA/S+iYiz4rIBUuZYRnyp4+srchbQUQWiUiciGyxfGetMkkqpbYCe4HQDPX9KCLnLN/FNSLSwJI+AhgMPG/57BdZeV+2WmQ7LyJTcnvfiwRKKX3Y6AAiga7AQeB2wBk4hTECVUCgJd8c4HfACwgEDgHDLdceBw4A1YHywCpLWRfL9V+BL4EyQEVgM/CY5dpQYF028gUAZqA+8Cyw66Zr8Rj/VlyBCkCo5dpnQDhQzdKn1oAb0BE4ldk9sLyfAKQA92AMOjyAMKAl4GLp+37gaUt+L+CsRTZ3y3kLy7UlwMgM7XwETMuinzfcB6AzEA00scg9DViT4boCllnut0cm9QVm/AwypN/Qf+BRYB/GP7y/gUmW9GpADNDLch+6Wc79LNc3AlMssrW3fA7fZdG3W+65Jd0JiABeA0oBNYFjwB2W61ne98zugaWdVOBNy/ehF3ANKGfJPwt4O4NM2eVdYDlKY3z3TpLF9/Tme22R+Rpwb4Y8j1i+G27Ax8CODNfS5bLyvmwEHrK89wRaOlqP2EQ3OVqA4nzwn+J/BXgP6GH5MblYvsyBGIozGaifodxjQLjl/Urg8QzXuqf9EIBKGGYajwzXBwKrLO+HZvWDslx/Je1HgqGMTEBjy/lLwK+ZlHECrgMhmVzrSM6Kf01W8ljyPJ3WrqUv27PI1x9Yb3nvDJwDmmeR94b7AHwLfJDh3BPjgRRoOVdA52xkTFNGlzMc47Lo/x/Abox/VG6WtBeAuTfl+xvj31cNDKVZJsO178m94m8BnLgp7SVgZk73PbN7YGnnOhkedsAFLIqRWxV/pnktn1UKUDfDtbez+p7edK+vW95PAiSL/D6WPGVvlsua+wKsAd4AfLP7nhb1ozDY7koCczG+UEHcZObB+BvvCkRlSIvCUMQAVTFGRBmvpRFgKXtWRNLSnG7Knx0PA18DKKVOi8hqDOWzHeMfxtFMyvhijL4zu2YNN8gmInUwRrdNMUaALhgjMrKRAYx/SF+ISBBQF7iilLLWY6cqsC3tRCmVICIxGPc8MjM5s8BXKZWaoS8dM8nzNYbyH6GUSrKkBQD9ROSuDPlcMf7NVQVi1Y3zEVEY9yI3BABVReRyhjRnYK1F1uzuexo334OYjP3FGHl7ZtF+Vnn9LG1lrNuqe42h0McAgzDuV7IYZs13gH6Wus0Z8l/JpJ5s7wswHOOfygEROQ68oZRabIV8RQpt47cDSqkojEneXsAvN12OxhgBBWRIqwGctrw/y40/+hoZ3p/EGPH7KqV8LIe3UqpBTjKJSGugNvCSxT56DmM0NEiMybyTQK1MikYDiVlcu0qGiWvLj9Lvpjw3h4OdjmHKqq2U8gZeBtKeYicx/orfglIqEfgBY17iIYyHq7WcIcP9FpEyGKas0xny5DtsrYh4YpgevgUmiEh5y6WTGCN+nwxHGaXURIzPu5xFpjRqkHtOAsdvasNLKdXLcj27+56GLUL3XsT4R+OfIc2qh5pSyqSUmoLx/XvCkjwIuBvjn3VZjH8I8F9fbu5DtvdFKXVYKTUQw2z6PvDTTZ9FsUArfvsxHOOv8w2eJUopE4YCe0dEvEQkABgLpPlt/wCMFhF/ESkHvJih7FngH2CyiHiLiJNlUrGDFfIMwTA71ceYKAsFgjHsuT2BeUBXEXlARFwsE3KhSikzMAOYYpkkcxaRViLihjE34S4ivS2TrK9g2F2zwwuIAxJEpB4wMsO1xUAVEXlaRNws96dFhutzMMw4fcid4p8PDBORUIvc7wKblFKRuajDGj4Btiql/gf8iTGJDsZne5eI3GG5f+6WCVF/yyBhK/CGiJQSkbbAXZlX/x+WOtIPjLmeeBF5QYyJd2cRCRaRZpYi2d13m2H5vv+C8SAsbWn74RyK3cxEjAnbtHmfJIw5ktIYn2VGznPj4CHb+yIiD4qIn+V7ftlSxkwxQyt+O6GUOqoMj4TMGIUxWj4GrMOw6c6wXPsaw/67E8M8cfM/hocxJqn2AbHAT0CV7GSx/GAewJgMPZfhOI6hQIcopU5g/EN5FrgE7ADSXBbHYditt1iuvQ84KaWuYIzEvsEYPV/FmMzOjnEYo7Z4S18Xpl1QSsVjTHzehWHDPwx0ynB9PcaPcptFYVqFUmo58CrwM8YIuxYwwNry1iAid2PM6aQp1LFAExEZrJQ6iTFKfRljBHwSeI7/fo+DMP59XQJe51bz4M1Uw7B/ZzyCgDsxHujHMf6pfYMxKoZs7rsdeMoixzmM79t8DOVtLX9ifNcfxbg3URjft33Avzfl/Raob/Fe+s3y4MnuvvQA9opIAsaDe4BS6npuO1jYEcuEhkZTJBGRlcD3SqlvHC2LJm+IyPtAZaXUEEfLUlLQI35NkcXy97wJ9h2tavKJiNQTkUZi0BzDDPqro+UqSWivHk2RRERmY6wHGGMxCWmKDl4Y5p2qGDb4yRheWho7YTNTj4jU5caRWE2MRRNzLOmBGK5zDyilYm0ihEaj0WhuwS42fotb32mMCasngUtKqYliBFsqp5R6weZCaDQajQawn42/C3DU4nlxNzDbkp72d12j0Wg0dsJeNv4BGDY9gEoW/3Mw3LkqZVbAEmBpBICHh0dY9eq5XbhoYDabcXIqWXPYus8lA93n4k9++3vo0KFopdTNiyhtH6sHw8c8GkPhA1y+6XpsTnWEhYWpvLJq1ao8ly2q6D6XDHSfiz/57S/GAsJbdKo9Hp09MRbYnLecnxeRKgCW1wt2kEGj0Wg0Fuyh+Afyn5kHjIBVaQs1hqDduDQajcau2FTxW4IbdePGMAMTMXZ7OowRWGmiLWXQaDQazY3YdHJXGQHJKtyUFoPh5ZMvUlJSOHXqFImJidnmK1u2LPv3789vc0WKktBnd3d3/P39cXV1zTmzRqO5gSK7cvfUqVN4eXkRGBhIhlj0txAfH4+Xl5cdJXM8xb3PSiliYmI4deoUQUFBjhZHoylyFFm/qMTERCpUqJCt0tcUT0SEChUq5PhvT+NA1n0Mx9fcmHZ8jZGucThFVvEDWumXYPRnX8ip1gR+HPqf8j++xjiv1sSRUmksFFl
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fixed_results_200e[:, 0], fixed_results_200e[:, 1], 'x-', label=\"Top-1 Accuracy\")\n",
    "plt.plot(fixed_results_200e[:, 0], fixed_results_200e[:, 2], 'x-', label=\"Top-5 Accuracy\")\n",
    "plt.plot(fixed_results_200e[:, 0], fixed_results_200e[:, 4], 'x-', label=\"Final Val. Accuracy\")\n",
    "\n",
    "plt.ylim(0)\n",
    "\n",
    "plt.title('Model Accuracy for Fixed Learning Rates')\n",
    "plt.ylabel('% Accuracy')\n",
    "plt.xlabel('Learning Rate')\n",
    "\n",
    "plt.legend()\n",
    "plt.xscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cdf18f4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEaCAYAAAAL7cBuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA+WUlEQVR4nO3deXxU5dXA8d9JCAkQSMIEwhLWwQVE9kViFbBWa2vRtta9gtZSa7GbttYuaq1vbd8uthZ9FRdAUdHaqrivxA3ZA8gmEpZAWANJIBCynvePe4NDnOxzZyaT8/185pOZu55n7mTO3Od57n1EVTHGGGNqi4t0AMYYY6KTJQhjjDFBWYIwxhgTlCUIY4wxQVmCMMYYE5QlCGOMMUFZgggTESkRkYEh2M6dIjIvFDHV2u40Efkw4HWd8dZethn7ek1EpjZ3/WgkIneLSIGI7PFwH33d4xIf4u226HiGg4j8WkQeiXQcbU27SAcQa0RkG5ABVAVMPllVkz3eb29gO3CKqubWmvc8kKuqtzR2e6GKV0TuBAap6tUB274gFNsOsq85wE5V/a0X269nv32Bm4F+qrovRNtU4ChQc6FSpaqmAp5+joLE0R/YCiSoamU49x1IVf/o1bZrvdfFwDPAL1S1qt4VCf75jiV2BuGNb6hqcsBjl9c7VNV84B3gu4HTRaQr8DVgrtcxtGF9gQPNSQ4iUt+PtOEBn6HUZkcX5Rp4D8JluPujaCJwGXBdhOOJCpYgwkREVEQGuc/niMj9IvKKiBwWkSUi4g9Y9p8iskNEDonIChE5q5G7mUutBAFcDqxX1U9E5Fcikuvuc72IfLOR8fpEZIEbz1LAX2vZoPGKyFeBXwOXuVUjq93p2SJyvfs8TkR+KyLbRWSfiDwuIinuvP5uHFNFJM+twvlNI9+L2uX5vohsFpGDbll6udNFRO51931IRD4RkaHuvK+579NhEckXkS+cgYnIucBbQC+3jHPc6VNEZJ2IFLnlHRywzjYRuVVE1gBHGvsFGfB+tBORriKyU0S+4c5Ldst3jfv6VBF5yy3vpyJyacB26j2eTXhPU0TkURHZ7b4/d9dUf4mIX0TeFZED7nF7UkRS63kPBtV3rCWgarWhz4WIdBCRuSJSKCIbROSXIrKzMWVS1c3AR8CIgO019fNd3/sySETeE5FiN+5nmvPeh42q2iOED2AbcG6Q6YpzKgowBzgAjMOp5nsSmB+w7NWAz513M7AHSHLn3QnMq2PfHXBOkb8UMO1j4Kfu8+8AvXB+GFwGHAF6uvOmAR/WEe984FmgEzAUyK+1bJPiBbKB693n1wGbgYE41Sf/BZ5w5/V343jYLdtwoAwYXEf55wB3B5l+DlAAjAISgX8B77vzzgdWAKmAAIMD3pPdwFnu8zRgVB37nYRTtVXz+mT3vf0KkAD80i1j+4DPyCqgD9Chjm0ef/8DptW8H+3c1+e573V39z16zp3eCdgBXOsek5Fu+Yc05njWt89a854HHnK30x1YCvzAnTfILX8i0A14H/hHrf+T4+9BQ8eagM9RI5b9E/Cee8wygTWBx6eB/81T3eP+sxZ8vut7X54GfoPzP5hEwP9qND4iHkCsPdwPfglQ5D5ecKfXThCPBKzzNWBjPdssxDkFDvqBrLXsI8As9/lJQDnQvY5lVwEXuc+nESRBAPFABXBqwLw/UscXSmPi5cQE8Q5wY8C8U9z9tQv4IsgMmL8UuLyO/c4heIJ4FPjfgNfJ7j764ySPTcAZQFyt9fKAHwBdGjjmkzgxQfwOeDbgdRzOl/CkgM/IdQ1sU4FDAZ+j+wjyZY2T7D5xt+9zp10GfFBrew8BdzT1eAbbpzs9A+dLuUPAtCuAhXVs52Igp9b/yXVB9hP0WBM8QdS17Bbg/IB519NwgjiEk9QV50s8sTmf74beF+BxYFZg7NH8sComb1ysqqnu4+I6lgns7XKUgMZHEbnFPTUuFpEiIAVIb+S+5wLfEZEknOqmN9StGxeRa0RklVvtUYTz67Gh7XbD+bLeETBte+ACLYy3V63tbXf3lxEwrc73qjn7UNUSnDO43qr6LjATuB/YJyKzRKSLu+i3cZL3drdaYEIz91eN8/71DlhmR+2VghgV8Dn6cR3LzMI5jnNU9YA7rR8wvuY4u8fkKqAHjTiejdQP5+xod8A+HsL5xYyIZIjIfLeK5RAwjy9+JoK9B0051nUt26vWthv1XrvrXwaMx/n1DzT5813v+4JzNinAUrcKMqrbOixBRBm3fvOXwKVAmjqNk8U4H6rG+BA4CFyEc2o8191uP5xT8hk4vzRTgbWN2O5+oBKnKqBG3ybEqw1sfxfOP1XgtiuBvQ2s1xQn7ENEOuFUGeQDqOp9qjoaGIJTPfQLd/oyVb0I55/7BZxqmebsT3Dev/yAZRp6Xxrk1mvPwvlVeqO4bUY4X4jvBSSXVHUaun9IA8ezCXbg/FJOD9hHF1U9zZ3/R5wynq6qXXA+i7U/ay1+D+qwG6dqqUafuhYMpI5ncaplb4dmfb7rfV9UdY+qfl9Ve+GcnT4QcNyijiWI6NMZ5x94P9BORG4HutS/yufUOY99HPgzTr36S+6sTjgf5v0AInItzi/PhrZXhdMucKeIdBSRIcDUJsS7F+gvInV91p4GfiYiA0QkGeeL5RltfpfKeBFJCni0d/dxrYiMEJFEdx9LVHWbiIwVkfEikoBTxXAMqBaR9iJylYikqGoFThVEdSNjeBb4uoh82d3uzThfGouaWaa6/BrnmF4H/AV43E0aLwMni8h3RSTBfYwVkcGNOJ51SQx8X3GO65vA30SkizidDfwiMtFdvjNOVWuxOF2wfxHSktfvWeA2EUlz9z2jiev/Cfi+iPSgiZ9vVd1NPe+LiHxHRGqSVyHO8Wvs5yrsLEFEnzeA13HqxbfjfGE15hQ50OM4vwqfUdUyAFVdD/wN59fRXuB0nN4ajTED5/R7D049/+wmxPtv9+8BEVkZZNuPAU/gNGJudde/qZFxBfMroDTg8a6qvo3TLvAfnF+XfpzeXeD8sz+M88+6Hafq6S/uvO8C29wqkhtwqmkapKqf4vxi/hdO4/A3cLo+l7egXCcQkdHAz4Fr3C/9P+N82fxKVQ/jNGBfjnM2s8edn+iuXt/xrEsJJ76v5wDXAO2B9Tjv33NAT3f53+NU2xQDr+AkpXC5C9iJ83l6242rrLErq+onOJ/HX9C8z3d978tYYImIlAALgJ+o6pYmli9sxG04McaYmCQiP8RpwJ7Y4MLmBHYGYYyJKSLSU0TOdKt3TsGp4ns+0nG1RtFwBaMxxoRSe5yeQwNwugjPBx6IZECtlVUxGWOMCcqqmIwxxgRlCcIYY0xQMdUGkZ6erv3792/WukeOHKFTp04NLxhDrMyxr62VF6zMTbVixYoCVe0WbF5MJYj+/fuzfPnyZq2bnZ3NpEmTQhtQlLMyx762Vl6wMjeViNR5qxWrYjLGGBOUJQhjjDFBWYIwxhgTlCUIY4wxQVmCMMaYVujB93JZlFtwwrRFuQU8+F5uyPbRphNEON7gaNMWy2xMLBqWmcKMp3KO/z8vyi1gxlM5DMtMCdk+2nSCCMcbHG3aYpmNiUVZ/nRmXjmSG+et5P9WH2PGUznMvHIkWf7GDubYsJi6DqKpat7gHzy+giFpyrqFK/jhpIEcq6ji3Y2hHNAsunzvS/35wRMrmNgLFn0Q+g+VMSY8uiUnUlWtLNldxQ0T+4X8/9jTBCEi24DDQBVQqapjas2/CrgVZ/i+w8APVXV1Y9YNlSx/OqUVVSzZ49y08C9vbPJiN1Hp5S0wY/IgSw7GtEJLtx5k2mNLKK2sZmJmPM8u38nZJ3drdWcQk1W1oI55W4GJqlooIhfgjK87vpHrhsSi3AI6to9nuE9ZfUD4xfmnMCwz1ctdRtyanUX88dWNlFZU8fAHW8jy+8gaZEnCmNbilTW7+cn8HKpV+celI0gp/oz2fYaGvJopolVMqho4Ru9iThxo3HM19e8Pfnc05TvWevIGR5tFuQXc+/ZnPDptDPPeXsmrWyv43tzlPDptTMyW2ZhY8uiHW7n7lfX
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fixed_results_200e[:, 0], fixed_results_200e[:, 3], 'x-', label=\"Final Validation Loss\")\n",
    "\n",
    "# plt.ylim(0)\n",
    "\n",
    "plt.title('Final Validation Loss for Fixed Learning Rates')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Learning Rate')\n",
    "\n",
    "# plt.legend()\n",
    "plt.xscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01a797d7",
   "metadata": {},
   "source": [
    "# Step-Down\n",
    "80/10/10 Split, 100 epochs\n",
    "\n",
    "## Index\n",
    "0. learning rate\n",
    "1. step size\n",
    "2. gamma\n",
    "3. top-1 accuracy\n",
    "4. top-5 accuracy\n",
    "5. last val loss\n",
    "6. last val accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a9023eeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "step_down_results = np.array([\n",
    "    [1e-2, 0.33, 0.1, 43.79, 70.85, 2.79, 47.24],\n",
    "    [1e-2, 0.33, 0.25, 45.52, 73.07, 2.90, 49.88],\n",
    "    [1e-2, 0.33, 0.5, 45.71, 71.71, 2.89, 49.39],\n",
    "    [1e-2, 0.33, 0.75, 40.09, 68.19, 3.00, 46.38]\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9b32b8fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABEQ0lEQVR4nO3deXxU5b348c8362RPSCAiIEHciqyCqEgFUW5trStFQVuXutRarFa51qv9WevS68J1qVqrtgpuRLBal6qtIMtVruyroCwSlC2QkIQsZH9+fzxnJjOTSTIJmZmE+b55zWvm7N85Gb7Pc55zznPEGINSSqnoERPpAJRSSoWXJn6llIoymviVUirKaOJXSqkoo4lfKaWijCZ+pZSKMpr4w0hE8kTEiEhcEPNeIyKfhSOurkysl0WkRESWRToeFToicreI/DXScUQDTfwtEJECEakVkRy/8aud5J0XodC8Y0kVkQoR+SjSsYTQWGAi0NcYM7ozVigi14nIVyJSLiKFIvKhiKQ502aKyIOdsZ0Wtm1EpNL5uxWLyHwRuTxU2wsypqArJKFkjPmjMeb6UKzbb7/vEpHHRSQ2yGXvE5HXQhFXpGjib912YKp7QESGAMmRC6eZSUANMFFEjgrnhsOYJPoDBcaYyvYuGChGERkH/BGYaoxJA74HvHnYUbbPMGNMKnAiMBN4RkR+H+YYwirShYrDvd/HAZcDP49wPJFjjNFXgBdQAPwOWO41bgZwD2CAPGdcBvAKsB/Y4SwT40yLdZYpAr4BfuUsG+e17N+APcAu4EEg1pl2DfBZGzF+CjwErAKm+00bCywBSoHvgGuc8UnA/zixlgGfOePGAzsD7INznc/3AW8BrwEHgeuB0cD/OdvYAzwDJHgtfzLwCXAAKATuBo4CqoBsr/lOcfZfvN/2rwOqgQagAviDM/4GYKuz3veAo72WMc5+3gJsD7DPpgP/aGF/3gjUAbXO9t53xh8N/N2JcTvwa69l3PvlTaDc+VsMa+VvZoDj/Mb9xPme2V7be8/5fluBG5zxLuAQkOMM3wPUA+nO8APAk87nmcCzwD+duJYCA1uIKQ+v36XftNZ+owOxv8Fi7G/8dSDT7/fzW2AdtoJynLOdq4FvnWXu8duXr/nF1NK8ScAsoATYBNyJ3++3tf0OzAGe9Rp+Cvv/5CCwEvi+M/485/dQ5/wm1gaxX44DFmH/fxUBb0Y6nzXbH5EOoKu+nB/tucDX2FphLLATWwP1TvyvAO8Cac6PdTNwnTPtJuAroB/QA1iAb+J/B3geSAF6AcuAXzjTrqGVxO/E0QgMAu4A1vlNK8cercQD2cBwZ9qzwEKgj/OdxgCJBJf464CLsUeKScBI4HQgzvnum4DbnPnTnP8Ud2ATVhpwmjPtQ+CXXtt5Ani6he/psx+ACc5/plOcuJ8GFntNN9jCpgeQFGB938cmzz8AZwKJftNnAg96DcdgE8G9QAJwLLYQ/4HffvmJs6+nYwuH+Ba+T6DEH49N4D90hhcDf3b223BsgTPBa9ok5/O/gW1+y13i9T2KsYVzHDYp57cQUx4tJ/7WfqPHYZvhEoGezvaf9Pv9rMH+/pO8tvOiMzwMWyB8z2tf+if+luZ9GJtcs4C+2MIlqMQPnIT9bf7Ga/pPsf9P4rC/2b2Ayz+uIPfLbGyhHOP8DcdGOp812x+RDqCrvmhK/L8D/htb8n/i/DCM88OMxdYGBnkt9wtgofP5U+Amr2n/4f4PBuQ6P+Qkr+lTgQXO52toPfH/DljjfO6DrRWPcIb/C3gnwDIx2KQ3LMC08f7/cWie+Be3FI8zz23u7TrfZXUL810OfO58jnX+k41uYV6f/YCtZT3qNZyKTbx5zrDBSZKtxPlD4H3skUoF8DhNtbWZ+Cb+04Bv/Zb/L+Blr/3yhd8+3oNTYwyw7WaJ3xm/F7gSmyQbgDSvaf8NzHQ+PwD8yfkN7QVuxSZB99FAttf3+KvXOn4EfNVCTHkESPxt/UYDrOdi77+58/v5eYDt9PUatwyY4rUv/RN/S/N6Cl9n+HraTvwHgUrn82z8Cn2/+Utw/p/gl/jb2i/YyuAL3rF3tVdXaHfr6l7F1mQGYP+g3nKwtbUdXuN2YBMx2EP27/ymufV3lt0jIu5xMX7zt+YqbG0IY8wuEVmEPSxejU0e2wIsk4NNEIGmBcMnNhE5AZs0R2HPfcRha8e0EgPYI6S/iMgAbDt3mTEm2Ct2jsY2pwBgjKkQkWLsPi8IFKc/Y8xHwEciEgOcDczFHtk9H2D2/sDRIlLqNS4W+F+vYc/2jDGNIrLTiTMoIhKPrTEfcJY7YIwp95plB3Yfg63lPo494lmPrYz8DXvktdUYU+y13F6vz1XYQrI9Wv2Nikgutonk+9gjuhhswvQW6G/Rnrhamtf//1Yw/29Owf4mJ2MLyxRsAkdEpmObFo/GFgzp2P8vgbT1f/dObAG9TERKgP8xxrwURHxhoyd322CM2YE9dP8R8Lbf5CJsbbO/17hjsG1+YGt+/fymuX2H/dHlGGMynVe6MebktmISkTHA8cB/icheEdmLrZle4ZxE+w7b/uqvCNuWHGhaJV4nrp0rHnr6zWP8hp/DNmUdb4xJx7bhu/8nfIdtFmnGGFONbWP9KfAzbOEarN147W8RScEeou/ymsc/zoCMMY3GmPnYI7PBLSz7HfZcQabXK80Y8yOveTx/Y6cw6evEGayLsE09y5zlerivMnJ4/6aWYAvLS4BFxpiNzvQfYQuFztTWb/SP2P01xPn7/5Smv79bUH+LDtiD3c9u/Vqa0Zux5mDPTd0LICLfxybry4AsY0wmtn3e/V0C/SZa3C/GmL3GmBuMMUdjWwD+LCLHdeRLhoom/uBch20+8LmyxBjTgE1gD4lImoj0B27HngDFmfZrEekrIlnAXV7L7sG20f6PiKSLSIyIDHSuOmnL1dia3iBsG/BwbOJKwjZjvA6cKyKXiUiciGSLyHBjTCPwEvC4iBwtIrEicoaIJGLPTbhE5HynBvo7bNtta9Kwh88VInIS8EuvaR8AvUXkNhFJdPbPaV7TX8E241xI+xL/bOBaERnuxP1HYKkxpiCYhUXkIhGZIiJZzj0Co7FXeXzhzFKIb4G1DCgXkd+KSJKzzwaLyKle84wUkUudQvc2bFL4gjaISA8RuRJ73uURY0yxMeY7bHL/bxFxichQ7O/vNQBjTBX2qOpXNCX6JdjzSYeb+BOdbbpExIXdF639RtOwTWVlItIH+M/D3H57zMFWfLKcbU9r5/IPAzc4V8OlYQve/UCciNyLrfG7FQJ5TqHe5v9dEZksIu5CqQRbcDR27GuGhib+IBhjthljVrQw+RZsbfkb7BUyb2CTK9immH8Ba7HNE/5HDFdhTxhuxP5A3gJ6txaL8x/yMuzJ0L1er+3YBHq1MeZbbA3wDmzzwRrsyTGwJx/XA8udaY9gr0IqA24G/oqtXVZiT2a3ZjpwBfZE8ot4XRbpNFVMBC7AHq5vwTaruKd/jv3PsMo5qgqKMWYe8P+wV9nswR69TAl2eex+vsGJ5yA2oT5mjHndmf43YJCIlIrIP5zC/cfYwnU79qjpr9irOtzexZ63KMEewVxqjKlrJYa1IlKBvWLneuxJxnu9pk/FtnHvxp5E/L3zvd0WYZsalnkNp2GbJA9HBfY8gfs1gdZ/o3/ANp+UYa8e8v99h9L92N/ndmCeE1dNsAsbY9Zj99d/Yv+Pfoyt/OzAHhV7Nx3Ndd6LRcTdzNjafjkVWOr8jd8DbjXGfNPO7xdS4pyMUCrsRORT4A1jTLe9W1NE7sOerP1ppGOJZiLyS+yJ32COmKOe1vhVRDhNJacQ/pun1BFARHqLyJlOM8uJ2KPbdyIdV3ehV/WosBORWdhL/271u3pFqWAlYK/CGoC9LDcfe++DCoI29SilVJTRph6llIoy3aKpJycnx+Tl5XVo2crKSlJSUjo3oDDR2COju8beXeMGjT1UVq5cWWSM8b8fp3sk/ry8PFasaOlqytYtXLiQ8ePHd25AYaKxR0Z3jb27xg0ae6iISMBLpbWpJ5p89iRs97vUe/t
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(step_down_results[:, 2], step_down_results[:, 3], 'x-', label=\"Top-1 Accuracy\")\n",
    "plt.plot(step_down_results[:, 2], step_down_results[:, 4], 'x-', label=\"Top-5 Accuracy\")\n",
    "plt.plot(step_down_results[:, 2], step_down_results[:, 6], 'x-', label=\"Final Val. Accuracy\")\n",
    "\n",
    "plt.ylim(0)\n",
    "\n",
    "plt.title('Model Accuracy for Step Down Learning Rates')\n",
    "plt.ylabel('% Accuracy')\n",
    "plt.xlabel('Gamma')\n",
    "\n",
    "plt.legend()\n",
    "# plt.xscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "69c98182",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzmElEQVR4nO3dd5wV9fX/8dfZyhZ6Wekd6b0JNiwJaqzYGyAEifqNsUTUmPw0JpbERE3QWCm2ICgqGkssgFFUYIGloyy9CcvCsrvA1vP7Ywa4rHd37y733rl39zwfj/vYe2fmzrzv3Nk5M5+ZOyOqijHGGFNWjNcBjDHGRCYrEMYYY/yyAmGMMcYvKxDGGGP8sgJhjDHGLysQxhhj/LICUQkRyRORDkEYz4Mi8lowMpUZ7xgR+crndbl5yw5bjWl9JCKjq/v+SCQifxKRLBHZ5XUWE1oiskpEzvQ6RzSxAuESkU0icshdwR55tFDVVFXdEMLpthSRYhHp6KffOyLyRFXGF6y8/gqaqp6nqtNPdNx+pjVNRP4U7PEGMN02wF1Ad1U9KUjjvFhElonIAbfwfCEi7d1+IdlI8Jn2GBEp8Vl+N4rIVBHpEqppBpgrpJ87UKraQ1XnBXu8Zeb7ARHJEJFfVOH9m0TknGDnCgYrEMe70F3BHnnsCPUEVXU78Dlwg293EWkEnA8EfYVsjmoD7FXV3VV9o4jE+enWCXgFp+jUB9oDzwAlJ5izKr5R1VR3+ucAh4B0EekZxgxh5+/7CLMj870B8CwwQ0QaeJooCKxAVEJE1P3HP7Kl+4yI/EdEckXkO98tfxF5WkS2ulsR6SJyWoCTmU6ZAgFcDaxW1RUicq+IZLrTXC0ilwaYt7GIzHHzLAQ6lhnWb14RGQncD1zlbhVluN3nich493mMiDwgIptFZLeIvCIi9d1+7dwco0Vki7sl/bsA50XZz/NLEVkvItnuZ2nhdhcRedKd9gERWXFkJSgi57vzKVdEtovI3X7Gew7wKdDC/YzT3O4XuU0R+93P283nPZtEZJKILAfy/ayU+gIbVfVzdeSq6tuquqWCeVpfRF4WkZ1u1j+JSKzbb4yIfC0ik0UkR0TWisjZgcw3VS1R1UxVvQWYDzzo8zn8fkYRGSsi7/sM94OIzPJ5vVVE+rrPVUQmusPsd/8vJJBsvkRkqIgscMeRIT5NQG6eNe73uEFEbvbpd6aIbHO/j13AVHH2VGa6y2Ku+xkH+rzn6JZ6AMP2F5Glbr9ZIvKmBLCXq6qlwKtACtDZHVdHcfYk97r/C6+LWzxE5FWcDZX33eXingDmyxh3fuSKs5d4XVXne8BU1R7O5UY2Aef46a5AJ/f5NGAvMBiIA14HZvgMez3Q2O13F7ALqOP2exB4rZxpJwE5wKk+3b4BfuM+vwJogVPQrwLygeZuvzHAV+XknQHMxFlYewLbywxbpbzAPGC8+/wmYD3QAUgFZgOvuv3auTledD9bH6AA6FbO558G/MlP97OALKA/kAj8E/jS7fdzIB1ni02Abj7zZCdwmvu8IdC/nOmeCWzzed3FnbfnAvHAPe5nTPBZRpYBrYEkP+PrABwGngRGAKll+vubp+8Az7vfUTNgIXCzz3dbDNzh5rnKXU4alfN5jlsWfLrfBPxY2Wd08+/HWc5aAJuPzB+33z4gxmc5+8Cd/22APcDIcnL95HO73Vvi/D+d707zXPd1U7f/BTgbNQKcARw88l26310x8Li7bCS50znsji8WeBT41t//eEXDuvNiM3C7O48uAwrxs4yWne/uuG51h2/mduvkfrZEoCnwJfBUeeueiuaLu5wcAE52h20O9AjZejFUI462h/sl5bn/IPuBd33+EXwLxEs+7zkfWFvBOPcBfSr6J/EZ9iXgBfd5Z98FzM+wy4CLyy6cvnndBbUI6OrT7xH8rEACzcvxBeJz4Baffie704vjWIFo5dN/IXB1OdOd5u+fD3gZ+IvP61R3Gu1wisf3wFDclZbPcFuAm4F6lXznZ3J8gfg9MNPndQxOUT3TZxm5qZJxDsUpyntwVkDTcAtF2XkKpOEUziSfbtcAc32+2x2AlJmPN5Qz7eOWBZ/uI4GiAD/jVpyCfDXwgju9rsBYYE6Z5cx3g2YmcG85uX6yLLndJ+FuVPh0+wQYXc543gVu9/nuCnE3aHym85nP6+7AIZ/Xmzi+QPgdFjjdnSe+8/0rKi4QxTjrjSKcZr0rK1hGLgGW+stV2XzBKRD7gVH42UgJ9sOamI53iao2cB+XlDOM79kuB3FWWgCIyN3uLnGOiOzHaQduEuC0pwNXiEgdnOamT9RtGxeRG8U58LnfHW/PAMbbFGdlvdWn22bfAU4w75EtTN9xx+Gs9I4od15VZxqqmoezJdVSVb8AJuO08e8WkRdEpJ476Cic4r1ZROaLyCnVnF4pzvxr6TPM1rJv8qWq36rqlaraFDgNZ2VTXvNaW5wt1J0+3+3zOHsSR2xXdw3h2ozTLHaaHDsYvaqSz9USyA7wM87HWfme7j6fh7P1fob72teJfr9tcZb5/T6f/1ScrWJE5DwR+Vac5sX9ON+p7/K5R1UPV5Kpjp+mwMqGbcFP53uF3zvO3kcDnD3WOTjfPe7nSBORGW4T4gHgNSr+Pyt3vqhqPs6e5ESc5eY/ItK1kmzVZgUiSMRpv78HuBJo6C4sOTi7x4H4Cuef+GKcpp/p7njb4jTV3AY0dse7MoDx7sHZqmnt061NFfL6/nP4swNnQfYddzHwYyXvq4rjpiEiKThNYtsBVPUfqjoAZ+uvC/Bbt/siVb0YZ0X7Ls7WbXWmJzjzb7vPMJXNl2MDqi7CaXo7coC47Hu34uxBNPHZMKmnqj18hmlZpm2/DbBDVf+nx06m6EHFLgX+5z6v7DMeKRCnuc/nU36BOFFbcbaUG/g8UlT1MRFJBN4GngDS3OXzQ45f7gP+LqpoJz+d763LG9iXuxHzK+AGEenndn4EJ2svVa2H8/9d0ecod7640/hEVc/FKaRrcdYPIWEFInjq4qwg9wBxIvIHoF7FbznG3Vp5BadNtQFw5GBhCs4CtAecA3ccW+FUNL4SnJXTgyKSLCLdcXZRA837I9BORMpbRv4N3CEi7UUkFeef4E1VLQ7g4/oTKyJ1fB4J7jTGikhfd4XxCPCdqm4SkUEiMkRE4nHa1A8DpSKSICLXiUh9VS3Caa8tDTDDTOACETnbHe9dOCvwBYG8WUROFeegejP3dVfgIuBbd5Dj5qmq7gT+C/xNROqJc+C/o4ic4TPaZsCvRSReRK7AOdbyYQBZYt3v5p84K/yHAvyM83GOnySp6jacwjISpzAvDWQ+lCOmzPebiLMlfaGI/NzNW0ecg8+tcI4DJOJu6IjIecDPTmD6VfENzplnt4lInIhcjHPcMSCqmo3TZPwHt1NdnObrHBFpibsh4+NHnGM8R5Q7X9y9kYvdjaUCd7yBLt9VZgUieD4BPsZpF9+Ms8KqbLe0rFdwthDfVNUCAFVdDfwNZ6H9EegFfB3g+G7D2e3fhdMWPrUKeY+cvbJXRJb4GfcUnLM1vgQ2uu//vwBz+XMvTtvtkccXqvoZTpv52zhbdR1x2sbBKWYv4hw32YzT9PRXt98NwCZ3d34iENBZHqq6Dmfr7p84B8cvxDn1uTDAz7AfpyCsEJE8nPn7DvAXt7+/eXojzspwtftZ3sJtYnF9h3NMKgv4M3C5qu6tIMMp7rQP4DQP1QMGqeqKQD6jqn6Ps9L5n/v6ALAB+Nrd6Kiuazj++81U1a04e8z34xSCrTgrzxhVzQV+jVPQ9gHX4jTdhJw7Ly4DxuF8p9fjHJAvqMJongLOF5HeOMW5P84e+n9wNtx8PQo84DYn3V3RfHEfd+LsCWbj7Nn9qsofMkByfDObMSZSiMgYnJMCTvU6S20nIt8Bz6nq1EoHrkFsD8IYY8oQkTNE5CS3iWk00Btnj7BW8frXh8YYE4lO5thviDbgNO3t9DZS+FkTkzHGGL+sickYY4xfNaqJqUmTJtquXbtqvTc/P5+UlJTgBgqTaM0erbnBsnvFsgdfenp6lvvDzp+oUQWiXbt
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(step_down_results[:, 2], step_down_results[:, 5], 'x-', label=\"Final Validation Loss\")\n",
    "\n",
    "# plt.ylim(0)\n",
    "\n",
    "plt.title('Final Validation Loss for Step-Down Learning Rates')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Gamma')\n",
    "\n",
    "# plt.legend()\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9c67f09",
   "metadata": {},
   "source": [
    "# Exponential Decay\n",
    "80/10/10 Split, 100 epochs\n",
    "\n",
    "## Index\n",
    "0. learning rate\n",
    "1. decay rate\n",
    "2. top-1 accuracy\n",
    "3. top-5 accuracy\n",
    "4. last val loss\n",
    "5. last val accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5322e4d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_results = np.array([\n",
    "    [1e-2, 0.70, 2.35, 8.09, 4.97, 2.75],\n",
    "    [1e-2, 0.80, 5.0, 15.57, 4.61, 7.23],\n",
    "    [1e-2, 0.90, 25.88, 52.5, 3.28, 29.17],\n",
    "    [1e-2, 0.925, 37.43, 63.37, 3.13, 40.81],\n",
    "    [1e-2, 0.95, 44.1, 71.22, 2.99, 48.84],\n",
    "    [1e-2, 0.98, 44.41, 71.83, 3.04, 47.61],\n",
    "    [1e-2, 0.99, 42.43, 69.55, 3.25, 45.47],\n",
    "    \n",
    "    [1e-1, 0.85, 12.91, 34.96, 3.85, 14.89],\n",
    "    [1e-1, 0.9, 0.8, 2.29, 5.29, 0.55]\n",
    "])\n",
    "two_results = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "959af09b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABh8UlEQVR4nO2dd3gU1deA35NGEgKhJoQiobeQhI6A0sWKWKhKsxfsWNGfqIiiqNj5LFSBAGJBFAtIQEEChN5rqAFCSEJC+uZ+f8xk2YSUDWSzKfd9nnl25paZc2Zm75nbzhWlFBqNRqPRALg4WwCNRqPRlB60UdBoNBqNFW0UNBqNRmNFGwWNRqPRWNFGQaPRaDRWtFHQaDQajRVtFByMiASKiBIRNzvSjhGRf0tCrtKMGMwUkTgR2eBsecoCInKdiOyzM22pf89E5BUR+cbZclREtFGwQUSiRCRdRGrlCt9iFuyBThLNVhYfEUkSkeXOlsWB9AD6A/WVUp2v9mQ2hjkp1zb06kV1DqY+TbOPlVL/KKVaFMN57f6IcSRKqclKqQcccW5Tv4vmO3BSRD4UEVc7804Uke8cIVdpQRuFyzkCDM8+EJG2gLfzxLmMu4A0oL+I1CnJC5dgQdEQiFJKXSxqxkJkrKaU8rHZFl65iJorxdkGxyREKeUD9ASGAvc5WZ5SgzYKlzMXGGVzPBqYY5tARHxFZI6IxIjIURF5VURczDhXEZkqIudE5DBwSx55vxWRaPMrZZK9Xyk28kwHtgP35jp3DxFZJyLxInJcRMaY4V4i8oEpa4KI/GuG9RKRE7nOESUi/cz9iSLyvYh8JyIXgDEi0llE/jOvES0in4mIh03+NiLyl4icF5EzZjNAHRFJFpGaNunam/fPPdf17we+Aa41v+TeMMMfFJGD5nmXikhdmzxKRB4XkQPAgSLcS0TEQ0S2isgT5rGriKwVkf/lugcLRSRRRDaLSIhN/lYiEm7ej10iMtAmbpaIfC4iv5p5I0SkiU18S5t7tU9EhtiTV0TWmMm2Zdd4cj9LEXlJRA6ZeXeLyB1FuS/53Kt8310RaSIif4tIrPnuzxORajZ5o0TkRRHZDlwUkabmcxstIsfMPBNs0lu/yOVS7SW/tF4iMluM5sY9IvJC7vc6P5RSB4G1QKjN+T4W4/9zQUQiReQ6M/xG4BVgqHnft9lxX5qKyGox/nfnRKT0f4gopfRmbkAU0A/YB7QCXIETGF+uCgg0080BfgaqAIHAfuB+M+4RYC/QAKgBrDLzupnxPwL/B1QG/IANwMNm3Bjg3wLkawhkAa2B54DtueISMWo57kBNINSM+xwIB+qZOnUDKgG9gBN53QNzfyKQAQzC+IDwAjoAXQE3U/c9wNNm+ipAtCmbp3ncxYz7DXjU5jofAZ/mo2eO+wD0Ac4B7U25PwXW2MQr4C/zfnvlcb5A22eQR3wQEGc+8wnAesA11z2427yv4zFqk+7mdhCjoPAw5UwEWph5ZwGxQGfzfs0Dwsy4ysBxYKwZ187UsXVheW10bmpznONZAoOBuuZzGwpcBAIKe88KulcU/O42xWjyqwTUBtYA03K9V1sx/hdeNtf52jwOwagBt7K579/lkim/tO8Cq4HqQH2MD6YTeemX+94BLTHe2Wds4u/F+P+4YbzLpwHP3HLZeV8WYLxTLhj/iR7OLucKLQedLUBp2rhkFF4F3gFuxChs3MwXKRCjUE3P/vOa+R4Gws39v4FHbOJuyP6TAf7my+xlEz8cWGXu5/tnNeNfBbaa+/UAC9DOPH4Z+DGPPC5ACkZ1OXdcr9x/Hi43Cmvyk8dM83T2dU1dtuSTbiiw1tx3Nf9onfNJm+M+AN8C79kc+2AU1IHmsQL6FCBjoJkmPtfWyibNcxgfA3FAM5vwicD6XPczGrjO3E4DLjbxC4CJ5v4s4BubuJuBvTb3459ccv4f8HpheW10ztco5HEPtgK3F/aekY9RoJB3N4/zDLJ9F8z36r48rlPfJmwDMMzmvuc2CvmlPQwMsIl7oJB7oYALGIZSmc+sUgHp4zD/P+QyCoXdF4wPyK9sZS/tW2lo2yuNzMX40mlErqYjoBbGF+JRm7CjGIU0GF9nx3PFZdPQzBstItlhLrnSF8QojK8llFInRWQ1RnPSFowvsEN55KmF8YWSV5w95JBNRJoDHwIdMfpa3IBIMzo/GcCoWU0XkUZACyBBKWXvyKK6wObsA6VUkojEYtzzqLzkzIdaSqnMfOJmA28DS5RSuZugrOdWSmWZTRPZzVfHlVJZNmlt3wUwjEY2yRgGDYx3oYuIxNvEu2G8e4XlLRQRGQU8i1GgYuatlW+Gwinw3RURf+BjDENZxYyLy3WOvJ5RUXTML23u/5w970J7jHd1MEZNozJG4Y6IjAfuN8+rgKrkf+8K+0+/ALwFbBCROOADpdQMO+RzGrpPIQ+UUkcxmghuBn7IFX0O4yu1oU3YNcBJcz8ao3C0jcvmOMaLV0spVc3cqiql2hQmk4h0A5oBL4vIaRE5DXQBRojRcXccaJJH1nNAaj5xF7HpRDfbQWvnSqNyHX+J0TzWTClVFaPpJPvfcBxonJf8SqlUYBFG1XwkOQu/wjiFzf0WkcoY1fuTNmlyy1lUvgCWAQNEpEeuOOvzFKPvqL4p0ymggRmWje27UBDHgdU270E1ZXR+P3pVWhgyNsT4eBgH1FRKVQN2cuk5XQmFvbuTMZ5BW/O9uDeP613tM8qPaIxnkk2D/BLaogwWAf8B2X1I12EU5EOA6ua9S+CSLrl1KPC+KKVOK6UeVErVxWhR+EJsRo2VRrRRyJ/7MZokcoyAUUpZMAq3t0WkivkHfBbIHqa2CHhSROqLSHXgJZu80cCfwAciUlVEXMwOup52yDMaoymrNUanWChGW7gXcBNGm3M/ERkiIm4iUlNEQs2v2BnAhyJSV4yO1GtFpBJGX4iniNwiRofvqxhtwgVRBaPqnSQiLQHbQmwZECAiT4tIJfP+dLGJn4PRdDGQohmFBcBYEQk15Z4MRCiloopwjnwRkZEYfSVjgCeB2SJi+8XaQUTuNI3v0xiFwHogAuOL9QURcReRXsBtQJgdl10GNBeRkWZedxHpJCKt7BT7DPkYYIyvXgXEmPqNxXhXikIlEfHM3szrFfTuVgGSgAQRqQc8X8TrXQ2LMD6WqpvXHlfE/O8CD4oxmq8KkIlx79zEGHBQ1SbtGSAw+0OgsP+0iAwWkWyDFYfxXGxrlqUObRTyQSl1SCm1KZ/oJzC+sg8D/wLzMQpeML7Q/gC2YTR55K5pjMLolNyN8ZJ8DwQUJIv5pxyC0TF72mY7glG4jlZKHcOo2TwHnMdoQw4xTzEe2AFsNOOmYLSDJwCPYYz2OWnqVNiojfHACIwO1a8B62gKpVQiRmfjbRhV/QNAb5v4tRh/iM1mbcwulFIrgNeAJRhfhU2AYfbmtyFecs5TeFZErgGmAaOUUklKqfnAJoyO8Gx+xugDiMOo5dyplMpQSqWbut6EUSP7wjzPXjt0SsTobxqGUeM4jfFcCjPK2UzEMF7xYjNqyTz3buADjC/gM0BbjBE2RSEJoy8qe+tDwe/uGxhNMgnAr1z+3juSNzHe2yPAClOuNHszK6V2YDQXP4/x3/0d44PpKEYt27Y5arH5Gysi2U2aBd2XTkCEiCQBS4GnlFKHi6hfiSJmZ4hGUyKIyN/AfKVUmZitKiITMTp07y0sraZ0ICKPYnRC21MD1+RC1xQ0JYaIdML4miz9Y7U1ZQYRCRCR7mbTTQuM2vKPzparrKJHH2lKBBGZjTFM8Smz6USjKS48MIbzNsIYahyG0ZSnuQJ085FGo9ForOjmI41Go9FYKRPNR7Vq1VKBgYFXlPfixYtUrly5eAVyMuVNJ61P6ae86VTe9IG8dYqMjDynlMo996hAyoRRCAwMZNOm/EaHFkx4eDi9evUqXoGcTHnTSetT+ilvOpU3fSBvnUTE7qHf2ejmI41Go9FY0UZBo9FoNFa0UdBoNBqNlTLRp5AXGRkZnDhxgtTU1ALT+fr6smfPnhKSqmQobzpdiT6enp7Ur18fd3f3whN
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(exp_results[:two_results, 1], exp_results[:two_results, 2], 'x-', label=\"Top-1 Accuracy\")\n",
    "plt.plot(exp_results[:two_results, 1], exp_results[:two_results, 3], 'x-', label=\"Top-5 Accuracy\")\n",
    "plt.plot(exp_results[:two_results, 1], exp_results[:two_results, 5], 'x-', label=\"Final Val. Accuracy\")\n",
    "\n",
    "plt.ylim(0)\n",
    "\n",
    "plt.title('Model Accuracy for Exponential Learning Rates')\n",
    "plt.ylabel('% Accuracy')\n",
    "plt.xlabel('Decay Rate')\n",
    "\n",
    "plt.legend()\n",
    "# plt.xscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fe8641ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9gklEQVR4nO3dd3hUZfbA8e9Jg0BCaCEEQk3oGEqwr0qwoSBs0RUFV10VXetafipbLKyurrprw7quDUvEuoAFCyCriEKAhI70EpBeQg+c3x/3BYeQMpNkMpPJ+TzPPLlz6zl3ysm97537iqpijDHG+Csq1AEYY4ypWaxwGGOMCYgVDmOMMQGxwmGMMSYgVjiMMcYExAqHMcaYgFjhKIGIFIpI+ypYz30i8kZVxFRsvVeIyDc+z0uNt/i8FdjWpyJyeUWXD0ci8oCIbBKR9aGOpSYQkdbuPRbtx7x9RWRNdcRVUSIyVEQ+D3UcNVmtLhwiskJE9rgPxeFHC1VNUNVlQdxuSxEpEpH0EqZ9KCKPBbK+qoq3pEKnquep6muVXXcJ23pVRB6o6vX6sd3WwO1AV1VtXkXrVBHZVex9dGdVrDsU3OfirMPPVXWVe48drIJ1q4hkVHY9laGqb6rqOcFYd7HvlPXufZ7g57KV+ievOtXqwuFc4D4Uhx8Fwd6gqq4FvgIu8x0vIo2B84Eq/6I2R7QGNqvqhkAXFJGYMib3KPY+eqTiIZqKEk+ov9cuUNUEoCfQCxgR2nCqXqh3cFjy/a/I/cfwjIh8LCI7ReR73yMFEXlSRFaLyA4RyRWR0/zczGsUKxzAEGC+qs4RkbtFZKnb5nwR+ZWf8TYRkbEunh+A9GLzlhiviPQH/gRc7P5bynPjJ4vI1W44SkT+IiIrRWSDiLwuIkluWlsXx+UissqdCvqzn/uieD7XiMgSEdnicmnhxouIPO62vUNE5ohIdzftfLefdorIWhG5o4T1ngV8AbRwOb7qxg8SkXkiss3l28VnmRUicpeI5AO7yikeJeXyiYj80+d5joi87IavEJFvRWSUiGwXkYUicqbPvC1c/lvc/rjGZ9p9IjLGvQY7Xfx9ii37vohsFJHlInKzP8uKyGi84jrO7aM7fV7bGDfPlSKywC27TESuDWSflLKf6ojIY+6985OIPC8i8W5aIxEZ73LZ6obTfJadLCIPisi3wG6gvYv3OhH50b2uz4iI+Ox331O9Zc0bLSL/dO/n5SJyo+++KIuqrgcm4BWQw9sq8XPt3nPPAye7/b7Nj/3S1O2Lbe498j+prqKpqrX2AawAziphvAIZbvhVYDNwAhADvAnk+Mw7DGjipt0OrAfqumn3AW+Usu14YDvwC59x3wF/dMMXAS3wivvFwC4g1U27AvimlHhzgDFAfaA7sLbYvAHFC0wGrnbDvweWAO2BBOADYLSb1tbF8W+XWw9gH9CllPxfBR4oYXw/YBPQG6gDPA1McdPOBXKBhoAAXXz2yTrgNDfcCOhdynb7Amt8nnd0+/ZsIBa40+UY5/MemQ20AuJLWeeR/V/CtObABpfXUGAZkOjzOhYBt7ptX+zeE43d9CnAs0BdvC+fjUA/n9dqL94RajTwEDDNTYty++keIM69XsuAc8tbtqTPhc9rG+OeD8D7h0SAM/C+rHuXtH/93VfA48BYoDGQCIwDHnLTmgC/Aeq5ae8CHxV7j64CuuG9r2Pddsa790prt+/6l/H5KW3e64D5QBre++pL331R1neKW2YO8KTPdL8/137sl4fwik2se5wGSLV8d1bHRsL14V7kQmCbe3xU/M2N9wX3ks8y5wMLy1jnVrzTFlBG4XDTXwJedMMdgP1As1LmnQ0MLukNdjhevC+BA0Bnn2l/L/5mDCReji4cXwHX+0zr5LYXw89fLmk+038AhpSy3VcpuXD8B3jE53mC20ZbvC/fxcBJQFSx5VYB1wINynnN+3J04fgrMMbneRRese3r8x75fTnrVGCHz/toG+5L2k3/DbAaryD6/qNwBVCAz4fd7bPL8ArVQVyRcdMeAl71ea2+9JnWFdjjhk8EVhWLcQTwSnnL+uRcauEoIf+PgFtK2r+l7KuMYuME7ws03WfcycDyUtbRE9ha7D06soTt+O7rMcDdZXx+Spt3InCtz7SzytkXK/C+U3a6+b4CGpaxP2ZT+ue6zP0CjAT+W3x/VsfDTlXBL1W1oXv8spR5fK++2Y33ZQaAiNzhDtu3u8PLJKCpn9t+DbhIROrifVlMUHfuXUR+JyKz3WHoNryjh/LWm4z3Jb7aZ9xK3xkqGW+LYutb6baX4jOu1H1VkW2oaiHeEV9LVZ0IjAKeATaIyIsi0sDN+hu8or5SRL4WkZMruL1DePuvpc88q4svVILePu+jhqo6wWfaOLyivkhVizd+rlX3LeCsdDG1ALao6s5i03zjKr6v67pTKG3wTsdt83n//ImyX6fDy5ZLRM4TkWnu9Mg2vP3u73uoJMl4RxO5PvF+5sYjIvVE5AXxTpHuwDsSayhHX+VV0msUyHuxtHlbFFu3P++FX6pqIl4R7YzPvgnwc13mfgEexTs6/tydMrzbj9iqhBWOShCvfeBO4LdAI1VtiHeqQfxcxTfAFmAw3imk19x62+Cd8rkRaOLWO9eP9W7EO/XRymdc6wDi9f0CK0kB3peS77qLgJ/KWS4QR21DROrjnapYC6CqT6lqFt5/yR2B/3Pjp6vqYKAZ3n/AYyq4PcHbf2t95ilvv5TnQWABkCoilxSb1vLw+XSntYupAGgsIonFpvnGVZrVeP+V+hayRFU93894S81XROoA7wOPASnuPfQJ/r/nS7IJ2AN084k3Sb0GZvBOqXYCTlTVBsDph8PxJ+ZKWod3yumwVqXNWJyqfo13ZP0Y+PW5Lp5DmftFVXeq6u2q2h4YBNwmPm1kwWSFo3IS8b44NwIxInIP0KDsRX7m/tN8HfgH3vnVcW5Sfbw30UbwGiPx/jMpb30H8dod7nP/pXUFLg8g3p+AtmU0sL0N3Coi7cS7xPDvwDuqWuRHuiWJFpG6Po84t40rRaSn+5L6O/C9qq4QkeNF5EQRicU7hN8LHBKROPGuzU9S1QN4p40O+RnDGGCAiJzp1ns7XtvM1ArmdBQROR24Evgd3mvxtIj4HjU0A24WkVgRuQiv3eYTVV3tYnjI7ZtM4CrAn98F/QDsFK9RP9418HYXkeP9DPsnvHaRksThtT1tBIpE5Dwg0Etb43xfd7wvzn8Dj4tIMzhyyfq5bv5EvC/QbeJdeXhvgNurjDHALS6ehsBdAS7/BHC2iPSg/M/1T0Ca+xwcPvotdb+IyEARyXD/eGzHO7Xp7/u+UqxwVM4EvEPHxXinEfbi36Gsr9fx/pN8R1X3AajqfOCfeI3lPwHHAd/6ub4b8Q6z1+P9t/NKAPG+6/5uFpGZJaz7ZWA03qmC5W75m/yMqyR3430hHH5MVNUv8dod3sf7by8d72oz8Ircv/HaZVbincJ61E27DFjhTmVch9cQXS5VXYR3tPc03n94F+BdTrk/wFzy5OjfcTzhTqO9DtyoqmtV9X94bTiv+BxlfI/XvrUJ78jkQlXd7KZdgte+UAB8CNzr9k95OR0EBuK1BSx3634J77SkPx4C/uJOjxx1dZo7dXYz3hfqVuBSvMbbQMzj6Nf9Srwv5CXANPcafol3lAHel2+8y2Ma3nu4uvwb+BzIB2bhHV0V4X1Jl0tVN+K9B+7x43M9EW/frBeRTW5cWfulg3te6Nb5rKpOqliagZGjT68aY6qLiFyBd+HBL0Idi/GPO8J6XlXblDtzBLMjDmOMKYU71Xe+iMS4U4z34h391WpWOIwxpnQC3I93Wm4W3kUO94Q0ojBgp6qMMcYExI44jDHGBCSg++6Es6ZNm2rbtm0rvPyuXbuoX79+1QUUYpGWD0ReTpGWD0ReTpGWDxybU25u7iZVTS5jkWNETOFo27YtM2bMqPDykydPpm/fvlUXUIhFWj4QeTlFWj4QeTlFWj5wbE4isrL0uUtmp6qMMcYExAqHMcaYgAS1cIjXl8Ecd1OvY84jiecp8foayBeR3j7TLhf
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(exp_results[:two_results, 1], exp_results[:two_results, 4], 'x-', label=\"Final Validation Loss\")\n",
    "\n",
    "# plt.ylim(0)\n",
    "\n",
    "plt.title('Final Validation Loss for Exponential Learning Rates')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Decay Rate')\n",
    "\n",
    "# plt.legend()\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dfbd4bb",
   "metadata": {},
   "source": [
    "# Sigmoid Decay\n",
    "80/10/10 Split, 100 epochs\n",
    "\n",
    "## Index\n",
    "0. learning rate\n",
    "1. step size\n",
    "2. gamma\n",
    "3. top-1 accuracy\n",
    "4. top-5 accuracy\n",
    "5. last val loss\n",
    "6. last val accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dc7ccd8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_results = np.array([\n",
    "    [1e-2, 50, 0.05, 46.94, 72.88, 2.79, 52.94],\n",
    "    [1e-2, 50, 0.1, 45.95, 73.63, 2.65, 51.29],\n",
    "    [1e-2, 50, 0.15, 41.94, 68.56, 2.94, 47.49],\n",
    "    [1e-2, 50, 0.2, 41.82, 68.13, 2.82, 45.16]\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5ab4be17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABFY0lEQVR4nO2deXwV5fX/3yd7QjZIIIRFgqgosgqiIpWIUu0iaq27xa1aa7W1lq+12l9LXb5ftdRq1Vptq4gLEax7RatIpIqyLyIoa1AghC2EhOzJ+f0xk5t7k5vkhtybXLjn/XpNZnmeeZ7PzNycZ50zoqoYhmEYkUNUVwswDMMwOhcz/IZhGBGGGX7DMIwIwwy/YRhGhGGG3zAMI8Iww28YhhFhmOEPM0QkR0RURGICiHuNiHzcGbrCGXF4VkSKRWRxJ+T3NxH5f6HOp735ur+bYzpTU3sQkaNEpExEortaS6Rjhr8DiEiBiFSLSGaT4yvcf8KcLpLmrSXZ/Web29VaQsh4YBLQT1XHBiNBEbleRL4UkVIRKRKRd0QkBUBVb1LVe4ORT3voSL4iki8iPw62pvagql+rarKq1gU7bRGZ4f4vlonIPhF5X0SOD/DcgCtbRwpm+DvOFuDyhh0RGQYkdZ2cZlwEVAGTRKR3Z2bcif9IA4ACVT3Y3hP9aRSRCcD/AperagpwAvByh1Ue4YSB4XxIVZOBvsB24J9drCdsMcPfcZ4HpnjtXw3M9I4gImkiMlNEdovIVhH5rYhEuWHRIjJdRPaIyGbge37O/aeIFIrIdhG5r51N5auBvwGrgauapD1eRBaKyH4R+UZErnGPJ4rIn1ytJSLysXssV0S2NUmjQETOdrenicgrIvKCiBwArhGRsSLyqZtHoYg8LiJxXuef6NbO9rk167tEpLeIlItIhle8k9z7F9sk/+uBfwCnubW9P7jHbxCRjW66b4pIH69zVER+JiIbgA1+7tnJwKequgJAVfep6nOqWuqeP0NE7vNK7w732naIyI+9u1zcuH8Vkbmuvk/c63vE7Zr6UkRGeaV1gls73y8iX4jIZK+wpvn+j1e+1/l7+IEgIteJyDpXz3siMsAr7FH3t3FARJaJyLe8wvw973wRude9zlIR+Y+4LeKmNevW4rrhU9zf4F4R+X/ev7XWUNUKYDYw0iut74nTEj/gXs80r1MWuOv97jM6rbX7Ig5/FpFdbnqfi8jQdt/4rkRVbTnEBSgAzga+wqkVRgPbcGqgCuS48WYCbwApQA6wHrjeDbsJ+BLoD/QA5rvnxrjhrwFPAd2AXsBi4Cdu2DXAx63oGwDUA0OAXwGrm4SV4rRWYoEMYKQb9gSQj1NzigbGAfFALrDN3z1wt6cBNcAFOJWKRGA0cCoQ4177OuA2N34KUOhqS3D3T3HD3gF+6pXPn4HHWrhOn/sATAT2ACe5uh8DFniFK/C+e78T/aT3LaAC+ANwOhDfJHwGcJ+7fS6wEzgRp6X3gpv+MV5x97j3IQH4EKeVOMW9t/cB8924scBG4C4gzr2OUmBwC/kWAUPd38ZL3vn6uaZ84Md+jp/v5nmC+4x+Cyz0Cr/K/W3EuM9pJ5DQyvPOBzYBx3ntP+DGz8H3t91a3CFAGU43Xhww3c3r7Bauz/vedMOpkK3yCs8Fhrk6h7v37gJ/utq6L8A5wDIgHRA3TnZX26N22a6uFnA4LzQa/t8C/+f+M77v/lDU/UFFA9XAEK/zfgLku9sfAjd5hX274UcIZOF00yR6hV9Oo6G4htYN/2+Ble52X6AOGOXu/wZ4zc85UThGb4SfsFzaNvwLWtLjxrmtIV/3Wla0EO9S4BN3OxrH4IxtIa7PfcBp4j/ktZ/sGo0cd1+BiW3o/A7wFrAfxwA9DES7YTNoNDLPAP/ndd4xNDf8f/cKvxVY57U/DNjvbn/Lvc4or/BZwLQW8n3AK95xHJrhn4tbCfF6/uXAgBbSKW74bfh73m4+v/Xavxl4193Oobnhbynu74BZXmFJOP9HrRn+Svd51eMUrsNbeb6PAH/2p6ut+4JTIK/HqdBEtZRHOC/W1RMcngeuwDFAM5uEZeLU5LZ6HduKY4gB+gDfNAlrYIB7bqHb9N+PU/vvFaCuKcCLAKq6HfgIp+sHnBbGJj/nZOLUTP2FBYL3tSAix4nI2yKy0+0O+F83j9Y0gNNCGiIiA3EGbktUNdAZO33wuo+qWgbspfGeN9PZFFWdq6rn4bQKzsd5tv4GR5s+P3/pFnltV/jZT/ZOS1XrvcK9fyut5bvVT5xAGAA86vX72odTi+0LICJT3e6OEjc8jcbnB/6vd6fXdjmN1+ePluL6XJ+qluM8w9aYrqrpOIa8AhjcECAip4jIfLe7sASnpZ3pNxWHFu+Lqn4IPI7TMt4lIk+LSGob2sIKM/xBQFW34tQwvgu82iR4D05tc4DXsaNwBp/A6ero3ySsgW9wavyZqpruLqmqemJbmkRkHHAs8BvX6O4ETgGucPtYvwEG+Tl1D07NyV/YQbwGrsUZa+jZJI422X8SpyvrWFVNxenGEK/rO9qfflWtxOmnvQr4EU7hGig78LrfItINp7tiu1ecpjr9oqr1qjoPp2Xmrx+3EOjntd/fT5xA2QH0F3f8x8X7t9I035Z+N+3hG5yuw3SvJVFVF7r9+XcAlwDdXaNaQuPzgwDv4yHgc19FJBHnGbaJqn4N/ALHcCe6h18C3gT6q2oazrhXw3X4u4YW74ubx19UdTROl9RxwP+0+wq7EDP8weN6nO4Dn5kl6kxdmw3cLyIp7gDR7Th9wbhhPxeRfiLSHbjT69xC4D/An0QkVUSiRGSQOLNO2uJqnG6nITiDXCNxDFciTjfGi8DZInKJiMSISIaIjHRrm88AD4tIH3EGn08TkXic5m2CO1AWi9OVFN+GjhTgAFAmzvS6n3qFvQ1ki8htIhLv3p9TvMJn4tS0J9M+wz8LuFZERrq6/xdYpKoFgZwsIueLyGUi0t0dyBsLTAA+8xN9tpvXCSKSBHRkfv8inFrvHSISKyK5wHlAXgv5XiMiQ9x8fx9A+jEikuC1xOIYwN+IyIngmUxwsRs/BagFdrvn/g7orJrtK8B5IjJOnMkA0/AtcFpFVd/HKUhvdA+lAPtUtdJ9nld4Rd+N0z3kXQlp8b6IyMluCyIWpzJU6Z5/2GCGP0io6iZVXdpC8K04P5DNwMc4tY9n3LC/A+8Bq4DlNG8xTMEZ3FqL07/6CpDdmhYRScCppT2mqju9li04BvRqt1b0XZwBu33ASmCEm8RU4HNgiRv2IE5fZglOP+w/cGqhB3EGs1tjKs4/Wal7rZ5pkerMkpmEY9x24sywOdMr/BOcf6jlbqsqIFT1AxwD/C+cmuMg4LJAz8e5zze4eg7gFNJ/VNUX/eQ1F/gLzqD8RhoLh6p25NeQVjXOvfgOTsvrr8AUVf2yhXwfwWmJbHTXbfEkThdIw/Ksqr6G83zz3K64NW7+4Pwu38Up8LfiGLhWu8iChap+gfN/k4fzDMuAXbTvvv4RpxCNx/nd3iMipTjjB7O98ioH7gc+cbt2Tm3jvqTi/JaLce7LXjevwwZxBy4MIywRkQ+Bl1T1H12tJRBE5AQcIxGvqrVdredIQUSScQZuj3UrMEYHsBq/EbaIyMk4UzLD+uUpEbnQ7arqjlNLfMuMfscRkfNEJMkdo5mO0wot6FpVRwZm+I2wRESeAz7AmfNf2tV62uAnON0Qm3CmzP609ehGgJyP00+/A2eiwmVqXRRBwbp6DMMwIgyr8RuGYUQYXe1UKSAyMzM1Jyenq2UAcPDgQbp169bVMlol3DWGuz4wjcEg3PVB+GvsqL5ly5btUdWm79ocHi4bRo8ereHC/Pnzu1pCm4S7xnDXp2oag0G461MNf40d1QcsVXPZYHQZHz8CWxb4HtuywDluGEanYobf6Bz6ngRzrmk0/lsWOPt9T+pKVYYRkRwWffzt5uNHHIMy8IzGY1sWwPblMP62rlLVNdTXQ30N1NVAXTXU1zrrOvdYvXu8zj3uiesdv6aVsADTqq+BlN7w/IWcGpMG/y2FIRdAyTb4ehFkDIK
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(sig_results[:, 2], sig_results[:, 3], 'x-', label=\"Top-1 Accuracy\")\n",
    "plt.plot(sig_results[:, 2], sig_results[:, 4], 'x-', label=\"Top-5 Accuracy\")\n",
    "plt.plot(sig_results[:, 2], sig_results[:, 6], 'x-', label=\"Final Val. Accuracy\")\n",
    "\n",
    "plt.ylim(0)\n",
    "\n",
    "plt.title('Model Accuracy for Sigmoid Learning Rates')\n",
    "plt.ylabel('% Accuracy')\n",
    "plt.xlabel('Gamma')\n",
    "\n",
    "plt.legend()\n",
    "# plt.xscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "49f5ca1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/qklEQVR4nO3dd3xUdfb/8ddJhxQ6ofcmUg3SLYiwrq4NsaFYEVFwse26q9/ddbs/CzZUQFRERawoawOECNJ7LyH03ktCSzu/P+5FxzhpZCZ3kpzn4zGPTOZ+7r3vuXNnztwy9yOqijHGGJNbmNcBjDHGhCYrEMYYY/yyAmGMMcYvKxDGGGP8sgJhjDHGLysQxhhj/LICEQAiki4iTQIwnadF5P1AZMo13btEZLbP/3nmzd32HOb1rYjcea7jhyIR+ZeIHBSRvSUwrydFZGyw51PU+YrIVhG5vCQzFVWg3ofmZ1YgisB9k5xyV8SztzqqGqeqm4M437oikiUiTf0MmyQizxdleoHK66+gqepvVfXd4k7bz7zGici/Aj3dQsy3AfAY0FpVawVomteKyHIROe4Wnhki0hhAVf+jqoMCMZ+iKM58vXptcgvW+9BdzzPd9/tREZkrIt2KML6KSLNA5yoJViCK7mp3RTx72x3sGarqLmA6MND3cRGpClwJBPwD2fykAXBIVfcXdUQRifDzWDNgPE7RqQQ0Bl4DsouZs0zztyxL2EeqGgdUB5KBTzzOUyKsQASA7zcE99vUayLytYikicgC32/+IvKyiOxwvz0uEZGLCjmbd8lVIIBbgLWqukpE/iQim9x5rhWR6wuZt5qITHbzLASa5mrrN6+IXAE8CdzsfrNa4T7+g4gMcu+Hicj/icg2EdkvIuNFpJI7rJGb404R2e5+k36qkMsi9/O5T0RSReSw+1zquI+LiLzozvu4iKwSkTbusCvd5ZQmIrtE5HE/070cmAbUcZ/jOPfxa0Rkjftt8gcROc9nnK0i8oSIrARO+Plg6wBsUdXp6khT1c9Udbs7/i+2ykTkDnf5HRKRv/ju6nHbfiIi77vPY5WItBCRP7vPeYeI9PWZVh13+Rx2l9d9PsNyz3egz3zP6XVxp/M7cbaWzn7zbuczLM91VpxdnXPc1+8Q8HQh3ltFeR/2FZENInJMRF4XkZln19v8qGoW8AFQV0RquNPqLCLz3Oe4R0RGikiUO2yWO+oKdx26uRDL5Ql3nUxzM/Y+t6UfAKpqt0LegK3A5X4eV6CZe38ccAjoDETgrEwTfdreDlRzhz0G7AVi3GFPA+/nMe8KwDGgp89j84CH3fs3AnVwiv7NwAmgtjvsLmB2HnknAh8DsUAbYFeutkXKC/wADHLv3wOkAk2AOOBz4D13WCM3x5vuc2sPnAHOy+P5jwP+5efxy4CDwAVANPAqMMsd9htgCVAZEOA8n2WyB7jIvV8FuCCP+V4K7PT5v4W7bPsAkcAf3ecY5bOOLAfqAxX8TK8JcBp4EegFxOUa/tMyBVoD6UBPIAp4HsjEXQfdtqfd5xmBs2WyBXjKzXYfTjE6O+1ZwOtADE6hOgBcls98L3aX6QggCz/rfgGvTUdgP9AFCAfudJdPdCHX2SzgIfe5VaDg91ah3oc4WwHHgX7usOHuch2Ux/PzXTZRwDM461yE+1gS0NWdViNgHe77MneugpYL0BLYAdTxeZ809ewzz6sZl8ab+yKmA0fd2xd5rJhjfca5ElifzzSPAO1zr4h5tB0LjHHvNwcygJp5tF0OXOvevws/BcJdOTOBVj7D/uPbtqh5+WWBmA486DOspTu/s28kBer5DF8I3JLHfMfh/0PoLeBZn//j3Hk0wikeKe6bNyzXeNuB+4GEAl7zS/llgfgL8LHP/2E4RfVSn3XkngKm2RWnKB/A+YAfh1so+OWH0V+BD33Gq+i+5r4FYprP8Ktx1s9w9/94dxlXxilY2UC8T/v/AuPymK/vB2+s73yL8Nq8Afwz12MbgEsKuc5u9zOfPN9bFPJ9CNwBzPMZJjgfyvkViAyc93w2TuG5NJ/X92Fgkr9cBS0XnPflfuByIDK/9agkbraLqeiuU9XK7u26PNr4nu1yEudDCwAReVxE1rmbtkdx9kNXL+S83wVuFJEYnN1NU9TdN+7uiji7yXoUZ2ugoOnWwPmw3uHz2DbfBsXMWyfX9La580v0eSzPZXUu81DVdJw3cF1VnQGMxNnHv19ExohIgtv0BpwPjW3u7oXCHnTMPb8cnOVX16fNjtwj+VLV+ap6k6rWAC7C+abubzdOHd9pqepJ97n52udz/xRwUFWzff4HZ5nWAQ6rappP+225cuc13xN+5lsYDYHHzq6T7vpT351+YdZZf8uxKOtLXm1zPz8FdhbwXD5W1co46+5qnK0G3OfRQkS+EpG9InIc50tWfu+RPJeLqqbiFJincdbZieLuMvWCFYgSJM7++z8CNwFV3BXuGM43mMKYDRwGrsXZ9fOuO92GOLtqhgHV3OmuLsR0D+Bsxtf3eaxBEfJqAdPfjfNm8J12Fr/8UCuuX8xDRGJxdontAlDVV1Q1CWe3SQvgD+7ji1T1WqAm8AXON/pzmZ/gLL9dPm0KWi4/N1RdhLPrrY2fwXuAej7zqoDz3M7FbqCqiMT7PNaAX+b2ne9P64SIVDzH+e4A/u3zhaqyqlZU1Q8Luc4WejkWUe7lKr7/50dVDwKDcY6J1HYffgNYDzRX1QScY3P5vffyXC7uPCaoak+c9UyB/1e0pxc4ViBKVjzOB+QBIEJE/gok5D/Kz9xvOuNxVpjKwP/cQbE4K9IBABG5G/8fOLmnl43z4fS0iFQUkdY4+0MLm3cf0EhE8lqPPgQeEZHGIhKH883qI3UO9J2LcBGJ8blFufO4W0Q6iEi0O48FqrpVRC4UkS4iEomzf/s0kCMiUSJym4hUUtVMnP3ROYXM8DFwlYj0dqf7GM6xk7mFGVlEeopzUL2m+38r4Bpgvp/mnwJXi0h397k+TeG/TPyCqu5wM/7XXXbtgHsBf7+7+RT4nZs1CvgHBX9W+Htt3gSGuK+BiEisiFzlFqlzWmcD5GugrYhcJ85JBEOBQp/CrKobgCk4X57AeZ8cB9Ld1/OBXKPswzn2dFaey0VEWorIZe66fBpnK7Cw62bAWYEoWVOA73D2i2/DWQHy3R3hx3icb34fqeoZAFVdC7yAc9B6H9AWmFPI6Q3D2fTei7Pf9p0i5D17qt8hEVnqZ9pvA+/hHBzd4o7/UCFz+fMnnDfM2dsMVf0e57jAZzjfDJvinN0FTjF7E+e4yTac3STPucMGAlvdXQJDgNsKE8D9cLgd52D4QZz9/lerakYhn8NRnIKwSkTScZbvJOBZP/Nag7O8JrrPLR1n//SZQs4rt1txjs3sduf5N3f5+ZvvUGCCO98jFLwLxt9rsxjnQPlIdxqpOMcWirvOFou7FXAjzjI/hLN1uZiiLdfngMFuoX8cGACk4axvH+Vq+zTwrrs76ab8lgvOgeqzB8H34mzh/rlozzBwxD1AYowJce5W2FGcXRlbPI5TZrhbwDuB21Q12es8ocS2IIwJYSJytbv7LxbnNNdVOGdKmWIQkd+ISGV3V87ZYwb+dvOVa1YgjAlt1+LsEtqNc2rzLWqb/YHQDdjEz7sJr1PVU/mPUv7YLiZjjDF+2RaEMcYYv4J2ASwRqY9zxk0izulsY1T15VxtquCc6dIU5wyXe1R1tTtsK85ZAdlAlqp2Kmie1atX10aNGgXwWZy7EydOEBsb63WMfIV6xlDPB6GfMdTzgWUMhOLkW7JkyUH3R5u/FqyfaAO1ca9vg3OecArOJZN92zyHc6odQCtgus+wrUD1oswzKSlJQ0VycrLXEQoU6hlDPZ9q6GcM9XyqljEQipMPWKwlfakNVd2jqkvd+2k4F7DK/bP+1sAMt816nB9dJWKMMcZzJXKQWkQa4fxYqo2qHvd5/D84V7x8REQ64/zSs4uqLhGRLTg/IlFgtKqOyWPag3F++k5iYmLSxIkTg/tkCik9PZ24uKJeVqhkhXrGUM8HoZ8x1POBZQyE4uTr1avXEs1rF35
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(sig_results[:, 2], sig_results[:, 5], 'x-', label=\"Final Validation Loss\")\n",
    "\n",
    "# plt.ylim(0)\n",
    "\n",
    "plt.title('Final Validation Loss for Sigmoid Learning Rates')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Gamma')\n",
    "\n",
    "# plt.legend()\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be993da6",
   "metadata": {},
   "source": [
    "# Best\n",
    "\n",
    "100 Epochs\n",
    "\n",
    "top-1 accuracy indexes: 1, 3, 2, 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6ab2b999",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhAAAAEWCAYAAAA3sa4JAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABCPUlEQVR4nO3debxc8/3H8dc7sSQkEiJVpIQmqDXWUrSxFK0q+qO2tpai2tq6opTQatFFVRdFlah9CWkpUcTS1hZCrBWkFdQSsiFk+fz++H4nORkzc2eSO/fOTd7Px2Me98z3fM85nzN37p3PfM/3fL+KCMzMzMwa0a2zAzAzM7OuxwmEmZmZNcwJhJmZmTXMCYSZmZk1zAmEmZmZNcwJhJmZmTXMCYSZtRxJa0saK2mapKM7O55yks6T9MPOjqOjSfqBpAs7O44iSSFpUDvta7SkQ9u7bnuQNDCf6xL5+d8kHVjHdk9IGtqMmJxAWJcnaYKkdyVNl/SWpJskfaSd9rtjlXUH5ONNz8eeU3g+fQGONd8/hzbqHpTr7tPocbqQ7wN3RkTviPj1wuxI0r75d6my8iUkvSbpc43uMyKOiIgfLUxcjWjk/dFMEfGTiGjKh6ak3XPSOFXSG5LukLRGM47VmQq/y9L/iwmSjm90PxHxmYi4pI5660XE6AUKtg1OIGxRsVtE9AJWBl4Fzm3mwSLisojolY/5GeDl0vNc1kwHAm8CX2nycebTwR9eqwNPLMiGFeK8AegLfKqsfBcggFsa3H/3BYmr1XVmcpJbEIYD3wH6AGsAvwVmd1ZMHaBv/l+xH3CypF06O6BGOYGwRUpEzACuBdYtlUlaWtLPJf1X0qu5+blnXreipL9KmizpTUn3SOom6VJgNeAv+VvC9+uNQdIqkq6T9LqkF4pN8JK2kPRQ/pb1qqRf5lV355+T8/G2qrLv1UkfhIcDO0v6cGFd99zE/Fxu+h9TaomRtJ6k2/I5virpB7n8Ykk/LuxjqKSJhecTJB0n6THg7fyt/fjCMZ6UtGdZjIdJeqqwfhNJ35N0XVm9X0s6p8I53gFsB/wmvxZrSeojaXh+Tf8j6SRJ3XL9gyT9Q9LZkiYBw4r7y++Jq/lgwvUV4PKImCXpGkn/kzRF0t2S1ivEc7Gk30u6WdLbwHYVXrfDJI3Pr+9ISavk8g+0HKjQ9C1pkKS78nHfkHRV+evRlvza/FHSK5JekvTjUpIj6aP5m/ykvP/LJPUtbFv++x2U4z0w/728IenEQv1hkv5cdm7V6vaUdIlSq+BTkr5ffG+VGQK8EBG3RzItIq6LiP/mfVV9b2c7SnpW6e/4t9K81iZJh+TjvyXpVqW/odK6T0t6Or/+vwGK280917LzrZho1TpOLRHxL1KyvL7S/56T8nv8tfye71PlePNdQlGFv7tcPrclNe+/9Pc7SdLVklbI63pI+nMunyzpQUkrtRW8H3506QcwAdgxLy8DXAIML6w/GxgJrAD0Bv4C/DSv+ylwHrBkfmwLqHy/bRx/KDAxL3cDxgAnA0sBawLPAzvn9f8CvpyXewFb5uWBpG/DS7RxrB8CD+TlccB3Cuu+l8vWJv0j3Ajol8/5FdK3ux75+cfzNhcDP650LoXXYCzwEaBnLtsbWCWf6z7A28DKhXUvAZvnGAaRWhNWzvX65npLAK8Bm1Y5z9HAoYXnw4Ebc+wDgX8DX83rDgJmAUfl/fassL+tgamFc+gDvAsMyc8PyfteGvgVMLaw7cXAlLyPbvk1nPu6AdsDbwCb5O3PBe6u9nstnhtwBXBiYb/bVHk9qr4/gBHAH4BlgQ8BDwBfy+sGAZ/OcfUnJaq/qvb7LRzngvx8I+A94GO5/jDgz2UxVat7BnAXsDwwAHiMwnur7BzWBGaQ/la3A3qVra/43s7rAvgrqZVpNeB1YJe8bndgPPCx/N44CfhnXrciMA3Yi/S3/y3S++jQ8nOt9Dso+z1WPU6t32U+l62Bd4AdSO/D8fn16AVcD1xax/Er/t1V+P94DHBf/n0sTXrfXJHXfY30v3EZoDuwKbBczf9Hzfqn7ocfHfXIfyDTgcnATOBlYIO8TqQPro8W6m9F+rYDcBrpg2lQlf02mkB8HPhv2foTgD/l5buBU4EVy+rM98+hxrGeBY4t7PfRwrpngN0rbLMf8EiV/V1M2wnEIW3ENLZ0XOBW4Jgq9f4GHJaXPwc8WWOfxX+O3YH3gXUL678GjM7LB5W/5jVeu/3z8mHF166sXt/8u+hTeI2Gl9WZ+7oBfwTOKqzrld+HAyv9XsvObThwPjCgjdgrvj+AlUgf2j0LZfuR+o9U2s8exfdC+e+3cJwBhbIHgH3z8jA+mEBUqzs3cc7PD6VKApHXb0lqKXqdlExcTE4kqPLezuuCQuKV93F84T331cK6bqQP69VJLVD3FdYJmMiCJRBVj1PjdzkZeAt4Cjg6r7sd+Eah7tr5vbREG8ev9Xc3gXkJxFPADoV1Kxf2fwjwT2DDtv6WSg9fwrBFxR4R0Zf0Le5I4C6l5v3+pIx6TG6Wm0y65t0/b/czUsY/StLzWoDOTGVWB1YpHSsf7wekf/QAXwXWAp7OTYR1d+CTtDXp2vCVuehyYANJQ/LzjwDPVdi0Wnm9XiyL4ytKnd1K57c+6dtcW8e6BPhSXv4ScGmdx1+R9A3xP4Wy/wCrVouxiuHMu4zx5fy81Dx+Rm7WnUr6h1s6bj37X6UYW0RMByaVxVfN90kfXA8o9ZY/pI5tilYnvTavFH4ffyC1RCBpJUlX5ksbU4E/M/95QeVz+19h+R1SUlRNtbqrlO275u8oIu6LiC9GRH9SS+AnSa0z0PZ7uFoMqwPnFF6bN0mv96rl8UX6RK3nfVRJreNUs2JELB8RH4t5HYXney/l5SWY9/+jmnr/xlcHRhTifIrUz2Ql0t/jrcCVkl6WdJakJWvtzAmELVIiYnZEXE/6o9iG1LT8LrBeRPTNjz6ROzpGutb6nYhYE/g88G1JO5R2twAhvEhq3ehbePSOiM/m4z0bEfuR/sGfCVwradk6j3Ug6Z/SWEn/A+4vlJeO/dEqMa1ZZZ9vkxKskg9XqDM3tnxd9wJSktYvJ22PM+/acbUYIHVm3FDS+qQWiMuq1Cv3Bulb0uqFstVITbYfiLGGS4EdlPqXbFk4/v6kJugdSZc2Buby4l0btfb/cjG2/Pvsl+N7OxdXfI0j4n8RcVhErEJqVfmdGrsl8UVSC8SKhffbchFR6sPxkxz7BhGxHClxU9k+FuR9Xo9XSE3lJXXfGRURD5Ka79fPRbXeV7W8SLqcU/x77BkR/8zxzY0p95soxljP30Y9x2nEfO8l0vt8FqljeC31vj4vAp8pi7NHRLwUETMj4tSIWBf4BOlvtGZHbScQtkhRsjvpuutTETGH9IF3tqTSt7JVJe2clz+n1HFMpOvcs4E5eXevUv2Dt5oHgGlKHdN65m+360vaPB/vS5L657gm523mkJpt51Q7nqQewBdJnSeHFB5HAfvnjl0XAj+SNDi/DhtK6ke6PryypGOVOpT2lvTxvOuxwGclrZBbbI5t4/xKyc7rOa6DmfdPnhzDdyVtmmMYVOpMFvM6uF5O6sfx3zaORd5uNqlZ+vQc++rAt0nfpusWEROAe0n9Dm6LiNK31t6kD+FJpA+MnzSy37y/gyUNkbR03v7+iJgQEa+TEokv5ffCIRT+0UvaW1LpQ/Yt0ms7h+qWzp3deuT3xKvAKOAXkpbLneQ+KulThXObDkyRtCqpL0FHuRo4QdLy+dhHVqsoaRulToClv9F1SAn9fblKtfd2W87LMayX99tH0t553U3AepK+kP9+jmb+JGEs8ElJqyl1ZDxhAY/TiCuAb0laQ1Iv0nvpqoiY1cZ2Vf/uKsR5emmdpP75/yWStpO0gVIH3KmkpL3We9EJhC0y/qI0/sJU4HTgwIgo3QZ4HOkyxX25GffvpGuLAIPz8+mkDo6/i4g787qfAifl5r7v1hNE/rD7HLlXOenb84Wkb7aQbh18Isd6Dul68bsR8U6O+x/5eFuW7XoPUkvK8Pyt9X/5A/AiUhPnLsAvSf+0R+X
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "best_top_1_results = list()\n",
    "best_labels = list()\n",
    "\n",
    "# Fixed\n",
    "b_fixed = fixed_results_100e[np.argmax(fixed_results_100e[:, 1])]\n",
    "best_top_1_results.append(b_fixed[1:3])\n",
    "best_labels.append(f'Fixed\\n{b_fixed[0]}')\n",
    "\n",
    "# Step Down\n",
    "b_sd = step_down_results[np.argmax(step_down_results[:, 3])]\n",
    "best_top_1_results.append(b_sd[3:5])\n",
    "best_labels.append(f'Step Down\\n{b_sd[0]}, Step: {b_sd[1]}, Gamma: {b_sd[2]}')\n",
    "\n",
    "# Exp\n",
    "b_exp = exp_results[np.argmax(exp_results[:, 2])]\n",
    "best_top_1_results.append(b_exp[2:4])\n",
    "best_labels.append(f'Exponential Decay\\n{b_exp[0]}, Rate: {b_exp[1]}')\n",
    "\n",
    "# Sig\n",
    "b_sig = sig_results[np.argmax(sig_results[:, 3])]\n",
    "best_top_1_results.append(b_sig[3:5])\n",
    "best_labels.append(f'Sigmoid Decay\\n{b_sig[0]}, Step: {b_sig[1]}, Gamma: {b_sig[2]}')\n",
    "\n",
    "# print(best_top_1_results)\n",
    "# print(best_labels)\n",
    "# print(best_top_1_results)\n",
    "plt.barh(range(len(best_labels)), [i[0] for i in best_top_1_results], tick_label=best_labels, label='Top-1')\n",
    "plt.barh(range(len(best_labels)), [i[1] - i[0] for i in best_top_1_results], tick_label=best_labels, label='Top-5', left=[i[0] for i in best_top_1_results])\n",
    "\n",
    "plt.legend()\n",
    "plt.grid(axis='x')\n",
    "plt.title('Best Test Accuracy for Various Learning Schedule Policies')\n",
    "plt.xlabel('% Test Accuracy')\n",
    "plt.ylabel('Learning Schedule Policies')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f36766a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}