DIGITS-CNN/cars/architecture-investigations/architecture.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "682fef9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75cc3c1d",
   "metadata": {},
   "source": [
    "# Dense Layers\n",
    "\n",
    "Exponential LR Decay: 0.98\n",
    "\n",
    "100 Epochs\n",
    "\n",
    "## Index\n",
    "0. fc layers\n",
    "1. nodes per layer\n",
    "2. top-1 accuracy\n",
    "3. top-5 accuracy\n",
    "4. last val loss\n",
    "5. last val accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "85cb4e35",
   "metadata": {},
   "outputs": [],
   "source": [
    "fc_results = np.array([\n",
    "    [1, 256, 52.44, 79.86, 2.49, 57.66],\n",
    "    [1, 512, 49.29, 73.93, 2.95, 53.25],\n",
    "    [1, 1024, 40.7, 68.38, 3.66, 45.22],\n",
    "    [1, 2048, 32.12, 58.93, 4.66, 35.72],\n",
    "    [1, 4096, 24.03, 46.76, 5.61, 27.94],\n",
    "    [1, 8192, 19.70, 41.01, 6.42, 23.96],\n",
    "    \n",
    "    [2, 256, 54.48, 81.22, 1.86, 57.11],\n",
    "    [2, 512, 56.64, 82.46, 1.94, 60.23],\n",
    "    [2, 1024, 56.39, 81.53, 2.08, 60.91],\n",
    "    [2, 2048, 51.39, 79.00, 2.38, 56.74],\n",
    "    [2, 4096, 44.41, 71.83, 3.04, 47.61], # DEFAULT ALEXNET\n",
    "    [2, 8192, 37.74, 64.36, 3.60, 42.40],\n",
    "    \n",
    "    [3, 256, 0.8, 2.1, 5.29, 0.55],\n",
    "    [3, 512, 30.7, 65.16, 2.57, 30.82],\n",
    "    [3, 1024, 48.36, 76.65, 2.30, 49.88],\n",
    "    [3, 2048, 54.11, 80.48, 2.38, 58.21],\n",
    "    [3, 4096, 54.48, 82.09, 2.39, 57.17],\n",
    "    [3, 8192, 50.71, 78.57, 2.55, 55.88],\n",
    "    \n",
    "    [4, 256, 0.8, 2.29, 5.29, 0.55],\n",
    "    [4, 512, 0.8, 2.1, 5.29, 0.55],\n",
    "    [4, 1024, 0.8, 2.1, 5.29, 0.55],\n",
    "    [4, 2048, 25.45, 60.9, 2.84, 28.55],\n",
    "    [4, 4096, 41.14, 73.81, 2.81, 46.32],\n",
    "    [4, 8192, 49.85, 77.58, 2.97, 53.92]\n",
    "])\n",
    "\n",
    "fc_0_results = [0, 196, 28.91, 54.05, 6.52, 33.21]\n",
    "\n",
    "layers = [1, 2, 3, 4]\n",
    "nodes = [256, 512, 1024, 2048, 4096, 8192]\n",
    "\n",
    "fc_matrix = np.zeros((len(layers), len(nodes)))\n",
    "for i in fc_results:\n",
    "    fc_matrix[layers.index(i[0]), nodes.index(i[1])] = i[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0fd9f416",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWUAAAEwCAYAAABvxmD/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAACwGElEQVR4nOydd3gc1dm377NVvViyZVm25CLJ6s2AgUAI1RQDocVAICT0BEJ5SYBAKCEJgZCQno8QakJCSQImhV5s04u6JcuWZMu2ZDWr121zvj92Z1lJW2bVbe99XXPZuzvlzGr2N8885ylCSkmIECFChJgf6OZ6ACFChAgR4gtCohwiRIgQ84iQKIcIESLEPCIkyiFChAgxjwiJcogQIULMI0KiHCJEiBDziJAoH8AIIb4uhHjDz+dfEUI0T/Mx7xBCPObx+hwhxF4hxKAQolgIsVoIUSGEGBBC3DCdxw4xFiFEkhBii+u7/uX4v81cI4R4RAhx11yP44BDSnnALcAmoAcwz/VY5tMCSCDd4/VXgOYgv9dRYADoB0qB2/19z0AjcLbH68eBX83R+d8LPBNgnSbACiSOe7/c9f0td71+CviJn+95CBgEWoCHAb2PdQVwA7DVtU0z8A8gfxrO9y7gRUDMg2vvm8D7cz2Og2E54CxlIcRy4FicP4yzZvnYhtk83hxxvZQyGkgGbgEuBF4RQggf66cBNX5ea2YWv99dwEUex80HIoLcR6GUMgo4EbgYuMrHer8BbsQpzAuATGAjcEaQx/NGGlArXao4Uxwi1/38Ya7vCsEuwN3ABzitk/+O+2wZTsuhE+gCfu/x2VXANpxWYC1Q4np/vHX5FC4LCZelCdwGtAF/BeKB/7qO0eP6/1KP7RcATwL7XJ9vdL2/FTjTYz0jsB8o9nKOm4HzXP//kmuMZ7henwhUuP7/TVzWCbCFsRbcBo/x3wJ0AK3At/x8t5uAK8e9lwoMA+tdr+8FngHMruOox2wE3gEcOK3tQZwCZAZ+AewB2oFHgHA/368Op3Xe6PobvgAscK2/3HW8y1z72w/c6frsVJwWsM117Eof59gE/BD4zOO9XwB3Epyl7HnN/AOPa83j/QzX93GEn+88FviL63ra7RqbzvPv6xpfD86byWke47O5znkQOIlxTwrAN1z77MJpVTcBJ3k7P8Y9VbnWvQ2oAiyAwePvov6GznGtm+36mztcY+n1cYyrgAagG/g3sGTcd3otUA/0An/A9QQApOP8TfS5/ubPz7UOzeRywFnKOC+0v7mWdUKIJAAhhB6nQO7G+eNNAZ5zfXYBzgv2G0AMTgu7S+PxFuMU2jTgapyi8aTrdSowAvzeY/2/4rS6coFFwK9c7/8FuMRjvdOBVilluZdjbsb5IwE4DtgJfNnj9ebxG0gp1c8LpZRRUsrnPcYfi/P7uAL4gxAiPtBJe+x3D/A5zqcTz/ct0mkpqsdcJaU8AXgPp7UdJaXcATyAU5yLcP64UnDeWFXGf7/fBb7qOs8lOMXoD+OGdQywGucN6m4hRLaU8jXgfpw/2CgpZaGf0/oYiBFCZLuumwtx3miCRgiRg/O78fZ3PBGn0H3qZxe/w/n3WYnznL8BfMvj87XAdiAR+DnwuBBCSCm/ifM38HPX+b7lZVx/BL6O86lHvQaC4SKcFn2clNKOU5CPde3rR8AzQohkKeU2nIL6kWssceN3JIQ4AfgZ8DXXeHbj+n16sB44HChwrbfO9f6PgTdwGkRLcX5nBy0HlCgLIY7B+eN9QUpZivMiudj18RE4f8Tfl1IOSSlHpZTvuz67EufF+5l00iCl3K3xsApwj0uERqSUXVLKf0kph6WUA8BPcf6YEEIkA6cB10ope6SUNimlKqDPAKcLIWJcry/FKeDe2KzuE6cY/8zjtVdR9oMNuM81lldwWjKrg9genFb/giC3weXyuBq4WUrZ7fq+7scpgipjvl+cP+47pZTNUkoLzpvp+eMeoX/k+ltUApWAPwH2xV9xCuDJOJ+gWoLcvkwI0QP8B3gM5416PAk4n0684nFD+IGUckBK2QT8Eue1obJbSvlnKaUDeBqnoCVpGN/5wH+klO9LKa04b4TBujl+K6Xc6/q7IKX8h5Ryn5RScd3063H+7rTwdeAJKWWZ6+/6A+AolztS5QEpZa/LEHgX540cnNdwGk7L2vN3fVByQIkyzsfWN6SU+12v/+56D5yui92uO/p4luEU8MnQKaUcVV8IISKEEH8SQuwWQvTjdBvEuX5gy4BuKWXP+J1IKffhdLucJ4SIwynef/NxzI+ATNdTQBFOK3uZECIR549gSxDj7xr3nQwDUb5W9kEKzkfOYFmI86mhVAjRK4ToBV5zva8y5vvF+eN7yWP9bTgfiz2FqM3j/5M5H3CK8sU4XQR/mcT2JVLKeNcTwg+llIqXdbpwiqgvEnG6sTwNhN2MtWjd5yqlHHb9V8v5LgH2jttW69Ohyl7PF0KIb7gia9S/TR7Oc9DCEjzOU0o56BqP13Nl7N/1VpwTpp8KIWqEEJcHdRYHGAeMKAshwnE+0hwnhGgTQrQBNwOFQohCnBdQqo9Jib3AKh+7HmbsJM/icZ+Pty5uwWlprpVSxvCFW0G4jrPAJbreeBqnC+MCnI96Xq0z1w+oFOcE0VaXpfMh8H9Ao8dNacYRQiwD1uB0SwTLfpzunVwpZZxrifVwe8DE73cvTr9pnMcS5uu7GodmS9D1pLQLpxvpRa3bBcnbwFIhxGE+Pt/PF1agSirBW+3eaMX5qA+4fz8JHp8P4f+6B4/vUwiRBvwZuB5IcLkotuK87ses64N9eJynECLSNZ6A5yqlbJNSXiWlXAJcA/xRCJEeaLsDlQNGlHH6GR1ADk7rsQjnBMN7OB9DP8V5IT4ghIgUQoQJIb7k2vYx4HtCiDXCSbrrIgOoAC4WQuiFEKfyhZvAF9E4haZXCLEAuEf9QErZCryK86KJF0IYhRBf9th2I1CCU2wDWWebcf4AVFfFpnGvvdGO0zc5ZVxPBMcBL+P8bl8Jdh8u6/HPwK+EEItc+00RQqzzs9kjwE/Vv48QYqEQ4myNh2wHlgshxlzXra2t5t7eXm836yuAE6SUQz72p3ddR+pi0jgOAKSU9Tj9us+6YsZNrv1cKIS43eWSeAHn+Ua7zvn/mKR/exz/BM4UQhztGve9fCGg4LzuTxdCLBBCLAZuCrC/SJzC2wkghPgWTktZpR3nDcjXd/Qs8C0hRJEQwozTjfWJy2XjFyHEBUII9QbT4xqHtyeTg4IDSZQvA56UUu5x3TnbpJRtOCfZvo7zgjsT52TSHpyz+hvA6QvD6fv9O86Z44184SO90bVdr2s/GwOM49dAOE4r52Ocj+OeXIrT+qnDGfFwk/qByzf3L2AFga2zzThvAFt8vPbGvcDTrsfLrwXYvy9+L4QYwPkj+7VrvKf6eDzXwm04Z9w/drl73sK/T/s3OGfm33CN42Ock11a+Ifr3y4hRBlAa2truhDiTw6H4/mBgYEx1qCUslFK+bmf/d2O8wasLu9oHIcnN+C8Rv+A8xprBM7B6YsG58TmEM7J3PdxXqNPTOI4Y5BS1rj2/RxOY2UQ5/Voca3yV5z++Cack2jPT9zLmP3V4vR3f4Tz2sjH6Y5TeQdnKGSbEGLCk5xrIvIunNdTK84n1wvHr+eDw4FPhBCDOK+NG6WUOzVue8ChhpyEmCWEEHcDmVLKSwKuHGLStLa2mnDebM8SQiwxGo3RJpPpU+DB6Ojo6XAPHFAIIaJw3hQypJS75ng4IfxwIFnKBzwud8cVwKNzPZaDmdbW1mU4rbIzcfqo1dhtAfxgYGBg2RwOb9YQQpzpckNF4ox1rsZpGYeYx4REeZYQQlyFUyBelVIGEz0RQiOtra361tbWdThjaONxCpCdL3ypXTh9kT8YGBhYPhdjnGXOxjnBtg9nIsuFMvRoPO8JuS9CHBS0trYuwvkUkoVzRt+qfiaEKDEajeEmk0n1gcYDYThdGQetbzLEgUnIUg5xQNPa2qpraGg4ef/+/U/iDCfbhYcguxCMDdnqwRkKefvAwEDG7Iw0RAhthEQ
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "X, Y = np.meshgrid(layers, nodes)\n",
    "\n",
    "# fig = plt.figure(figsize=(8, 5))\n",
    "fig = plt.figure()\n",
    "ax = plt.axes(projection='3d')\n",
    "\n",
    "surf = ax.plot_surface(X, Y, fc_matrix.T, cmap='viridis')\n",
    "\n",
    "ax.set_title('Accuracy with Different MLP Configurations')\n",
    "ax.set_xlabel('Fully Connected Layers')\n",
    "ax.set_ylabel('Nodes Per Layer')\n",
    "ax.set_zlabel('Top-1 % Test Accuracy')\n",
    "\n",
    "ax.view_init(60, -130)\n",
    "fig.colorbar(surf, shrink=0.3, aspect=6)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dc9d04b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABtsklEQVR4nO2dd3wUxfvH35N66T0hhd5D6EoQkSIWRBARRcGGqGDHXr/WH3YUCyg2igVQUVCxoZAIKhAJnYReEwIhlbRLuZvfH7sJl34JSe6SzPv12ld2Z3dnP7d3eWb3mZnnEVJKFAqFQtF6cLC1AIVCoVA0LcrwKxQKRStDGX6FQqFoZSjDr1AoFK0MZfgVCoWilaEMv0KhULQylOFX2C1CiLuFEKeEELlCiABb66mIrqtTPc+VQoguDa1JobAGZfibKUKIWCFEphDC1dZaGgMhhDPwNnCZlNJTSpl+jvX9JoR4qYry8UKIk0IIp7rWqes6dC66qkL/bo1CiBwhxBkhRLwQ4kl7+66FEEeEEJfYWoei7ijD3wwRQnQALgIkcFUTX7vOBrKehAAGYHddTxQaFX/bi4GbhBCiQvnNwFdSypI61N8U9+A+KaUXEAo8AtwA/FKF/lZLE/4WWxzK8DdPbgE2AouAWy13CCHaCiG+F0KcFkKkCyHmWuy7UwiRqD9JJgghBujl5dwOQohFQohZ+voIIUSSEOIJIcRJYKEQwk8IsUq/Rqa+HmFxvr8QYqEQ4oS+f6VevksIMc7iOGchRJoQon+Fz9AN2KtvZgkh1urlQ4QQ/wkhsvW/QyzOiRVCvCyE+AfIByq6YFYCAWgNZuk5fsBY4HMhxCAhxAYhRJYQIkUIMVcI4WJxrBRC3CuE2A/sr3jfhBCuQojZQohjuntqvhDCzeL8x/R6TwghplX6RqtBSpknpYxFa+AvAK7U63PQ3wIO6t/zN0IIf31fB13brbqeNCHEMxZaBgkhNutvE6eEEG9b7BsshPhXvw/bhRAjrNVqeV+r+30IIa4TQsRXOP5hIcQPtd3Han6LgXr9WUKIDCHE+ioafUVFpJRqaWYLcAC4BxgIFAMherkjsB2YA3igPTEP1fddByQD5wMC6AK01/dJoItF/YuAWfr6CKAEeB1wBdzQDOhEwB3wAr4FVlqc/zPwNeAHOAPD9fLHga8tjhsP7KzmM3bQdTnp2/5AJtoTuhMwWd8O0PfHAseAXvp+5yrq/AT41GJ7BrBNXx8IDNbP7QAkAg9aHCuBP3QdbhXvm37Pf9T3ewE/Aa/q+0YDp4Ao/XtZUvGeV9AZC9xRRfk64HV9fSZa4x+hfy8fAUsr3LtP9O+rL1AI9NT3bwBu1tc9gcH6ejiQDoxBeyi8VN8OqkbnEeCSKsqr/X3oWjNKtehlW4GJVtzHEVT+Lb4KzEf7nTmjNezC1v+j9r7YXIBa6viFwVA0Yx+ob+8BHtLXLwBOoxvLCuf9Dsysps7aDH8RYKhBUz8gU18PBcyAXxXHhQE5gLe+vRx4vJo6S41XqeG/GYircMwGYKq+Hgu8ZMW9yyr9LMA/pfeuimMfBFZUuEcXV3Xf0BrSPKCzxb4LgMP6+gLgNYt93Sre8wr1xlK14V8GfKKvJwKjLPaF6r+L0oZLAhEW++OAG/T1dcCLpb8hi2OeAL6o4ndzazU6j1CF4a/p96Fvfwi8rK/3QmvAXa24j5V+i8BLwA/V3Uu1VL2oV6Lmx63Aaillmr69hLPunrbAUVm1v7otcLCe1zwtpTSWbggh3IUQHwkhjgohzqAZEl8hhKN+nQwpZWbFSqSUJ9CM7UQhhC9wBfCVlRrCgKMVyo6iPaWWcrymCqSUfwNpwNVCiM7AILT7hxCim+4yOKl/pleAwApVVFd/ENrTbbzucsgCftPLS7Vbnlvxc1hLONrTMkB7YIXF9RIBE1rfSCknLdbz0Z7uAW5Ha3z26C6zsRZ1Xldap17vULRGxWpq+X2A1t8yRQgh0Br0b6SUhdR+H6HCbxF4E+0NeLUQ4pAQ4sm6aG2tqM6RZoTu65wEOOo+TtCelHyFEH3RjEs7IYRTFcb/ONC5mqrz0f7hSmkDJFlsVwzh+gjQHYiWUp4UQvRDe10X+nX8hRC+UsqsKq61GLgD7be3QUqZXN3nrcAJNMNkSTs0w1Cdzqr4HK2PpDvwu5TylF7+of4ZJkspc4QQDwLXVji3uvrTgAKgVzWfJwWtQbTUXSeEEG3R3FGv60XHgWlSyn+qOLZDTXVJKfcDk3Vf+DXAcqENlz2O9sR/Z131VaCm3wdSyo1CiCI0t8wUfYHa7yNU+A6klDn69R4RQkQBa4UQ/0kp15zjZ2jRqCf+5sXVaE91kWivz/2AnsB6NGMWh2ZkXhNCeAghDEKIC/VzPwUeFUIMFBpdhBClhnQb2hOYoxBiNDC8Fh1eaP+gWXqH4vOlO6SUKcCvwAd6J5+zEGKYxbkrgQFoPurP6/DZfwG6CSGmCCGchBDX6/dhVR3qQL/mJcCdaI2Q5Wc6A+QKIXoAd1tboZTSjOZPnyOECAYQQoQLIS7XD/kGmCqEiBRCuGNxv2pDf3oejubOiEO7D6D5tV8u/Q6FEEFCiPFW1nmTECJI152lF5uBL4FxQojL9d+CQe9Qjai2MnDWjytdnKjh92HB58BcoFh/E7PmPlb1Wcbqv2UBZKP9f5ituQ+tGlv7mtRi/YL2dPtWFeWT0F7rndCeJleidcqlAe9ZHHcX2miZXGAX0F8vPw9t2GQO8AWwlPI+/qQK1wtD80PnAvvQOkkrdsQuRuvQzAS+r3D+p2i+XM8aPmsHyzr1sqFAPNo/eDx6x7W+L5Yq/OLV1B2r63K1KBuG1l+Si9aQvgT8bbG/kk+e8p27BjT30CG0BiQReMDi2Cf17+gEMK2q+iroM+rfRw7a0/IzlPdtOwAP699nDpob75Ua7l3Z/UEz8Kn6Z90NXG1xXDTwF5pL6TRaR327anQe0a9jucyq7fehn9sOzUC/WKHOau8jVf8WH9J15KG9pT5r6//T5rAI/eYpFE2GEOI5oJuU8iZba1HYBt1tmQoMkJrrSdGEKB+/oknRX/1vR+vUU7Re7gb+U0bfNijDr2gyhBB3Au+gdSCus7EchY0QQhxB6+i92rZKWi/K1aNQKBStDDWqR6FQKFoZzcLVExgYKDt06FCuLC8vDw8PD9sIqgV71aZ01R171aZ01R171daYuuLj49OklEGVdth6WJE1y8CBA2VFYmJiKpXZC/aqTemqO/aqTemqO/aqrTF1AZtlFTZVuXoUCoWilaEMv0KhULQylOFXKBSKVoYy/AqFQtHKUIZfoVAoWhmt2/D//Q4crjCB9PA6rVyhUChaKK3b8IcPgG+nnjX+h9dp2+ED6l6XakQUCkUzoVlM4Go0Og6DaxfC0skQ1g+St8J5t0FuKuz9FVw8wNlD++viDi6e4OwOTq4gRPm6ShuR6xZp26WNSOm2QqFQ2Amt2/AX5sKWz6EoF478rZVtmFv7ecJRbww8tIagdN23PXw5kYFuEbAhDa7/UmtcFAqFwo5ovYY/7QB8fROc3qMZ7+i7IH4RXPkWhERBcR4UVViK87VGoihf366wn0Jw9cIr95B2jT+fhwvug8irwbH13mqFQmFftE5rlPgTrLhbc9e4esENX2lP5p1HnnXP1OdJXXfvHG07kfanfoPcU/Dd7fDnizD4Lhhwi3Y9hUKhsCGtq3PXVAJ/PKc96Qd1g/PvOGv0Qft73SJI3lL3ui18+oc73wKTl0FJIYz8H/i2hd+fhrcjYfWzkG1tfnGFQqFoeFrPE39uKiyfBkfWw3nTYPRrWidtRToOq9/TfvKWs28KR2PLNyK3/QLJ8fDvXNgwDzZ+AL2ugSH3QWjfc/xgCoVCUTdapuH/+x1tlE2pAT8eB0uuh8IzcPV86De54a859MHKZZaNSPhAuG4hZB6FTfO1TuWd32j7hzwAXS6pPFJIYXP+zP4T9xR3BoUOKiuLS4ljV/oupkVNs6EyhaL+tExXT+nQykN/waaPYcFoMGbBmLcax+jXBb/2MPpVeGg3XPqS1sn81bXwwWCtMSg
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plt.plot([196], fc_0_results[2], 'x-', label=f'0 Layers')\n",
    "\n",
    "for i in layers:\n",
    "    plt.plot(nodes, fc_matrix[i-1, :], 'x-', label=f'{i} Layers')\n",
    "\n",
    "plt.annotate('Standard\\nAlexNet', \n",
    "             (4096, fc_matrix[layers.index(2), nodes.index(4096)]),\n",
    "             textcoords=\"offset points\",\n",
    "             xytext=(40, 10),\n",
    "             ha='center',\n",
    "             arrowprops={'arrowstyle': 'simple'}\n",
    "            )\n",
    "    \n",
    "plt.title('Accuracy for Varied Dense Layers')\n",
    "plt.xlabel('Nodes Per Layer')\n",
    "plt.ylabel('Top-1 % Test Accuracy')\n",
    "\n",
    "plt.grid()\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "340c4eb8",
   "metadata": {},
   "source": [
    "# Convolutional Non-Linearity\n",
    "\n",
    "Exponential LR Decay: 0.98\n",
    "\n",
    "100 Epochs\n",
    "\n",
    "Taking conovlutional layers and distributing the standard number of filters into separate conv layers with ReLu nonlinearity\n",
    "\n",
    "## Index\n",
    "0. convolutional layer\n",
    "1. number of divisions\n",
    "2. top-1 accuracy\n",
    "3. top-5 accuracy\n",
    "4. last val loss\n",
    "5. last val accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3426107c",
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_nonlin_results = np.array([\n",
    "#     [1, 1, 44.41, 71.83, 3.04, 47.61], # STANDARD ALEXNET\n",
    "#     [1, 2],\n",
    "#     [1, 4],\n",
    "    \n",
    "    [4, 1, 44.41, 71.83, 3.04, 47.61], # STANDARD ALEXNET\n",
    "    [4, 2, 42.31, 71.96, 3.07, 49.75],\n",
    "    [4, 4, 0.8, 2.47, 5.29, 0.55],\n",
    "    \n",
    "    [5, 1, 44.41, 71.83, 3.04, 47.61], # STANDARD ALEXNET\n",
    "    [5, 2, 46.08, 73.87, 3.00, 49.20],\n",
    "    [5, 4, 0.8, 2.47, 5.29, 0.55]\n",
    "])\n",
    "\n",
    "nonlin_layers = [4, 5]\n",
    "nonlin_div = [1, 2, 4]\n",
    "\n",
    "conv_nonlin_matrix = np.zeros((len(nonlin_layers), len(nonlin_div)))\n",
    "for i in conv_nonlin_results:\n",
    "    conv_nonlin_matrix[nonlin_layers.index(i[0]), nonlin_div.index(i[1])] = i[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d7457e0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABG30lEQVR4nO3dd3gU1frA8e+bDqRBEkKXLiIdpARFQGxIkya9iQjY2xV/V71Y7hX12hVQQJr0ogKKFZArAREQEEGp0ntNgBSS8/tjJrCElE3Z3WTzfp5nn+xOfc9O9p2ZMzPniDEGpZRSRYePpwNQSinlXpr4lVKqiNHEr5RSRYwmfqWUKmI08SulVBGjiV8ppYoYTfxFjIiMEJGjIhIvIhGejic9O66quZzXiEj1/I7JVUSksh2zXy7n7ysi3+V3XBmsZ4qIvOrq9eSViIwXkRc8tO6lIjLQE+vODU38DkRkhYicFpFAT8fiCiLiD7wN3GGMCTbGnMzj8r4RkZczGN5ZRI7kJqHZce3OS1yZEZE7RWSliMSJyHER+UlEOrliXfkto52EMWaGMeYOD8c1yI7rH+mGHxCR1i5YX2sROZDROGPMcGPMK/m9TmcYY+42xkyFy9/Jz56Iw1ma+G0iUhm4BTCAW5NBbo/4ciEaCAL+yOmMYkn//zIV6Ccikm54f2CGMeZSDpbv0u9ARLoD84BpQAWs7+JFoKMr11tEnAL+ISIhng7E3TL5XRR8xhh9WU8vvwiswjoiXpJuXEVgIXAcOAl86DDuAWAbEAdsBRrZww1Q3WG6KcCr9vvWwAHgWeAIMB0oCSyx13Hafl/BYf5SwGTgkD3+C3v4FqCjw3T+wAmgYboy1ATO23HFA8vs4THAr8BZ+2+MwzwrgH/b38tFx/LY44vZ87VyGFYSSADqA02B1cAZ4DDwIRDgMK0BHgJ2AHvSf29AIPBfYB9wFBgPFHOY/xl7uYeAIem/c4fpxF7GM1lsfx/geWAvcAxrBxFmj6tsL3ugvZwTwD/tceXs76aUw7Ia2tP4O7lcP/vz30A7h+WMBj6z3+9z2HbxQAtgEPCzw/TZbctX7G0ZB3wHRDqMn4f1v3gWWAncmNH/bgbf2yDgZ2Ax8C+H4QeA1g7b8V17Ox2y3wem+y08ZX8/h4HBWWyn1sCBTMZdjjO75ZLF/xbZ/xZXkO53YQ8bCtyA9f+fYm+nM8BN9jp8HZbRFdjkqXxX+PZUrjMAmGG/7hSRaAAR8cXa8Huxfqjlgdn2uB5YP84BQCjWmYKz1SdlsJL5dcAwrAQx2f5cCesf6kOH6acDxYEbgdLAO/bwaUA/h+naA4eNMb85rswYs92eFyDcGNNWREoBXwHvAxFYO72v0tX997fjC7G/A8dlXgTm2uVP0xP40xizCeuf/wkgEitR3QaMTPc9dAGaAbW51hisHVYDrB9XeawdNCJyF/A0cDtQA2iXwfxprsfaec/PYppB9qsNUBUI5urvH+Bme1m3AS+KyA3GmENYO7duDtP1AeYbY5KdXK4zWtl/w41VHbbacaST27IPMBjr/ycA6/tLsxTreywNbMD6HeTEC8Djdhzp/RNojrUd0w4InncYXwYIw9q+9wMfiUjJHK4/I1ktN9P/LbL/LUImvwtjzDZgOLDa3k7hxphfsfLCHenmn5YPZcwdT+1xCtIL6wedjH0EBPwJPGG/b4G15/fLYL5vgccyWWZ2R/xJQFAWMTUATtvvywKpQMkMpiuHdQQXan+eD/wjk2VW5uojzP7A2nTTrAYG2e9XAC878d2dSSsL1lHQE5lM+zjwebrvqG1G3xvWUfp5oJrDuBZcOTP4FBjjMK5m+u/cYVxLe1xW3/ePwEiHz9fb/xN+Dt+b41HfWqCX/X4oV86gBNiPfRbk5HKdOeK/alp72CDsI34nt+XzDuNGAt9k8l2E2+sKS/+/m8G0jjHMBV633zse8e8C2jvMcyfwt8Nv4WK6ch0DmmeyvtY4f8Sf4XKz+9/K6reY2e/CHjY0/XfiMP5ZrOpPsA74LgBls/ptufKlR/yWgcB3xpgT9ueZ9jCwjhT3mozrqyti/VPnxnFjTELaBxEpLiIfi8heETmHdbodbp9xVAROGWNOp1+IsY44VwHdRCQcuBvnj9bKke4o3v5c3uHz/qwWYIz5Gatao4uIVMM6mptpl6mmiCyxL/SeA/6DdfTvKLPlR2Gd4awXkTMicgb4xh6eFrvjvOnL4SjtLKxsFtOk/y72YiXnaIdhRxzeX8A6egdYALQQkbJYR+apwP9ysNz84My2zDB+EfEVkTEissveTn/b06TfVtl5ERiRdracRWx77WFpTqb7fV0AgkWkkn2XV7yIxOcwlkyXSzb/W9n8FtNk+bvIwGdARxEpgXVW/D9jzOFclClfFPnELyLFsDbErXaCOoJVPVFfROpjbeBKmVx83A9Uy2TRF7D+udKUSTfepPv8FNbRYDNjTChXTu3TjiBL2Yk9I1Oxqnt6YJ1iHsxkuvQOYZ3OOqoEOM6fPs6MTMOq7ukHfGuMOWoPH4d19lTDLtP/YZXHUWbLP4F1xHajsU6Xw40xYcaYtGR7GGuH6Bh3Zv7C+g67ZTFN+u+iEnAJq242S/YO+TvgPqzqlNnGPrTL4XLPk/n/THbbwZltmZk+QGes6rIwrLMLuHZbZckY8yfWtbB/ZhNbJXtYdsvbZ6zqkmCH7Z4fsvvfyuq3eDm8rEK/ZoD1m1yNVbffH6vq1mOKfOLHqmNOwapjbmC/bsA6YhuAdUp/GBgjIiVEJEhEWtrzTgSeFpHG9tX96iKS9g++EehjH03dBdyaTRwhWP+MZ+x60n+ljbCPDJYCY0WkpIj4i0grh3m/ABoBj5GzesOvgZoi0kdE/ETkPvt7WJKDZWCvsx3Whe6p6cp0DogXkVrACGcXaIxJBSYA74hIaQARKS8id9qTzAUGiUhtESmOw/eVwbIM8CTwgogMFpFQEfERkZtF5BN7slnAEyJSRUSCsc5O5mRyppeRmVj/L93t92lystyNQC97+zaxl5XmONaZRGbPOORlW4YAiVhnRsXtGHPrJazrCOEOw2YBz4tIlIhEYp0ZfJaHdWD/Dh1fTu+knPjfyvS36KSjQAURCUg3fBrwD6Au1g7SYzTxW1U6k+2jiyNpL6yLOX2x9vIdseqd92HVXd4HYIyZh3V1fyZWPfsXWPV3YCXhjlj1333tcVl5F+sumRPAGqxTT0f9seqG/8Sqq3w8bYSxLrIuAKqQg38oY93H3wHrCOck1j9lB4cqL2eX8zcQC5QAFjmMehrraDIO64c2JyfLxaoX3QmssU+5f8A6EsMYsxTrO1tmT7MsmxjnY223IVhHm0eBV4Ev7Uk+xToKWwnswboz45EcxLoI6+LoEWNd2E6Tk+W+gHUGeRorgV7egRhjLmDfSWJXTzRPV768bMtpWNUvB7HuTFvjxDwZMsbswSpvCYfBrwLrgM3A71gXj/PyQFh5rMTs+MrszDszmf5vkf1vMTvLsG6ZPiIijt//51hnPp/b29Nj5MoZqSrMRORFoKYxpl+2EyulPEJEdgEPGmN+8GQc7npwSLmQfTp6P9ZZgVKqABKRblj1/1menbqDVvUUciLyANaFy6XGmJWejkcpdS0RWYF1s8ND9jUGj9KqHqWUKmL0iF8ppYqYQlHHHxkZaSpXrpyrec+fP0+JEiWyn7AQ8JayeEs5QMtSUHlLWfJajvXr158wxkSlH14oEn/lypVZt25druZdsWIFrVu3zt+APMRbyuIt5QAtS0HlLWXJazlEJMMn2rWqRymlihhN/EopVcRo4ldKqSKmUNTxK6WKruTkZA4cOEBCQkL2E9vCwsLYtm2bC6NyD2fLERQURIUKFfD393dquZr4lVIF2oEDBwgJCaFy5co42xZbXFwcISGFvydIZ8phjOHkyZMcOHCAKlWqOLVcrepR7vHzu7An3YPFe1Zaw5XKQkJCAhEREU4n/aJGRIiIiMjRGZEmfuUe5RvBvEFXkv+eldbn8o08GZUqJDTpZy2n349W9SjXSLkECWfh4im4eBqSLkDDATCzF/WCa8Av+6D
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for idx, i in enumerate(nonlin_layers):\n",
    "    plt.plot(nonlin_div, conv_nonlin_matrix[idx, :], 'x-', label=f'Layer {i}')\n",
    "\n",
    "plt.title('Accuracy for Varied Convolutional Non-Linearity')\n",
    "plt.xlabel('Layer Divisor')\n",
    "plt.ylabel('Top-1 % Test Accuracy')\n",
    "\n",
    "plt.grid()\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}