#LyX 2.3 created this file. For more info see http://www.lyx.org/
\lyxformat 544
\begin_document
\begin_header
\save_transient_properties true
\origin unavailable
\textclass article
\begin_preamble
\def\changemargin#1#2{\list{}{\rightmargin#2\leftmargin#1}\item[]}
\let\endchangemargin=\endlist
\pagenumbering{gobble}
\usepackage{pxfonts}
\usepackage{color}
\definecolor{commentgreen}{RGB}{0,94,11}
\definecolor{darkblue}{rgb}{0,0,0.75}
\definecolor{darkred}{rgb}{0.6,0,0}
\end_preamble
\use_default_options true
\begin_modules
customHeadersFooters
minimalistic
todonotes
\end_modules
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding utf8
\fontencoding global
\font_roman "default" "default"
\font_sans "default" "default"
\font_typewriter "default" "default"
\font_math "auto" "auto"
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100 100
\font_tt_scale 100 100
\use_microtype true
\use_dash_ligatures true
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command biber
\index_command default
\paperfontsize default
\spacing onehalf
\use_hyperref true
\pdf_title "Training Neural Networks With Backpropagation"
\pdf_author "Andy Pack"
\pdf_subject "EEEM005"
\pdf_keywords "EEEM005"
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen false
\pdf_bookmarksopenlevel 1
\pdf_breaklinks false
\pdf_pdfborder true
\pdf_colorlinks false
\pdf_backref false
\pdf_pdfusetitle true
\papersize default
\use_geometry true
\use_package amsmath 1
\use_package amssymb 1
\use_package cancel 1
\use_package esint 1
\use_package mathdots 1
\use_package mathtools 1
\use_package mhchem 1
\use_package stackrel 1
\use_package stmaryrd 1
\use_package undertilde 1
\cite_engine biblatex
\cite_engine_type authoryear
\biblio_style plain
\biblio_options urldate=long
\biblatex_bibstyle ieee
\biblatex_citestyle ieee
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date true
\justification true
\use_refstyle 1
\use_minted 0
\index Index
\shortcut idx
\color #008000
\end_index
\leftmargin 1.8cm
\topmargin 2cm
\rightmargin 1.8cm
\bottommargin 2cm
\secnumdepth 3
\tocdepth 3
\paragraph_separation skip
\defskip medskip
\is_math_indent 0
\math_numbering_side default
\quotes_style english
\dynamic_quotes 0
\papercolumns 1
\papersides 1
\paperpagestyle fancy
\listings_params "language=Python,breaklines=true,frame=tb,otherkeywords={self},emph={State},emphstyle={\ttb\color{darkred}},basicstyle={\ttfamily},commentstyle={\bfseries\color{commentgreen}\itshape},keywordstyle={\color{darkblue}},emphstyle={\color{red}},stringstyle={\color{red}}"
\bullet 1 0 9 -1
\bullet 2 0 24 -1
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header
\begin_body
\begin_layout Title
\size giant
Training Neural Networks With Backpropagation
\end_layout
\begin_layout Author
Andy Pack
\end_layout
\begin_layout Standard
\begin_inset VSpace 15pheight%
\end_inset
\end_layout
\begin_layout Standard
\align center
\begin_inset Graphics
filename surrey.png
lyxscale 15
width 40col%
\end_inset
\end_layout
\begin_layout Standard
\begin_inset VSpace vfill
\end_inset
\end_layout
\begin_layout Standard
\noindent
\align center
EEEM005
\begin_inset Newline newline
\end_inset
May 2021
\size large
\begin_inset Newline newline
\end_inset
Department of Electrical and Electronic Engineering
\begin_inset Newline newline
\end_inset
Faculty of Engineering and Physical Sciences
\begin_inset Newline newline
\end_inset
University of Surrey
\end_layout
\begin_layout Standard
\begin_inset Newpage newpage
\end_inset
\end_layout
\begin_layout Right Footer
Andy Pack / 6420013
\end_layout
\begin_layout Left Footer
March 2021
\end_layout
\begin_layout Standard
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
pagenumbering{arabic}
\end_layout
\begin_layout Plain Layout
\backslash
setcounter{page}{1}
\end_layout
\end_inset
\end_layout
\begin_layout Section*
Experiment 1
\end_layout
\begin_layout Standard
The below results were obtained using a single hidden layer neural network
using the Tensorflow library.
Two test cases are shown, I was expecting the test error rate to increase
more as epochs went on so the latter has a much higher learning rate to
demonstrate.
\end_layout
\begin_layout Subsection*
Case 1
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
Iterations
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
30
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Nodes
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
[2, 8, 16, 24, 32]
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Epochs
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
[1, 2, 4, 8, 16, 32, 64, 100, 150, 200]
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Optimiser
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
Stochastic Gradient Descent
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Learning Rate
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
0.05
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Momentum
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
0.01
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Standard
\begin_inset Float figure
placement H
wide false
sideways false
status open
\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
filename ../graphs/exp1-test5-error-rate-curves.png
lyxscale 10
width 75col%
\end_inset
\end_layout
\begin_layout Plain Layout
\begin_inset Caption Standard
\begin_layout Plain Layout
Test error rates for different numbers of nodes over different epochs
\end_layout
\end_inset
\end_layout
\end_inset
\end_layout
\begin_layout Standard
\begin_inset Float figure
placement H
wide false
sideways false
status open
\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
filename ../graphs/exp1-test5-test-train-error-rate.png
lyxscale 10
width 100col%
\end_inset
\end_layout
\begin_layout Plain Layout
\begin_inset Caption Standard
\begin_layout Plain Layout
Test and train error rates for different numbers of nodes over different
epochs
\end_layout
\end_inset
\end_layout
\end_inset
\end_layout
\begin_layout Standard
\begin_inset Float figure
placement H
wide false
sideways false
status open
\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
filename ../graphs/exp1-test5-test-train-error-rate-std.png
lyxscale 10
width 100col%
\end_inset
\end_layout
\begin_layout Plain Layout
\begin_inset Caption Standard
\begin_layout Plain Layout
Test and train error rate standard deviations for different numbers of nodes
over different epochs
\end_layout
\end_inset
\end_layout
\end_inset
\end_layout
\begin_layout Standard
\begin_inset Newpage pagebreak
\end_inset
\end_layout
\begin_layout Subsection*
Case 2
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
Iterations
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
30
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Nodes
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
[2, 8, 16, 24, 32]
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Epochs
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
[1, 2, 4, 8, 16, 32, 64, 100, 150, 200]
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Optimiser
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
Stochastic Gradient Descent
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Learning Rate
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
1.5
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Momentum
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
0.5
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Standard
\begin_inset Float figure
placement H
wide false
sideways false
status open
\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
filename ../graphs/exp1-test6-error-rate-curves.png
lyxscale 10
width 75col%
\end_inset
\end_layout
\begin_layout Plain Layout
\begin_inset Caption Standard
\begin_layout Plain Layout
Test error rates for different numbers of nodes over different epochs
\end_layout
\end_inset
\end_layout
\end_inset
\end_layout
\begin_layout Standard
\begin_inset Float figure
placement H
wide false
sideways false
status open
\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
filename ../graphs/exp1-test6-test-train-error-rate.png
lyxscale 10
width 100col%
\end_inset
\end_layout
\begin_layout Plain Layout
\begin_inset Caption Standard
\begin_layout Plain Layout
Test and train error rates for different numbers of nodes over different
epochs
\end_layout
\end_inset
\end_layout
\end_inset
\end_layout
\begin_layout Standard
\begin_inset Float figure
placement H
wide false
sideways false
status open
\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
filename ../graphs/exp1-test6-test-train-error-rate-std.png
lyxscale 10
width 100col%
\end_inset
\end_layout
\begin_layout Plain Layout
\begin_inset Caption Standard
\begin_layout Plain Layout
Test and train error rate standard deviations for different numbers of nodes
over different epochs
\end_layout
\end_inset
\end_layout
\end_inset
\end_layout
\end_body
\end_document