#LyX 2.3 created this file. For more info see http://www.lyx.org/
\lyxformat 544
\begin_document
\begin_header
\save_transient_properties true
\origin unavailable
\textclass article
\use_default_options true
\begin_modules
customHeadersFooters
minimalistic
todonotes
\end_modules
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman "utopia" "default"
\font_sans "default" "default"
\font_typewriter "default" "default"
\font_math "auto" "auto"
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100 100
\font_tt_scale 100 100
\use_microtype false
\use_dash_ligatures true
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command biber
\index_command default
\paperfontsize default
\spacing single
\use_hyperref false
\pdf_title "Holoportation"
\pdf_author "Andy Pack"
\pdf_subject "The use of Kinect cameras to stream 3D video from client to server"
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen false
\pdf_bookmarksopenlevel 1
\pdf_breaklinks false
\pdf_pdfborder false
\pdf_colorlinks false
\pdf_backref false
\pdf_pdfusetitle true
\papersize default
\use_geometry true
\use_package amsmath 1
\use_package amssymb 1
\use_package cancel 1
\use_package esint 1
\use_package mathdots 1
\use_package mathtools 1
\use_package mhchem 1
\use_package stackrel 1
\use_package stmaryrd 1
\use_package undertilde 1
\cite_engine biblatex
\cite_engine_type authoryear
\biblio_style plain
\biblatex_bibstyle ieee
\biblatex_citestyle ieee
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date true
\justification true
\use_refstyle 1
\use_minted 0
\index Index
\shortcut idx
\color #008000
\end_index
\leftmargin 2cm
\topmargin 2cm
\rightmargin 2cm
\bottommargin 2cm
\secnumdepth 3
\tocdepth 3
\paragraph_separation skip
\defskip medskip
\is_math_indent 0
\math_numbering_side default
\quotes_style english
\dynamic_quotes 0
\papercolumns 1
\papersides 2
\paperpagestyle fancy
\bullet 1 0 9 -1
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Title

\size giant
Multi-Source Holoportation
\end_layout

\begin_layout Author
Andy Pack
\end_layout

\begin_layout Standard
\align center

\size largest
Mid-Term Report
\end_layout

\begin_layout Standard
\begin_inset VSpace bigskip
\end_inset


\end_layout

\begin_layout Standard
\align center
\begin_inset Graphics
	filename ../surreylogo.png
	lyxscale 30
	width 60col%

\end_inset


\end_layout

\begin_layout Standard
\begin_inset VSpace vfill
\end_inset


\end_layout

\begin_layout Standard
\align center

\size large
Department of Electrical and Electronic Engineering
\begin_inset Newline newline
\end_inset

Faculty of Engineering and Physical Sciences
\begin_inset Newline newline
\end_inset

University of Surrey
\end_layout

\begin_layout Standard
\begin_inset Newpage newpage
\end_inset


\end_layout

\begin_layout Abstract
The scope and current state of the multi-source holoportation project is
 examined.
 The aim is to extend a suite of 3D video capture software from it's current
 capabilities supporting multiple sensors but a single captured environment
 to handling multiple surroundings concurrently during frame collection
 and display.
 
\end_layout

\begin_layout Abstract
Currently the display methods have been developed in line with the specification
 in order to allow simultaneous display and arbitrary real-time placement
 within the display space.
 
\end_layout

\begin_layout Abstract
The future work for the project is described including the current designs
 these endeavours.
 The bulk of this remaining work involves developing the network capabilities
 of the software to accommodate multiple sources, this is on track to be
 completed by May 2020.
\end_layout

\begin_layout Standard
\begin_inset CommandInset toc
LatexCommand tableofcontents

\end_inset


\end_layout

\begin_layout Standard
\begin_inset VSpace medskip
\end_inset


\end_layout

\begin_layout Standard
\begin_inset FloatList figure

\end_inset


\begin_inset CommandInset toc
LatexCommand lstlistoflistings

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Newpage newpage
\end_inset


\end_layout

\begin_layout Right Footer
Andy Pack / 6420013
\end_layout

\begin_layout Left Footer
January 2020
\end_layout

\begin_layout Section
Introduction
\end_layout

\begin_layout Standard
The aim of this project is to develop a piece of software capable of supporting
 multi-source holoportation (hologram teleportation) using the 
\emph on
\noun on
LiveScan3D
\emph default
\noun default

\begin_inset CommandInset citation
LatexCommand cite
key "livescan3d"
literal "false"

\end_inset

 suite of software as a base.
\end_layout

\begin_layout Standard
As the spaces of augmented and virtual reality become more commonplace and
 mature, the ability to capture and stream 3D renders of objects and people
 over the internet using consumer-grade hardware has many possible applications.
\end_layout

\begin_layout Standard
This represents one of the most direct evolutions of traditional video streaming
 when applied to this new technological space.
\end_layout

\begin_layout Standard
A view of what multi-source achieves can be seen in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:premise"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
 Both single and multi-view configurations of cameras are shown, the latter
 allowing more complete renders of the subject to be acquired.
 Both shapes are presented through the 
\emph on
user experience
\emph default
, control schemes and visual language can vary between implementations across
 AR/VR and traditional 2D displays.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
	filename ../media/premise.png
	lyxscale 30
	width 70col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
Demonstration of a multi-source holoportation system including single and
 multiple view camera configurations
\begin_inset CommandInset label
LatexCommand label
name "fig:premise"

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Standard

\noun on
LiveScan3D
\noun default
 is a suite of 3D video software capable of recording and transmitting video
 from client to server for rendering.
 The suite is fast and uses consumer grade hardware for capture in the form
 of 
\noun on
Xbox Kinect
\noun default
 cameras, it is used in various projects at the 
\noun on
University of Surrey
\noun default
 and has multiple setups in dedicated lab space.
\end_layout

\begin_layout Standard

\noun on
LiveScan3D's
\noun default
 use
\noun on
 
\noun default
of 
\noun on
Xbox Kinect
\noun default
 cameras allows the capture and stream of 3D renders in single or multi-view
 configurations using calibrated cameras however the server is only able
 to process and reconstruct one environment at a time.
\end_layout

\begin_layout Standard
The capability to concurrently receive and reconstruct streams of different
 objects further broadens the landscape of possible applications, analogous
 to the movement from traditional phone calls to conference calling.
\end_layout

\begin_layout Section
Literature Review
\end_layout

\begin_layout Standard
The significance of 3D video like that captured and relayed using the 
\noun on
LiveScan
\noun default
 suite is related to the development of new technologies able to immersively
 display such video content.
 Therefore before discussing the specific extension that this project will
 make to the 
\noun on
LiveScan
\noun default
 software it is important to contextualise it within the space of 3D video
 capture while also considering it's implications for AR and VR applications.
\end_layout

\begin_layout Subsection
Cross Reality (XR)
\end_layout

\begin_layout Standard
Cross reality is a broad term describing the combination of technology with
 a user's experience of their surroundings in order to alter the experience
 of reality.
 It is used as an umbrella term for virtual, mixed and augmented reality
 experiences and technology.
 Before continuing, the differences between these technologies is considered.
\end_layout

\begin_layout Description
Virtual The replacement of a user's experience of their surroundings, rendering
 a new space that the user appears to inhabit.
 Typically achieved through face mounted headsets (
\emph on
Facebook Oculus, HTC Vive, Playstation VR, Valve Index
\emph default
).
\end_layout

\begin_layout Description
Augmented The augmentation of a users surroundings by overlaying the environment
 with digital alterations.
 Can be achieved with translucent/transparent headsets 
\emph on
(Microsoft Hololens, Google Glass)
\emph default
 or through mobile experiences 
\emph on
(Android ARCore, Apple ARKit)
\emph default
 both when head mounted 
\emph on
(Google Cardboard, Google Daydream, Samsung Gear VR)
\emph default
 and handheld 
\emph on
(Pokemon GO)
\emph default
.
\end_layout

\begin_layout Description
Mixed A combination of virtual and augmented elements in order to allow
 interaction with an augmented reality.
 Can be achieved in different ways typically starting with either a typical
 AR or VR experience and including aspects of the other.
 At a higher level, mixed reality can be described as a continuous scale
 between the entirely real and entirely virtual with augmented reality occurring
 in between.
\end_layout

\begin_layout Standard
The burgeoning of these three forms of XR via consumer hardware such as
 the 
\noun on
Microsoft Hololens
\noun default
 and 
\noun on
Oculus Rift
\noun default
 represents a new space for the consumption of interactive media experiences.
\end_layout

\begin_layout Standard
Although VR and AR headsets have accelerated the development of XR technology,
 they are not the only way to construct XR experiences.
 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "roomalive"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "roomalive"
literal "false"

\end_inset

 demonstrate 
\emph on
RoomAlive
\emph default
, an AR experience using depth cameras and projectors (refereed to as 
\emph on
procams
\emph default
) to construct experiences in any room.
 This is presented through games and visual alterations to the users surrounding
s.
 A strength of the system is it's self contained nature, able to automatically
 calibrate the camera arrangements using correspondences found between each
 view.
 Experience level heuristics are also discussed regarding capturing and
 maintaining user attention in an environment where the experience can be
 occurring anywhere, including behind the user .
 
\end_layout

\begin_layout Standard
A point is also made about how the nature of this room based experience
 breaks much of the typical game-user interaction established by virtual
 reality and video games.
 In contrast to traditional and virtual reality game experiences where the
 game is ultimately in control of the user or user avatar, AR experiences
 of this type have no physical control over the user and extra considerations
 must be made when designing such systems.
\end_layout

\begin_layout Standard
Traditional media consumption is not the only area of interest for developing
 interactive experiences, an investigation into the value of AR and VR for
 improving construction safety is presented by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "ar/vr-construction"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "ar/vr-construction"
literal "false"

\end_inset

.
 A broad look at the applicability is taken with assessments including VR
 experiences for developing worker balance to aid in working at elevation
 and AR experiences incorporated into the workplace for aiding in task sequencin
g to reduce the effect of memory on safety.
\end_layout

\begin_layout Standard
\begin_inset CommandInset citation
LatexCommand citeauthor
key "remixed-reality"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "remixed-reality"
literal "false"

\end_inset

 demonstrate an example of mixed reality through the use of 
\noun on
Kinect
\noun default
 cameras and a virtual reality headset.
 Users are placed in a virtual space constructed from 3D renders of the
 physical environment around the user.
 Virtual manipulation of the space can then be achieved with visual, spatial
 and temporal changes supported.
 Objects can be scaled and sculpted in realtime while the environment can
 be paused and rewinded.
 The strength of mixed reality comes with the immersion of being virtually
 placed in a version of the physical surroundings, tactile feedback from
 the environment compounds this.
\end_layout

\begin_layout Subsection
Kinect and RGB-D Cameras
\end_layout

\begin_layout Standard
Initially designed as a motion control accessory for the 
\noun on
Xbox
\noun default
, the 
\noun on
Kinect
\noun default
 is a series of depth aware cameras produced my 
\noun on
Microsoft
\noun default
.
 The device uses additional infrared lights and sensors alongside an RGB
 camera in a configuration referred to as a time of flight camera to generate
 3D renders of a surroundings.
 The device also includes motion tracking and skeleton isolation for figures
 in view.
\end_layout

\begin_layout Standard
Following the release of an SDK for Windows in 2012, 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "original-kinect-microsoft"
literal "false"

\end_inset

 at 
\noun on
Microsoft Research
\noun default
 reflects on the original camera's capabilities and the applications to
 computer vision research in 
\begin_inset CommandInset citation
LatexCommand cite
key "original-kinect-microsoft"
literal "false"

\end_inset

.
\end_layout

\begin_layout Standard
Here 3D conference calling of the type described in the introduction without
 AR or VR applications is presented, instead users watch a composite conference
 space on a screen with all participants rendered within.
 Work was undertaken to achieve mutual gaze between participants, a marked
 advantage over traditional conference calls where the lack of such aspects
 of group interaction make the experience more impersonal.
 Methods of achieving more natural virtual interactions or 
\emph on
telepresence
\emph default
 are covered in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Holoportation-and-Telepresence"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
A second version of the camera, v2, was released alongside the 
\noun on
Xbox One
\noun default
 in 2013 and presented many improvements over the original.
 A higher quality RGB camera captures 1080p video at up to 30 frames per
 second with a wider field of view than the original
\begin_inset CommandInset citation
LatexCommand cite
key "kinect-specs"
literal "false"

\end_inset

.
 The physical capabilities of the camera are discussed by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "new-kinect"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "new-kinect"
literal "false"

\end_inset

.
 The second version of the camera was found to gather more accurate depth
 data than the original and was less sensitive to daylight.
 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "kinectv1/v2-accuracy-precision"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "kinectv1/v2-accuracy-precision"
literal "false"

\end_inset

 found similar results with the v2 achieving higher accuracy results over
 the original.
 The second version did, however, achieve lower precision results than the
 v1 with recommendations made to include pre-processing on acquired depth
 images to control for random noise, 
\emph on
flying pixels
\emph default
 and 
\emph on
multipath interference
\emph default
.
\end_layout

\begin_layout Standard
The 
\noun on
Kinect
\noun default
 is used successfully by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "greenhouse-kinect"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "greenhouse-kinect"
literal "false"

\end_inset

 for object detection in the context of an autonomous vehicle navigating
 a greenhouse.
 The depth information was used in conjunction with the RGB information
 to identify obstacles, while the paper lays out some limitations of the
 camera it was found to be effective in it's aim and was capable of running
 on a reasonable computer.
\end_layout

\begin_layout Standard
This second iteration on the 
\noun on
Kinect
\noun default
 is frequently used in computer vision experiments with many of the works
 cited here using it for acquisition.
\end_layout

\begin_layout Subsection
Holoportation and Telepresence
\begin_inset CommandInset label
LatexCommand label
name "subsec:Holoportation-and-Telepresence"

\end_inset


\end_layout

\begin_layout Standard
The term Holoportation is defined and exemplified in a 
\noun on
Microsoft Research
\noun default
 paper by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "holoportation"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "holoportation"
literal "false"

\end_inset

 where an end-to-end pipeline is laid out for the acquisition, transmission
 and display of 3D video facilitating real-time AR and VR experiences.
 The 
\noun on
Microsoft Research
\noun default
 paper builds on works including by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "Immersive-telepresence"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "Immersive-telepresence"
literal "false"

\end_inset

 2 years earlier which describes attempts at achieving 
\begin_inset Quotes eld
\end_inset


\emph on
telepresence
\emph default

\begin_inset Quotes erd
\end_inset

, a term coined by Marvin Minksy to describe the transparent and intuitive
 remote control of robot arms as if they were the controllers own
\begin_inset CommandInset citation
LatexCommand cite
key "marvin-minksy"
literal "false"

\end_inset

.
 The term was broadened by Bill Buxton
\begin_inset CommandInset citation
LatexCommand cite
key "buxton-telepresence"
literal "false"

\end_inset

 to include the space of telecommunications to describe technology being
 used to make someone feel present in a different environment.
 In the context of holoportation this is through the use of 3D video reconstruct
ion.
 The aforementioned work by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "Immersive-telepresence"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "Immersive-telepresence"
literal "false"

\end_inset

 used 10 
\noun on
Kinect
\noun default
 cameras to capture a room before virtually reconstructing the models.
 
\end_layout

\begin_layout Standard
In service of demonstrating it's applicability to achieving 
\emph on
telepresence
\emph default
, a figure was isolated from the surroundings and stereoscopically rear-projecte
d onto a screen for a single participant, a result of this can be seen in
 figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:stereoscopic"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename ../media/telepresence-stereoscopic.png
	lyxscale 30
	width 40col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
An example of stereoscopic projection of depth aware footage captured by
 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "Immersive-telepresence"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "Immersive-telepresence"
literal "false"

\end_inset


\begin_inset CommandInset label
LatexCommand label
name "fig:stereoscopic"

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Standard
The 
\noun on
Microsoft Research
\noun default
 paper demonstrates a system using 8 cameras surrounding a space.
 Each camera captured both Near Infra-Red and colour images to construct
 a colour-depth video stream, a more complex camera configuration than in
 the others cited.
\end_layout

\begin_layout Standard
\begin_inset CommandInset citation
LatexCommand citeauthor
key "velt"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "velt"
literal "false"

\end_inset

 demonstrates a similar holoportation experience to 
\noun on
LiveScan3D
\noun default
 capable of supporting multi-view configurations, it also supports both
 point clouds and meshes.
 Calibrating multiple view points is completed using the extrinsics and
 intrinsics of the camera.
 The extrinsics are the relative positions of each 
\noun on
Kinect
\noun default
 camera while the intrinsics describe the internal properties of each camera,
 the focal length and optical centre.
 
\end_layout

\begin_layout Standard
The intrinsics of the 
\noun on
Kinect
\noun default
 camera can be retrieved from the 
\noun on
Kinect
\noun default
 SDK while the extrinsics are obtained in one of two ways.
 Extrinsics can be imported and parsed from XML for manual selection or
 estimated using 
\noun on
OpenCV
\noun default
 and a checkerboard pattern.
 When considering holoportation systems of this kind, comparatively few
 implement multiple views as a result the increased complexity involved
 in calibration.
\end_layout

\begin_layout Subsection
Multi-Source Holoportation
\end_layout

\begin_layout Standard
The space of multi-source holoportation has been explored by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "group-to-group-telepresence"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "group-to-group-telepresence"
literal "false"

\end_inset

 in the context of shared architectural design spaces in virtual reality
 similar to a conference call.
 Two groups of people were captured in 3D using clusters of 
\noun on
Kinect
\noun default
 cameras before having these renders transmitted to the other group.
 Each group reconstructs the other's render for display in virtual reality
 in conjunction with their own.
 In doing so a shared virtual space for the two groups has been created
 and it can be seen to implement the process of holoportation.
 The strength of the system as a shared architectural design experience
 is emergent of the semantics of the virtual space where a World in Miniature
 (WIM) metaphor is used.
\end_layout

\begin_layout Standard
The Worlds in Miniature is described by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "wim"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "wim"
literal "false"

\end_inset

 as a set of interfaces between the user and the virtual space they experience
 using tactile and visual tools.
 The interface involves providing the user with a miniature render of the
 world they are inhabiting that can interacted with in order to affect the
 full scale environment around them.
\end_layout

\begin_layout Standard
This navigation tool maps well to 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "group-to-group-telepresence"
literal "false"

\end_inset

's
\begin_inset CommandInset citation
LatexCommand cite
key "group-to-group-telepresence"
literal "false"

\end_inset

 architecture groupware design, an image captured during the work can be
 seen in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:World-in-Miniature-group-by-group"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename ../media/group-by-group.png
	lyxscale 30
	width 50col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
World in Miniature render demonstrated in a multi-source holoportation context
 by 
\begin_inset CommandInset citation
LatexCommand citeauthor
key "group-to-group-telepresence"
literal "false"

\end_inset


\begin_inset CommandInset citation
LatexCommand cite
key "group-to-group-telepresence"
literal "false"

\end_inset


\begin_inset CommandInset label
LatexCommand label
name "fig:World-in-Miniature-group-by-group"

\end_inset


\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Section
LiveScan3D
\end_layout

\begin_layout Standard

\noun on
LiveScan3D
\noun default
 is a suite of software developed by Marek Kowalski, Jacek Naruniec and
 Michal Daniluk of the Warsaw University of Technology in 2015
\begin_inset CommandInset citation
LatexCommand cite
key "livescan3d"
literal "false"

\end_inset

.
 The suite utilises the 
\noun on
Xbox Kinect
\noun default
 v2 camera to record and transmit 3D renders over an IP network.
 A server can manage multiple clients simultaneously and is responsible
 for processing, reconstructing and displaying the renderings in real-time.
\end_layout

\begin_layout Standard
These renderings take the form of a point cloud, a collection of 3D co-ordinates
 indicating the position of each voxel (3D pixel) and it's associated RGB
 colour value.
 As a result of it's analogous nature to a traditional frame of 2D video,
 the terms 
\begin_inset Quotes eld
\end_inset

render
\begin_inset Quotes erd
\end_inset

, 
\begin_inset Quotes eld
\end_inset

point cloud
\begin_inset Quotes erd
\end_inset

 and 
\begin_inset Quotes eld
\end_inset

frame
\begin_inset Quotes erd
\end_inset

 are used interchangeably from here.
\end_layout

\begin_layout Standard
The majority of the development being conducted in this project is regarding
 the server component of the software and as such this is covered in more
 detail.
\end_layout

\begin_layout Subsection

\noun on
LiveScan
\noun default
 Client
\end_layout

\begin_layout Standard
The 
\noun on
LiveScan
\noun default
 Client is responsible for interfacing with the 
\noun on
Kinect
\noun default
 sensor via the 
\noun on
Kinect
\noun default
 v2 SDK and transmitting frames to the 
\noun on
LiveScan
\noun default
 Server.
 Body detection takes place client side, as does calibration when using
 multiple sensors.
\end_layout

\begin_layout Standard
Only one 
\noun on
Kinect
\noun default
 sensor can be connected to each computer as a result of the SDK's restrictions.
 A system used by multiple clients in this way lends itself well to multi-source
 configurations over the internet.
\end_layout

\begin_layout Subsection

\noun on
LiveScan
\noun default
 Server
\end_layout

\begin_layout Standard
The server component of the 
\noun on
LiveScan
\noun default
 suite is responsible for managing and receiving 3D renders from connected
 clients.
 These renders are reconstructed in an interactive 
\noun on
OpenGL 
\noun default
window.
 When considering the code architecture of this application there are three
 main components.
 
\end_layout

\begin_layout Description
OpenGLWindow Presentation layer of the application.
 Separate window spawned by the 
\noun on
LiveScanServer
\noun default
 responsible for drawing point clouds and responding to user control.
 
\end_layout

\begin_layout Description
KinectServer Network layer of the application.
 The main window make requests of this component to receive transmitted
 point clouds.
\end_layout

\begin_layout Description
KinectSocket
\noun on
 
\noun default
Child objects contained within the 
\noun on
KinectServer
\noun default
.
 A traditional network socket object representing a single TCP connection
 between the server and a client.
\end_layout

\begin_layout Standard
This structure can be seen in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:server-structure"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename ../media/initial-state.png
	lyxscale 30
	width 50col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
Initial architecture of the 
\noun on
LiveScan3D
\noun default
 server
\begin_inset CommandInset label
LatexCommand label
name "fig:server-structure"

\end_inset


\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
Received frames in the form of lists of vertices, RGB values, camera poses
 and bodies overwrite shared variables between the main window and the 
\noun on
OpenGL
\noun default
 window.
\end_layout

\begin_layout Subsection
Calibration & Multi-View Configurations
\end_layout

\begin_layout Standard
When using a single client setup, frames are transmitted in their own coordinate
 space with the origin defined as the 
\noun on
Kinect
\noun default
 camera and the captured scene rendered in front of it.
\end_layout

\begin_layout Standard
When using multiple sensors, the server would be unable to combine these
 unique Euclidean spaces without knowledge of the sensors positions relative
 to each other, the extrinsics of the system.
\end_layout

\begin_layout Standard
In order to make a composite frame a calibration process is completed client
 side following instruction by the server.
\end_layout

\begin_layout Standard
Calibration is completed in two steps, an initial estimation followed by
 a refinement process.
 The initial estimation is completed by informing the server of which calibratio
n marker layouts are being used within the space.
 Client's identify possible visible markers like that seen in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:calibration-marker"
plural "false"
caps "false"
noprefix "false"

\end_inset

 using thresholding.
 Following this identification, the location of the marker can be found
 within the sensors coordinate system.
 
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
	filename /home/andy/uni/dissertation/media/calibration.png
	lyxscale 30
	width 20col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
Example marker used within the LiveScan3D calibration process
\begin_inset CommandInset label
LatexCommand label
name "fig:calibration-marker"

\end_inset


\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
This information can be used to transform points from the cameras coordinate
 system to the markers frame of reference.
 As the relative locations of different markers are defined at the server,
 a world coordinate system can be defined as the centre of these markers.
 Typically 4 different markers are placed on the faces around the vertical
 axis of a cuboid allowing views in 360°.
\end_layout

\begin_layout Standard
This world coordinate space has shifted the origin from being the position
 of the single 
\noun on
Kinect
\noun default
 sensor to being a point in the centre of the calibration markers that each
 camera now orbits.
 As part of this calibration process the server distributes transformations
 to each client defining where they sit within this world coordinate space.
 Client's can now transform acquired renders from their own frame of reference
 to the world coordinate system at the point of capture and each point cloud
 can be merged coherently.
\end_layout

\begin_layout Standard
The refinement process is completed server side by requesting a single frame
 from each connected client and using Iterative Closest Points
\begin_inset CommandInset citation
LatexCommand cite
key "ICP"
literal "false"

\end_inset

 (ICP) to improve the inter-camera relationships.
\end_layout

\begin_layout Standard
The 
\noun on
OpenGL
\noun default
 display space has it's origin within the centre of the visible box, this
 means that for single sensor setups this is also the location of the camera.
\end_layout

\begin_layout Subsection
Design Considerations
\end_layout

\begin_layout Standard
When assessing 
\noun on
LiveScan
\noun default
's suitability for extension to a multi-source context, the original network
 design should be investigated.
\end_layout

\begin_layout Standard
The original applications were best suited to a local environment as a result
 of many of the network functions being blocking.
 Should any delays or interruptions have occurred during a network operation,
 then the application would need to stop and wait for remediation before
 continuing.
 Interruptions of this type are more common when moving from a LAN environment
 to communicating over the open internet.
\end_layout

\begin_layout Standard
From a network perspective the need to make these actions non-blocking would
 present benefits for both multi-source and multi-view configurations.
\end_layout

\begin_layout Standard
Additionally, the network polling rates are higher than the frame rate of
 the produced video, when the server requests a frame before a new one has
 been captured by the client, the same previous frame is resent.
 This presents unnecessary bandwidth usage.
\end_layout

\begin_layout Standard
Moving to a multi-source context implies transmitting over the internet
 as opposed to local operation, this will make blocking actions and bloated
 bandwidth more dangerous to user experience.
\end_layout

\begin_layout Standard
Work has been undertaken that allows multiple concurrent TCP connections
 to be used by each client to increase bandwidth.
 Further work is being undertaken to un-block network actions.
\end_layout

\begin_layout Section
Current Work
\end_layout

\begin_layout Standard
The required development to take the existing 
\noun on
LiveScan
\noun default
 codebase to the desired multi-source result can be split into two areas
 of concern.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\series bold
\emph on
Network
\series default
\emph default
 The network layer of the 
\noun on
LiveScan
\noun default
 server must be updated in order to accommodate multiple clients logically
 grouped into 
\begin_inset Quotes eld
\end_inset

sources
\begin_inset Quotes erd
\end_inset

 for which separate frames are collected for display.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\series bold
\emph on
Display
\series default
\emph default
 Finally the display element of the server should be extended to allow the
 simultaneous presentation of multiple point clouds.
 These objects should be individually arrangeable in the display space allowing
 both movement and rotation.
\end_layout

\begin_layout Standard
As of January 2020 the native method for displaying renderings, the server's
 
\noun on
OpenGL
\noun default
 window, has been modified such that it can construct and render point clouds
 from multiple sources.
 To do so a dynamic sub-system of geometric transformations has been written
 in order to coherently arrange sources within the space when reconstructed.
 The default arrangements can be overridden with keyboard controls facilitating
 arbitrary placement and rotation of separate sources within the window's
 co-ordinate space.
\end_layout

\begin_layout Subsection
Geometric Transformations
\end_layout

\begin_layout Standard
Within the 
\noun on
LiveScan3D
\noun default
 server source code are utility structures and classes which were extended
 in order to develop a wider geometric manipulation system.
 Structures defining Cartesian coordinates in both 2D and 3D spaces called
 
\noun on
Point2f
\noun default
 and 
\noun on
Point3f
\noun default
 respectively are used in drawing skeletons as captured by the 
\noun on
Kinect
\noun default
 camera.
 There is also a class defining an affine transformation, the definitions
 for all three can be seen in appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Existing-Data-Structures"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
Affine transformations are a family of geometric transformations that preserve
 parallel lines within geometric spaces.
 Some examples of affine transformations include scaling, reflection, rotation,
 translation and shearing.
\end_layout

\begin_layout Standard
The class definition is made up of a three-by-three transformation matrix
 and single 3D vector for translation, within the native codebase it is
 used for both camera poses and world transformations.
 
\end_layout

\begin_layout Standard
A camera pose is the affine transformation defining the position and orientation
 of the 
\noun on
Kinect
\noun default
 camera when drawn in the 
\noun on
OpenGL
\noun default
 space as a green cross.
 The world transformations are used as part of the calibration process when
 using multi-view configurations.
 
\end_layout

\begin_layout Standard
When considering how each source's render would be arranged in the space,
 the use of this class definition was extended.
 As the use of affine transformations is mostly limited to use as a data
 structure within the base source code, some utility classes and functions
 were required in order to fully maximise their effectiveness.
\end_layout

\begin_layout Subsubsection
Transformer
\end_layout

\begin_layout Standard
The motivation in writing the 
\noun on
Transformer
\noun default
 was to create a generic framework of geometric transformations that could
 be utilised by the 
\noun on
OpenGL
\noun default
 display to arrange separate point clouds.
 At a high level this is done by implementing matrix arithmetic functions
 in the context of their use for applying linear transformations to Cartesian
 coordinates.
 
\end_layout

\begin_layout Standard
The 
\noun on
Transformer
\noun default
 class has static methods to apply 
\noun on
AffineTransform
\noun default
s to both 
\noun on
Point3f
\noun default
 structures and lists of raw vertices as received from 
\noun on
LiveScan
\noun default
 clients.
\end_layout

\begin_layout Standard
Additionally there are utility functions to bidirectionally cast between
 
\noun on
Point3f
\noun default
 data structures and the lists of received vertices.
\end_layout

\begin_layout Standard
Finally static methods generate common rotation transformations about each
 axis given an arbitrary angle.
 This provided a foundation on which to define how the 
\noun on
OpenGL
\noun default
 space would arrange separate sources within it's combined co-ordinate space.
\end_layout

\begin_layout Standard
Currently missing is the ability to combine transformations into compound
 matrices.
 Applying multiple transformations to large numbers of coordinates would
 be more computationally expensive than applying one compound matrix and
 when running in realtime this should be considered.
 This is not yet included due to the current lack of need to apply multiple
 successive transformations.
 If the need were to arise following further refinements, it would be implemente
d as described here.
\end_layout

\begin_layout Subsection
Separation of Network and Presentation Layer
\end_layout

\begin_layout Standard
During initial testing frames received from a live sensor were intercepted
 and serialized to XML in local storage.
 These frames were loaded into memory as the server was started and merged
 with those received live before display.
\end_layout

\begin_layout Standard
The composite frame can be seen in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:Initial-composite-frame"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename ../media/pretransform.jpg
	lyxscale 10
	width 50col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
Initial multi-source composite testing frame
\begin_inset CommandInset label
LatexCommand label
name "fig:Initial-composite-frame"

\end_inset


\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
The objects can be seen to be occupying the same space due to their similar
 positions in the frame during capture.
 This is not a sufficient solution for displaying separate sources and so
 geometric transformations like those described above were employed to separate
 the two.
 The change in software structure at this stage can be seen in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:Initial-testing-layout"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
 A rotation of 180° in the vertical (
\begin_inset Formula $y$
\end_inset

) axis pivoted the frames such that they faced those being received live,
 the results can be seen in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:180-degree-rotation"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename ../media/local-testing.png
	lyxscale 30
	width 70col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
Initial testing process transforming frames loaded from local storage
\begin_inset CommandInset label
LatexCommand label
name "fig:Initial-testing-layout"

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename ../media/180flip.jpg
	lyxscale 10
	width 50col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
Composite testing frame following 180° rotation of recorded source in 
\begin_inset Formula $y$
\end_inset

 axis
\begin_inset CommandInset label
LatexCommand label
name "fig:180-degree-rotation"

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Standard
At this point it was noted that transforming and arranging figures within
 the main window before passing the 
\noun on
OpenGL
\noun default
 window a complete point cloud spreads responsibility for the display logic
 to the main window.
\end_layout

\begin_layout Standard

\noun on
LiveScan3D
\noun default
 is capable of supporting more display methods than just the native 
\noun on
OpenGL
\noun default
 implementation with versions available for both 
\noun on
Microsoft Hololens
\noun default
 and Mobile AR applications.
 Therefore when designing the multi-source capabilities, the separation
 of logic between the network and presentation layer is important.
 
\end_layout

\begin_layout Standard
The way in which the 
\noun on
OpenGL
\noun default
 window arranges the figures in it's display space should be defined by
 the 
\noun on
OpenGL
\noun default
 window itself.
 The network layer should be display agnostic and not make assumptions about
 how the display will process figures.
\end_layout

\begin_layout Standard
In order to follow this design the transformations were moved to instead
 occur within the 
\noun on
OpenGL
\noun default
 window class.
 To allow this the shared variables between the 
\noun on
MainWindow
\noun default
 and 
\noun on
OpenGL
\noun default
 were changed.
 A Frame structure was defined to wrap an individual point cloud with a
 client ID to allow differentiation, the definition can be seen in appendix
 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Frame"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
 The structure holds fields for each of the lists previously shared between
 the two objects including a list of vertices (co-ordinates) and the RGB
 values for each as well as the camera poses and bodies.
\end_layout

\begin_layout Standard
The original 
\noun on
LiveScan3D
\noun default
 cleared each of these variables before retrieving a new frame, when moving
 to a multi-source architecture the ability to individually update source
 point clouds was prioritised.
 This would avoid blocking the entire display when unable to receive frames
 from a specific client, other clients would still be able to have frames
 updated promptly.
\end_layout

\begin_layout Standard
To accomplish this a dictionary was used as the shared variable with each
 client's frame referenced by it's client ID.
 In doing so only one frame per client is kept and each new frame overrides
 the last.
 During rendering the dictionary is iterated through and each point cloud
 combined.
 During combination a client specific transformation is retrieved from an
 instance of the 
\noun on
DisplayFrameTransformer
\noun default
 class.
 This object is a member of the 
\noun on
OpenGL
\noun default
 window and is responsible for defining the orientation and position of
 each point cloud.
\end_layout

\begin_layout Subsection
DisplayFrameTransformer
\end_layout

\begin_layout Standard
The 
\noun on
DisplayFrameTransformer
\noun default
 is responsible for generating transformations for the sources displayed
 within the 
\noun on
OpenGL
\noun default
 window, a UML diagram for the class can be seen in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:UML-displayframetransformer"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename ../media/DisplayFrameTransformer.png
	lyxscale 50
	width 60col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
UML diagram for 
\noun on
DisplayFrameTransformer
\noun default

\begin_inset CommandInset label
LatexCommand label
name "fig:UML-displayframetransformer"

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Standard
Each client is assigned a default transformation which can be overridden
 using keyboard controls.
\end_layout

\begin_layout Standard
Clients are initially arranged in a circle around the origin in the center
 of the space.
 This is done by retrieving a transformation from the 
\noun on
Transformer
\noun default
 for a rotation in the 
\begin_inset Formula $y$
\end_inset

 axis for each client, 
\begin_inset Formula $n$
\end_inset

.
 Each angle of rotation, 
\begin_inset Formula $\alpha$
\end_inset

, is calculated using the below, 
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\alpha\left(n\right)=\frac{n}{\sum clients}\cdotp360\textdegree
\]

\end_inset


\end_layout

\begin_layout Standard
Similar to the shared variables between the 
\noun on
MainWindow
\noun default
 and 
\noun on
OpenGL
\noun default
 window, client transformations are stored within a dictionary indexed by
 client ID.
\end_layout

\begin_layout Standard
The 
\noun on
DisplayFrameTransformer
\noun default
 also has methods to override these initial transforms with the RotateClient()
 and TranslateClient() methods.
 When these methods are called for the first time on a point cloud, an object
 defining the position and rotation is populated using the default rotation.
 From here the presence of a client override results in applied transforms
 being defined by these values as opposed to the default orientation.
\end_layout

\begin_layout Standard
This leaves the current architecture of the server application as described
 in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:current-state-diagram"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename ../media/december-state.png
	lyxscale 30
	width 60col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
Current state of 
\noun on
LiveScan
\noun default
 server structure with 
\noun on
OpenGL
\noun default
 window-based transformer
\begin_inset CommandInset label
LatexCommand label
name "fig:current-state-diagram"

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Subsection
Control Scheme
\end_layout

\begin_layout Standard
The movement of objects within the 
\noun on
OpenGL
\noun default
 space is implemented through keyboard controls.
 While using the mouse would allow fine-grained and intuitive control, the
 number of axes for motion and rotation available to objects makes defining
 specific keys for each more flexible.
 This additionally removes the need to redefine or overload the camera controls.
\end_layout

\begin_layout Standard
The 
\begin_inset Quotes eld
\end_inset

I
\begin_inset Quotes erd
\end_inset

 key is used to cycle through displayed sources, the currently selected
 source is the subject of each of the movement actions.
 Sources are moved across the horizontal plane (
\family roman
\series medium
\shape up
\size normal
\emph off
\bar no
\strikeout off
\xout off
\uuline off
\uwave off
\noun off
\color none

\begin_inset Formula $x$
\end_inset


\family default
\series default
\shape default
\size default
\emph default
\bar default
\strikeout default
\xout default
\uuline default
\uwave default
\noun default
\color inherit
, 
\family roman
\series medium
\shape up
\size normal
\emph off
\bar no
\strikeout off
\xout off
\uuline off
\uwave off
\noun off
\color none

\begin_inset Formula $z$
\end_inset


\family default
\series default
\shape default
\size default
\emph default
\bar default
\strikeout default
\xout default
\uuline default
\uwave default
\noun default
\color inherit
) of the display space using a WASD-esque layout of the UHJK keys.
 Objects can be rotated about the vertical (
\begin_inset Formula $y$
\end_inset

) axis using the B and N keys.
 Finally the placement of an object can be reset to default using the R
 key, the addition of the shift modifier resets all clients.
\end_layout

\begin_layout Standard
Worth noting is that this represents arbitrary placement of sources in only
 two axes of position and one of rotation.
 This was a conscious choice as these are the most common and intuitive
 axes with which sources will need to be manipulated.
 The ability to allow movement in all axes would require only binding these
 actions to keys.
\end_layout

\begin_layout Standard
There is room to improve these controls as the directions of movement are
 in relation to the fixed axes of the display space as opposed to the view
 of the viewpoint camera.
 In practice this means that when moving objects in the display space the
 orientation of the space must be considered in order to identify which
 direction the object should be moved.
 
\end_layout

\begin_layout Standard
This is less intuitive than could be expected in other areas where such
 a control scheme is used such as video games or modelling software.
 In such implementations when moving objects the directions are typically
 taken from the camera's frame of reference.
 The feasibility of employing a similar control philosophy should be considered.
\end_layout

\begin_layout Subsection
Challenges
\end_layout

\begin_layout Standard
The main challenge encountered throughout the project so far was initially
 intercepting the live frames and serializing these as XML files in local
 storage.
 With no previous experience developing in C#, the opening steps of the
 project involved both getting to grips with the language based on previous
 work in C-like languages (Java, C) and understanding the layout of the
 codebase.
\end_layout

\begin_layout Standard
Initial attempts to serialize the frame structures resulted in no output
 to the file system and the multi-threaded nature of the graphical application
 led to no feedback for debugging.
 This was fixed by removing the affine transformations representing camera
 poses from the frame structure for the testing process.
 
\end_layout

\begin_layout Standard
This would imply that something about the nature of the 
\noun on
AffineTransform
\noun default
 class type is causing errors when serializing.
 Java requires classes implement a 
\emph on
serializable
\emph default
 interface in order to successfully save to file and further work will be
 required in order to determine whether the same concept is to blame in
 this situation.
 However for now the camera poses of local frames are not displayed in the
 
\noun on
OpenGL
\noun default
 window.
\end_layout

\begin_layout Section
Future Work
\end_layout

\begin_layout Standard
Following the extension of the 
\noun on
OpenGL
\noun default
 window, the network layer of the 
\noun on
KinectServer
\noun default
 can now be developed with the advantage of a fully functional display method
 for debugging.
\end_layout

\begin_layout Standard
The aim of this aspect of the project will be to alter the the 
\noun on
KinectServer
\noun default
 in order to allow the logical grouping of connected clients into sources
 for separate display.
\end_layout

\begin_layout Standard
When integrated together the server as a whole will then be able to collect
 discrete point clouds from different sources and coherently display them
 separately in the space, achieving the objectives for this project.
\end_layout

\begin_layout Subsection
Network Layer Design Considerations
\end_layout

\begin_layout Standard
Some thought as to the design for the network layer has been undertaken.
 Although this has not yielded a final design for implementation, it has
 made apparent some of the conditions and constraints which must be considered.
\end_layout

\begin_layout Standard
When considering the initial steps for the project, it was thought that
 the network layer should be developed first.
 The design would involve separating much of the logic contained within
 the 
\noun on
KinectServer
\noun default
 object into a new 
\noun on
KinectSource
\noun default
 object which would represent a group of clients acting as a single source.
 It would function as a group of 
\noun on
KinectSocket
\noun default
s that could be individually polled for new frames using the same interface
 currently being used by the 
\noun on
KinectServer
\noun default
.
 The 
\noun on
KinectServer
\noun default
 object itself would be reduced to simply managing these 
\noun on
KinectSource
\noun default
s.
\end_layout

\begin_layout Standard
An advantage of this approach would be that it provide a suitable location
 to store additional information which should exist per source such as the
 calibration data and settings.
\end_layout

\begin_layout Standard
However it would also have represented a significant architecture change
 in the entire server application and without a functioning display method
 it would have been challenging to debug.
 This was the motivation for initially working on the display method.
\end_layout

\begin_layout Standard
Coming back to the network design following this work, a different design
 has been considered.
 A separate body of work currently being undertaken is investigating the
 network behaviour of the suite with a focus on unblocking the network sockets
 to aid in parallel operation.
\end_layout

\begin_layout Standard
In order to ease integration with developments in this work a less disruptive
 design was proposed.
\end_layout

\begin_layout Subsubsection
Socket Handshake
\end_layout

\begin_layout Standard
The aim is to implement a method by which clients are grouped into sources
 that also allows them to identify themselves consistently when communicating
 over multiple sockets.
 Multiple sockets can be used by clients in order to make simultaneous connectio
ns to the server and increase bandwidth.
 However when doing so it is important to be able to identify which sockets
 represent which client when some may be at the IP address.
\end_layout

\begin_layout Standard
A method for doing so would involve a handshake process when new clients
 connect to the 
\noun on
KinectServer
\noun default
.
 The proposed handshake would be initiated by the client when connecting
 to the server, at this point they include which source they should be grouped
 with using an integer ID.
 The server groups the socket as such and, if one has not been received,
 responds with a random identifier string that should be used across all
 sockets to identify the client.
 Should the newly connected socket be for a client that is already connected
 then the client will respond with it's existing identifier to inform the
 server that this ID has been ignored.
 In doing so the client now has a method of identifying itself agnostic
 of socket, and the server has a way of identifying the source which each
 socket is representing.
\end_layout

\begin_layout Subsection
Deliverables and Additional Goals
\end_layout

\begin_layout Standard
At this point in the project it is worth considering the viability of the
 final deliverables with relation to the time remaining.
 Based on the work completed so far the original objectives of multi-source
 holoportation remain viable with a round of complete testing undertaken.
\end_layout

\begin_layout Standard
This testing suite is yet to be defined but will comprise performance evaluation
 for both the network and display aspects of the software.
\end_layout

\begin_layout Standard
Should the original specification be delivered and evaluated with time remaining
, additional goals and investigations should be examined.
 Initially, aspects already completed should be investigated for further
 refinement, namely the control scheme as mentioned above.
 
\end_layout

\begin_layout Standard
When considering the design principle of network and presentation separation
 in combination with the relevance of the technology to the spaces of AR
 and VR, an interesting analysis could be made into the applicability of
 multi-source network developments to additional display methods.
 Mobile AR and 
\noun on
Hololens
\noun default
 display for 
\noun on
LiveScan
\noun default
 have both been demonstrated and either could prove interesting when considered
 in a multi-source context.
\end_layout

\begin_layout Subsection
Final Report Structure
\end_layout

\begin_layout Standard
The final report for the project will, to an extent, be based off of this
 piece in terms of structure and content.
 The literature review will be expanded to include an investigation of simultane
ous video streaming in order to contextualise the coming work on the network
 layer of the application suite.
\end_layout

\begin_layout Standard
A results section will describe the quantitative and qualitative results
 of the final product, these will then be evaluated and discussed in the
 discussion section.
 At this point a discussion can also be had about further developments that
 can be made, both to the 
\noun on
LiveScan
\noun default
 codebase and the multi-source version presented by this project.
 The structure will be as follows,
\end_layout

\begin_layout Itemize
Title Page
\end_layout

\begin_layout Itemize
Abstract
\end_layout

\begin_layout Itemize
Table of Contents
\end_layout

\begin_deeper
\begin_layout Itemize
Including lists of figures, code listings
\end_layout

\end_deeper
\begin_layout Itemize
Acknowledgements
\end_layout

\begin_layout Itemize
Introduction
\end_layout

\begin_layout Itemize
Literature Review
\end_layout

\begin_layout Itemize
Methodology and Developments
\end_layout

\begin_layout Itemize
Results
\end_layout

\begin_layout Itemize
Discussion
\end_layout

\begin_deeper
\begin_layout Itemize
A description of possible further developments
\end_layout

\end_deeper
\begin_layout Itemize
Conclusion
\end_layout

\begin_layout Itemize
References
\end_layout

\begin_layout Itemize
Appendices
\end_layout

\begin_layout Section
Summary
\end_layout

\begin_layout Standard
Within this piece the process of extending the 
\noun on
LiveScan3D
\noun default
 software to include multi-source holoportation has been introduced.
 The use of such a system has many applications from those inherited from
 traditional 2D video such as conference calls to new utilisations that
 are wholly unique to the environment.
\end_layout

\begin_layout Standard
The literature review contextualises the 
\noun on
LiveScan
\noun default
 suite and the wider spaces of AR, VR, 3D video and multi-source holoportation
 itself.
 Previous examples of holoportation are described and their aims of achieving
 telepresence are discussed.
\end_layout

\begin_layout Standard
The current state of the project is laid out showing good progress through
 the required areas of development.
 Of these areas of concern, the display element has been extended in order
 to allow the rendering of multiple environments simultaneously with a dynamic
 sub-system of geometric transformations.
 The transformations are responsive to user input allowing arbitrary placement
 and orientation of individual sources within the display space.
 While this control interface allows free movement in the most naturally
 traversed axes it could use some additional tuning to make it feel more
 intuitive.
\end_layout

\begin_layout Standard
The next steps for the project leading up to its completion are presented,
 the initial and current plans for the remaining work is described and additiona
l stretch goals are defined for any additional time.
 How the work will be presented in a final report is also described.
\end_layout

\begin_layout Section
Conclusions
\end_layout

\begin_layout Standard
Holoportation and multi-source configurations thereof are important technologies
 for cross reality experiences with broad appeal to many applications.
 The use of consumer hardware, specifically the 
\noun on
Kinect
\noun default
, have accelerated the space.
\end_layout

\begin_layout Standard
At roughly halfway through the time allowed for this project the native
 display has successfully been extended to meet the deliverable specification.
 This has resulted in the 
\noun on
OpenGL
\noun default
 window being capable of simultaneously rendering multiple sources with
 arbitrary placement and orientation within the display space.
\end_layout

\begin_layout Standard
From this point the network layer of the suite will be developed to also
 match the specification, allowing connected clients to be grouped into
 sources for polling and processing.
\end_layout

\begin_layout Standard
Following the development of the two, testing methodologies will be defined
 and carried out to gather quantitative results for the final product.
 A final report on the results will be available in May 2020.
\end_layout

\begin_layout Standard
\begin_inset Newpage newpage
\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset bibtex
LatexCommand bibtex
btprint "btPrintCited"
bibfiles "/home/andy/uni/dissertation/references"
options "bibtotoc"

\end_inset


\end_layout

\begin_layout Section
\start_of_appendix
Existing Data Structures
\begin_inset CommandInset label
LatexCommand label
name "sec:Existing-Data-Structures"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset include
LatexCommand lstinputlisting
filename "../snippets/point2f.cs"
lstparams "language={[Sharp]C},caption={Cartesian coordinate in 2 dimensions}"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset include
LatexCommand lstinputlisting
filename "../snippets/point3f.cs"
lstparams "language={[Sharp]C},caption={Cartesian coordinate in 3 dimensions}"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset include
LatexCommand lstinputlisting
filename "../snippets/affinetransform.cs"
lstparams "language={[Sharp]C},caption={Affine transformation matrix with translation}"

\end_inset


\end_layout

\begin_layout Section
New Data Structures
\end_layout

\begin_layout Subsection
Frame
\begin_inset CommandInset label
LatexCommand label
name "subsec:Frame"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset include
LatexCommand lstinputlisting
filename "../snippets/frame.cs"
lstparams "language={[Sharp]C},caption={Point cloud with Client ID}"

\end_inset


\end_layout

\end_body
\end_document