849 lines
16 KiB
Plaintext
849 lines
16 KiB
Plaintext
#LyX 2.3 created this file. For more info see http://www.lyx.org/
|
|
\lyxformat 544
|
|
\begin_document
|
|
\begin_header
|
|
\save_transient_properties true
|
|
\origin unavailable
|
|
\textclass article
|
|
\use_default_options true
|
|
\begin_modules
|
|
customHeadersFooters
|
|
minimalistic
|
|
todonotes
|
|
\end_modules
|
|
\maintain_unincluded_children false
|
|
\language english
|
|
\language_package default
|
|
\inputencoding auto
|
|
\fontencoding global
|
|
\font_roman "default" "default"
|
|
\font_sans "default" "default"
|
|
\font_typewriter "default" "default"
|
|
\font_math "auto" "auto"
|
|
\font_default_family default
|
|
\use_non_tex_fonts false
|
|
\font_sc false
|
|
\font_osf false
|
|
\font_sf_scale 100 100
|
|
\font_tt_scale 100 100
|
|
\use_microtype false
|
|
\use_dash_ligatures true
|
|
\graphics default
|
|
\default_output_format default
|
|
\output_sync 0
|
|
\bibtex_command biber
|
|
\index_command default
|
|
\paperfontsize default
|
|
\spacing single
|
|
\use_hyperref false
|
|
\pdf_title "Holoportation"
|
|
\pdf_author "Andy Pack"
|
|
\pdf_subject "The use of Kinect cameras to stream 3D video from client to server"
|
|
\pdf_bookmarks true
|
|
\pdf_bookmarksnumbered false
|
|
\pdf_bookmarksopen false
|
|
\pdf_bookmarksopenlevel 1
|
|
\pdf_breaklinks false
|
|
\pdf_pdfborder false
|
|
\pdf_colorlinks false
|
|
\pdf_backref false
|
|
\pdf_pdfusetitle true
|
|
\papersize default
|
|
\use_geometry true
|
|
\use_package amsmath 1
|
|
\use_package amssymb 1
|
|
\use_package cancel 1
|
|
\use_package esint 1
|
|
\use_package mathdots 1
|
|
\use_package mathtools 1
|
|
\use_package mhchem 1
|
|
\use_package stackrel 1
|
|
\use_package stmaryrd 1
|
|
\use_package undertilde 1
|
|
\cite_engine biblatex
|
|
\cite_engine_type authoryear
|
|
\biblio_style plain
|
|
\biblatex_bibstyle ieee
|
|
\biblatex_citestyle ieee
|
|
\use_bibtopic false
|
|
\use_indices false
|
|
\paperorientation portrait
|
|
\suppress_date true
|
|
\justification true
|
|
\use_refstyle 1
|
|
\use_minted 0
|
|
\index Index
|
|
\shortcut idx
|
|
\color #008000
|
|
\end_index
|
|
\leftmargin 2cm
|
|
\topmargin 2cm
|
|
\rightmargin 2cm
|
|
\bottommargin 2cm
|
|
\secnumdepth 3
|
|
\tocdepth 3
|
|
\paragraph_separation indent
|
|
\paragraph_indentation default
|
|
\is_math_indent 0
|
|
\math_numbering_side default
|
|
\quotes_style english
|
|
\dynamic_quotes 0
|
|
\papercolumns 1
|
|
\papersides 1
|
|
\paperpagestyle fancy
|
|
\bullet 1 0 9 -1
|
|
\tracking_changes false
|
|
\output_changes false
|
|
\html_math_output 0
|
|
\html_css_as_file 0
|
|
\html_be_strict false
|
|
\end_header
|
|
|
|
\begin_body
|
|
|
|
\begin_layout Title
|
|
Multi-Source Holoportation
|
|
\end_layout
|
|
|
|
\begin_layout Author
|
|
Andy Pack / 6420013
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\align center
|
|
Mid-Term Report
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Newpage newpage
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Abstract
|
|
abstract
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset CommandInset toc
|
|
LatexCommand tableofcontents
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout List of TODOs
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Newpage newpage
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Right Footer
|
|
Andy Pack / 6420013
|
|
\end_layout
|
|
|
|
\begin_layout Left Footer
|
|
January 2020
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Introduction
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The aim of this project is to develop a piece of software capable of supporting
|
|
multi-source holoportation (hologram teleportation) using the
|
|
\emph on
|
|
\noun on
|
|
LiveScan3D
|
|
\emph default
|
|
\noun default
|
|
|
|
\begin_inset CommandInset citation
|
|
LatexCommand cite
|
|
key "livescan3d"
|
|
literal "false"
|
|
|
|
\end_inset
|
|
|
|
suite of software as a base.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
As the spaces of augmented and virtual reality become commonplace and mature,
|
|
the ability to capture and stream 3D renderings of objects and people over
|
|
the internet using consumer-grade hardware has many possible applications.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
suite uses
|
|
\noun on
|
|
Xbox Kinect
|
|
\noun default
|
|
cameras to capture and stream 3D renders of objects from one or many angles
|
|
simultaneously however the destination server is only able to process and
|
|
reconstruct one object or surroundings at a time.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The capability to concurrently receive and reconstruct streams of different
|
|
objects further broadens the landscape of possible applications, analogous
|
|
to the movement from 1-to-1 phone calls to conference calling.
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Literature Review
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Augmented and Virtual Reality
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Holoportation
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Kinect
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
LiveScan3D
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
is a suite of software developed by Marek Kowalski, Jacek Naruniec and
|
|
Michal Daniluk of the Warsaw University of Technology in 2015
|
|
\begin_inset CommandInset citation
|
|
LatexCommand cite
|
|
key "livescan3d"
|
|
literal "false"
|
|
|
|
\end_inset
|
|
|
|
.
|
|
The suite utilises the
|
|
\noun on
|
|
Xbox Kinect
|
|
\noun default
|
|
camera to record and transmit 3D renders over an IP network.
|
|
A server which can manage multiple clients processes, reconstructs and
|
|
displays the renderings in real-time.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
LiveScan Client
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
LiveScan Server
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The server component of the
|
|
\noun on
|
|
LiveScan
|
|
\noun default
|
|
suite is responsible for managing and receiving 3D renders from connected
|
|
clients.
|
|
These renderings are reconstructed in an
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window, the structure of the LiveScan server can be seen in figure
|
|
\begin_inset CommandInset ref
|
|
LatexCommand ref
|
|
reference "fig:server-structure"
|
|
plural "false"
|
|
caps "false"
|
|
noprefix "false"
|
|
|
|
\end_inset
|
|
|
|
.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Float figure
|
|
wide false
|
|
sideways false
|
|
status open
|
|
|
|
\begin_layout Plain Layout
|
|
\align center
|
|
\begin_inset Graphics
|
|
filename ../media/initial-state.png
|
|
lyxscale 30
|
|
width 50col%
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
\begin_inset Caption Standard
|
|
|
|
\begin_layout Plain Layout
|
|
Initial structure of the
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
server
|
|
\begin_inset CommandInset label
|
|
LatexCommand label
|
|
name "fig:server-structure"
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The
|
|
\noun on
|
|
KinectServer
|
|
\noun default
|
|
is responsible for the network layer of the program, managing client connection
|
|
s via
|
|
\noun on
|
|
KinectSocket
|
|
\noun default
|
|
s and frame reception.
|
|
Received frames in the form of lists of vertices, RGB values, camera poses
|
|
and bodies override shared variables between the main window and the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Multi-View Configurations
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Current Work
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
As of January 2020 the method for displaying renderings, the server's
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window, has been modified such that it can construct and render point clouds
|
|
from multiple sources.
|
|
In doing so a sub-system of geometric transformations has been included
|
|
such that the renders of individual sources are arranged coherently within
|
|
the space when reconstructed.
|
|
These default arrangements can be overridden with keyboard controls allowing
|
|
arbitrary placement and rotation of separate sources within the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window's co-ordinate space.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Geometric Transformations
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
Within the
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
server source code are utility structures and classes which were extended
|
|
in order to develop a wider geometric manipulation system.
|
|
Structures defining points in both 3D and 2D space called
|
|
\noun on
|
|
Point3f
|
|
\noun default
|
|
and
|
|
\noun on
|
|
Point2f
|
|
\noun default
|
|
respectively are used in drawing skeletons.
|
|
There is also a class defining an affine transformation.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
An affine transformation is a family of geometric transformations that preserve
|
|
parallel lines within geometric spaces.
|
|
Some examples of affine transformations include scaling, reflection, rotation,
|
|
translation and shearing.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The class definition is made up of a three-by-three transformation matrix
|
|
and single 3D vector for translation and is used for both camera poses
|
|
and world transformations.
|
|
A camera pose is the affine transformation defining the position and orientatio
|
|
n of the
|
|
\noun on
|
|
Kinect
|
|
\noun default
|
|
camera when drawn in the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
space.
|
|
The world transformations are used when using multiple sensors simultaneously.
|
|
When completing the calibration process, the origin of the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
space shifts from being the position of the single
|
|
\noun on
|
|
Kinect
|
|
\noun default
|
|
sensor to being the calibration markers that each camera now orbits.
|
|
The server, however, still receives renders from each sensor defined by
|
|
their own Euclidean space and as such the server must transform each view
|
|
into a composite one.
|
|
The world transforms define the transformations for each sensor that correctly
|
|
construct a calibrated 3D render.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
When considering how each source's render would be arranged in the space
|
|
the use of this class definition of affine transformations was extended.
|
|
As the use of the class is fairly limited within the base source code,
|
|
some utility classes and functions were required in order to fully maximise
|
|
their effectiveness.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The
|
|
\noun on
|
|
Transformer
|
|
\noun default
|
|
class has static methods to apply
|
|
\noun on
|
|
AffineTransform
|
|
\noun default
|
|
s to both
|
|
\noun on
|
|
Point3f
|
|
\noun default
|
|
structures and raw vertices when received from
|
|
\noun on
|
|
LiveScan
|
|
\noun default
|
|
clients.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
It also has static methods to generate affine transformations for rotations
|
|
in each axis given an arbitrary angle.
|
|
This provided a foundation on which to define how the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
space would arrange separate sources within it's combined co-ordinate space.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Separation of Network and Presentation Layer
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
During initial testing frames received from a live sensor were intercepted
|
|
and serialized to XML files in local storage.
|
|
These frames were loaded back as the server started and the values were
|
|
merged with those received live before display.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The composite frame can be seen in figure
|
|
\begin_inset CommandInset ref
|
|
LatexCommand ref
|
|
reference "fig:Initial-composite-frame"
|
|
plural "false"
|
|
caps "false"
|
|
noprefix "false"
|
|
|
|
\end_inset
|
|
|
|
.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Float figure
|
|
wide false
|
|
sideways false
|
|
status open
|
|
|
|
\begin_layout Plain Layout
|
|
\align center
|
|
\begin_inset Graphics
|
|
filename ../media/pretransform.jpg
|
|
lyxscale 10
|
|
width 50col%
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
\begin_inset Caption Standard
|
|
|
|
\begin_layout Plain Layout
|
|
Initial composite testing frame
|
|
\begin_inset CommandInset label
|
|
LatexCommand label
|
|
name "fig:Initial-composite-frame"
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The objects can be seen to be occupying the same space due to their similar
|
|
positions in the frame.
|
|
This is not a sufficient solution for displaying separate sources and so
|
|
geometric transformations like those mentioned above were employed.
|
|
This can be seen in figure
|
|
\begin_inset CommandInset ref
|
|
LatexCommand ref
|
|
reference "fig:Initial-testing-layout"
|
|
plural "false"
|
|
caps "false"
|
|
noprefix "false"
|
|
|
|
\end_inset
|
|
|
|
.
|
|
A rotation of 180° in the
|
|
\begin_inset Formula $y$
|
|
\end_inset
|
|
|
|
axis pivoted the frames such that they faced those being received live,
|
|
the results can be seen in figure
|
|
\begin_inset CommandInset ref
|
|
LatexCommand ref
|
|
reference "fig:180-degree-rotation"
|
|
plural "false"
|
|
caps "false"
|
|
noprefix "false"
|
|
|
|
\end_inset
|
|
|
|
.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Float figure
|
|
wide false
|
|
sideways false
|
|
status open
|
|
|
|
\begin_layout Plain Layout
|
|
\align center
|
|
\begin_inset Graphics
|
|
filename ../media/local-testing.png
|
|
lyxscale 30
|
|
width 70col%
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
\begin_inset Caption Standard
|
|
|
|
\begin_layout Plain Layout
|
|
Initial testing process transforming frames loaded from local storage
|
|
\begin_inset CommandInset label
|
|
LatexCommand label
|
|
name "fig:Initial-testing-layout"
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Float figure
|
|
wide false
|
|
sideways false
|
|
status open
|
|
|
|
\begin_layout Plain Layout
|
|
\align center
|
|
\begin_inset Graphics
|
|
filename ../media/180flip.jpg
|
|
lyxscale 10
|
|
width 50col%
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
\begin_inset Caption Standard
|
|
|
|
\begin_layout Plain Layout
|
|
Composite frame following 180° rotation of recorded frame in
|
|
\begin_inset Formula $y$
|
|
\end_inset
|
|
|
|
axis
|
|
\begin_inset CommandInset label
|
|
LatexCommand label
|
|
name "fig:180-degree-rotation"
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
At this point it was noted that transforming and arranging figures within
|
|
the main window before passing the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window a complete point cloud spreads responsibility for the display process
|
|
logic to the main window.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
is capable of supporting more display methods than just the native
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
implementation with versions available for both
|
|
\noun on
|
|
Microsoft Hololens
|
|
\noun default
|
|
and Mobile AR applications.
|
|
Therefore when designing the multi-source capabilities the separation of
|
|
logic between the network and presentation layer will be important.
|
|
The way in which the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window arranges the figures within should be defined by the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window.
|
|
The network layer should be display agnostic and not make assumptions about
|
|
how the display will process figures.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
In order to follow this design the transformations were moved to instead
|
|
occur within the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window class.
|
|
To allow this the shared variables between the
|
|
\noun on
|
|
MainWindow
|
|
\noun default
|
|
and
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
were changed.
|
|
The Frame structure was defined to wrap an individual point cloud with
|
|
a client ID to allow differentiation.
|
|
The structure holds fields for each of the lists previously shared between
|
|
the two objects including a list of vertices or co-ordinates and the RGB
|
|
values for each as well as the camera poses and bodies.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The original
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
cleared each of these variables for each newly retrieved frame, when moving
|
|
to a multi-source architecture the ability to individually update source
|
|
point clouds was required.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
To accomplish this a dictionary was used as the shared variable with each
|
|
clients frame being keyed by it's client ID.
|
|
In doing so only one frame per client is kept and each new frame overrides
|
|
the last.
|
|
During rendering the dictionary is iterated through and each point cloud
|
|
combined.
|
|
Before combination a client specific transformation is retrieved from an
|
|
instance of the
|
|
\noun on
|
|
DisplayFrameTransformer
|
|
\noun default
|
|
class.
|
|
This object is a member of the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window and is responsible for defining the orientation and position of
|
|
each point cloud.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
DisplayFrameTransformer
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Float figure
|
|
wide false
|
|
sideways false
|
|
status open
|
|
|
|
\begin_layout Plain Layout
|
|
\align center
|
|
\begin_inset Graphics
|
|
filename ../media/december-state.png
|
|
lyxscale 30
|
|
width 60col%
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
\begin_inset Caption Standard
|
|
|
|
\begin_layout Plain Layout
|
|
Current state of LiveScan server structure with
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window-based transformer
|
|
\begin_inset CommandInset label
|
|
LatexCommand label
|
|
name "fig:current-state-diagram"
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Future Work
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Summary
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Conclusions
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Newpage pagebreak
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\start_of_appendix
|
|
\begin_inset CommandInset bibtex
|
|
LatexCommand bibtex
|
|
btprint "btPrintCited"
|
|
bibfiles "references"
|
|
options "bibtotoc"
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_body
|
|
\end_document
|