508 lines
10 KiB
Plaintext
508 lines
10 KiB
Plaintext
#LyX 2.3 created this file. For more info see http://www.lyx.org/
|
|
\lyxformat 544
|
|
\begin_document
|
|
\begin_header
|
|
\save_transient_properties true
|
|
\origin unavailable
|
|
\textclass article
|
|
\use_default_options true
|
|
\begin_modules
|
|
customHeadersFooters
|
|
minimalistic
|
|
todonotes
|
|
\end_modules
|
|
\maintain_unincluded_children false
|
|
\language english
|
|
\language_package default
|
|
\inputencoding auto
|
|
\fontencoding global
|
|
\font_roman "default" "default"
|
|
\font_sans "default" "default"
|
|
\font_typewriter "default" "default"
|
|
\font_math "auto" "auto"
|
|
\font_default_family default
|
|
\use_non_tex_fonts false
|
|
\font_sc false
|
|
\font_osf false
|
|
\font_sf_scale 100 100
|
|
\font_tt_scale 100 100
|
|
\use_microtype false
|
|
\use_dash_ligatures true
|
|
\graphics default
|
|
\default_output_format default
|
|
\output_sync 0
|
|
\bibtex_command biber
|
|
\index_command default
|
|
\paperfontsize default
|
|
\spacing single
|
|
\use_hyperref false
|
|
\pdf_title "Holoportation"
|
|
\pdf_author "Andy Pack"
|
|
\pdf_subject "The use of Kinect cameras to stream 3D video from client to server"
|
|
\pdf_bookmarks true
|
|
\pdf_bookmarksnumbered false
|
|
\pdf_bookmarksopen false
|
|
\pdf_bookmarksopenlevel 1
|
|
\pdf_breaklinks false
|
|
\pdf_pdfborder false
|
|
\pdf_colorlinks false
|
|
\pdf_backref false
|
|
\pdf_pdfusetitle true
|
|
\papersize default
|
|
\use_geometry true
|
|
\use_package amsmath 1
|
|
\use_package amssymb 1
|
|
\use_package cancel 1
|
|
\use_package esint 1
|
|
\use_package mathdots 1
|
|
\use_package mathtools 1
|
|
\use_package mhchem 1
|
|
\use_package stackrel 1
|
|
\use_package stmaryrd 1
|
|
\use_package undertilde 1
|
|
\cite_engine biblatex
|
|
\cite_engine_type authoryear
|
|
\biblio_style plain
|
|
\biblatex_bibstyle ieee
|
|
\biblatex_citestyle ieee
|
|
\use_bibtopic false
|
|
\use_indices false
|
|
\paperorientation portrait
|
|
\suppress_date true
|
|
\justification true
|
|
\use_refstyle 1
|
|
\use_minted 0
|
|
\index Index
|
|
\shortcut idx
|
|
\color #008000
|
|
\end_index
|
|
\leftmargin 2cm
|
|
\topmargin 2cm
|
|
\rightmargin 2cm
|
|
\bottommargin 2cm
|
|
\secnumdepth 3
|
|
\tocdepth 3
|
|
\paragraph_separation indent
|
|
\paragraph_indentation default
|
|
\is_math_indent 0
|
|
\math_numbering_side default
|
|
\quotes_style english
|
|
\dynamic_quotes 0
|
|
\papercolumns 1
|
|
\papersides 1
|
|
\paperpagestyle fancy
|
|
\bullet 1 0 9 -1
|
|
\tracking_changes false
|
|
\output_changes false
|
|
\html_math_output 0
|
|
\html_css_as_file 0
|
|
\html_be_strict false
|
|
\end_header
|
|
|
|
\begin_body
|
|
|
|
\begin_layout Title
|
|
Multi-Source Holoportation
|
|
\end_layout
|
|
|
|
\begin_layout Author
|
|
Andy Pack / 6420013
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Newpage newpage
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Abstract
|
|
abstract
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset CommandInset toc
|
|
LatexCommand tableofcontents
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout List of TODOs
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Newpage newpage
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Right Footer
|
|
Andy Pack / 6420013
|
|
\end_layout
|
|
|
|
\begin_layout Left Footer
|
|
January 2020
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Introduction
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The aim of this project is to develop a piece of software capable of supporting
|
|
multi-source holoportation (hologram teleportation) using the
|
|
\emph on
|
|
\noun on
|
|
LiveScan3D
|
|
\emph default
|
|
\noun default
|
|
|
|
\begin_inset CommandInset citation
|
|
LatexCommand cite
|
|
key "livescan3d"
|
|
literal "false"
|
|
|
|
\end_inset
|
|
|
|
suite of software as a base.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
As the spaces of augmented and virtual reality become commonplace and mature,
|
|
the ability to capture and stream 3D renderings of objects and people over
|
|
the internet using consumer-grade hardware has many possible applications.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
suite uses
|
|
\noun on
|
|
Xbox Kinect
|
|
\noun default
|
|
cameras to capture and stream 3D renders of objects from one or many angles
|
|
simultaneously however the destination server is only able to process and
|
|
reconstruct one object or surroundings at a time.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The capability to concurrently receive and reconstruct streams of different
|
|
objects further broadens the landscape of possible applications, analogous
|
|
to the movement from 1-to-1 phone calls to conference calling.
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Literature Review
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Holoportation
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Augmented and Virtual Reality
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Kinect
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
LiveScan3D
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
is a suite of software developed by Marek Kowalski, Jacek Naruniec and
|
|
Michal Daniluk of the Warsaw University of Technology in 2015
|
|
\begin_inset CommandInset citation
|
|
LatexCommand cite
|
|
key "livescan3d"
|
|
literal "false"
|
|
|
|
\end_inset
|
|
|
|
.
|
|
The suite utilises the
|
|
\noun on
|
|
Xbox Kinect
|
|
\noun default
|
|
camera to record and transmit 3D renders over an IP network.
|
|
A server which can manage multiple clients processes, reconstructs and
|
|
displays the renderings in real-time.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
LiveScan Client
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
LiveScan Server
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The server component of the
|
|
\noun on
|
|
LiveScan
|
|
\noun default
|
|
suite is responsible for managing and receiving 3D renders from connected
|
|
clients.
|
|
These renderings are reconstructed in an
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window, the structure of the LiveScan server can be seen in figure
|
|
\begin_inset CommandInset ref
|
|
LatexCommand ref
|
|
reference "fig:server-structure"
|
|
plural "false"
|
|
caps "false"
|
|
noprefix "false"
|
|
|
|
\end_inset
|
|
|
|
.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Float figure
|
|
wide false
|
|
sideways false
|
|
status open
|
|
|
|
\begin_layout Plain Layout
|
|
\align center
|
|
\begin_inset Graphics
|
|
filename ../initial-state.png
|
|
lyxscale 30
|
|
width 50col%
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Plain Layout
|
|
\begin_inset Caption Standard
|
|
|
|
\begin_layout Plain Layout
|
|
Initial structure of the
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
server
|
|
\begin_inset CommandInset label
|
|
LatexCommand label
|
|
name "fig:server-structure"
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The
|
|
\noun on
|
|
KinectServer
|
|
\noun default
|
|
is responsible for the network layer of the program, managing client connection
|
|
s via
|
|
\noun on
|
|
KinectSocket
|
|
\noun default
|
|
s and frame reception.
|
|
Received frames in the form of lists of vertices, RGB values, camera poses
|
|
and bodies override shared variables between the main window and the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Multi-View Configurations
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Current Work
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
As of January 2020 the method for displaying renderings, the server's
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window, has been modified such that it can construct and render point clouds
|
|
from multiple sources.
|
|
In doing so a sub-system of geometric transformations has been included
|
|
such that the renders of individual sources are arranged coherently within
|
|
the space when reconstructed.
|
|
These default arrangements can be overridden using keyboard controls to
|
|
allow arbitrary placement and rotation of separate sources within the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
window's co-ordinate space.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Geometric Transformations
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
Within the
|
|
\noun on
|
|
LiveScan3D
|
|
\noun default
|
|
server source code are utility structures and classes which were extended
|
|
in order to develop a wider geometric manipulation system.
|
|
Structures defining points in both 3D and 2D space called
|
|
\noun on
|
|
Point3f
|
|
\noun default
|
|
and
|
|
\noun on
|
|
Point2f
|
|
\noun default
|
|
respectively are used in drawing skeletons.
|
|
There is also a class defining an affine transformation.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
An affine transformation is a family of geometric transformations that preserve
|
|
parallel lines within geometric spaces.
|
|
Some examples of affine transformations include scaling, reflection, rotation,
|
|
translation and shearing.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The class definition is made up of a three-by-three transformation matrix
|
|
and single 3D vector for translation.
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The class is used for both camera poses and world transformations.
|
|
A camera pose is the affine transformation defining the position and orientatio
|
|
n of the
|
|
\noun on
|
|
Kinect
|
|
\noun default
|
|
camera when drawn in the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
space.
|
|
The world transformations are used when using multiple sensors simultaneously.
|
|
When completing the calibration process, the origin of the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
space shifts from being the position of the single
|
|
\noun on
|
|
Kinect
|
|
\noun default
|
|
sensor to being the calibration markers that each camera now orbits.
|
|
The server, however, still receives renders from each sensor defined by
|
|
their own Euclidean space and as such the server must transform each view
|
|
into a composite one.
|
|
The world transforms define the transformations for each sensor that correctly
|
|
construct a calibrated 3D render.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
When considering how each source's render would be arranged in the space
|
|
the use of this class definition of affine transformations was extended.
|
|
As the use of the class is fairly limited within the base source code,
|
|
some utility classes and functions were required in order to fully maximise
|
|
their effectiveness.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
The
|
|
\noun on
|
|
Transformer
|
|
\noun default
|
|
class has static methods to apply
|
|
\noun on
|
|
AffineTransform
|
|
\noun default
|
|
s to both
|
|
\noun on
|
|
Point3f
|
|
\noun default
|
|
structures and raw vertices when received from
|
|
\noun on
|
|
LiveScan
|
|
\noun default
|
|
clients.
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
It also has static methods to generate affine transformations for rotations
|
|
in each axis given an arbitrary angle.
|
|
This provided a foundation on which to define how the
|
|
\noun on
|
|
OpenGL
|
|
\noun default
|
|
space would arrange separate sources within it's combined co-ordinate space.
|
|
\end_layout
|
|
|
|
\begin_layout Subsection
|
|
Separation of Network and Display
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Future Work
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Summary
|
|
\end_layout
|
|
|
|
\begin_layout Section
|
|
Conclusions
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\begin_inset Newpage pagebreak
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\begin_layout Standard
|
|
\start_of_appendix
|
|
\begin_inset CommandInset bibtex
|
|
LatexCommand bibtex
|
|
btprint "btPrintCited"
|
|
bibfiles "references"
|
|
options "bibtotoc"
|
|
|
|
\end_inset
|
|
|
|
|
|
\end_layout
|
|
|
|
\end_body
|
|
\end_document
|