2019-12-03 10:14:59 +00:00
|
|
|
#LyX 2.3 created this file. For more info see http://www.lyx.org/
|
|
|
|
|
\lyxformat 544
|
|
|
|
|
\begin_document
|
|
|
|
|
\begin_header
|
|
|
|
|
\save_transient_properties true
|
|
|
|
|
\origin unavailable
|
2019-12-11 20:51:53 +00:00
|
|
|
\textclass article
|
2019-12-03 10:14:59 +00:00
|
|
|
\use_default_options true
|
2019-12-06 16:13:29 +00:00
|
|
|
\begin_modules
|
|
|
|
|
customHeadersFooters
|
|
|
|
|
minimalistic
|
|
|
|
|
todonotes
|
|
|
|
|
\end_modules
|
2019-12-03 10:14:59 +00:00
|
|
|
\maintain_unincluded_children false
|
|
|
|
|
\language english
|
|
|
|
|
\language_package default
|
|
|
|
|
\inputencoding auto
|
|
|
|
|
\fontencoding global
|
2020-01-09 14:10:03 +00:00
|
|
|
\font_roman "utopia" "default"
|
2019-12-03 10:14:59 +00:00
|
|
|
\font_sans "default" "default"
|
|
|
|
|
\font_typewriter "default" "default"
|
|
|
|
|
\font_math "auto" "auto"
|
|
|
|
|
\font_default_family default
|
|
|
|
|
\use_non_tex_fonts false
|
|
|
|
|
\font_sc false
|
|
|
|
|
\font_osf false
|
|
|
|
|
\font_sf_scale 100 100
|
|
|
|
|
\font_tt_scale 100 100
|
|
|
|
|
\use_microtype false
|
|
|
|
|
\use_dash_ligatures true
|
|
|
|
|
\graphics default
|
|
|
|
|
\default_output_format default
|
|
|
|
|
\output_sync 0
|
2019-12-06 16:13:29 +00:00
|
|
|
\bibtex_command biber
|
2019-12-03 10:14:59 +00:00
|
|
|
\index_command default
|
|
|
|
|
\paperfontsize default
|
2020-01-09 19:43:23 +00:00
|
|
|
\spacing onehalf
|
2019-12-03 10:14:59 +00:00
|
|
|
\use_hyperref false
|
2019-12-06 16:13:29 +00:00
|
|
|
\pdf_title "Holoportation"
|
|
|
|
|
\pdf_author "Andy Pack"
|
|
|
|
|
\pdf_subject "The use of Kinect cameras to stream 3D video from client to server"
|
|
|
|
|
\pdf_bookmarks true
|
|
|
|
|
\pdf_bookmarksnumbered false
|
|
|
|
|
\pdf_bookmarksopen false
|
|
|
|
|
\pdf_bookmarksopenlevel 1
|
|
|
|
|
\pdf_breaklinks false
|
|
|
|
|
\pdf_pdfborder false
|
|
|
|
|
\pdf_colorlinks false
|
|
|
|
|
\pdf_backref false
|
|
|
|
|
\pdf_pdfusetitle true
|
2019-12-03 10:14:59 +00:00
|
|
|
\papersize default
|
|
|
|
|
\use_geometry true
|
|
|
|
|
\use_package amsmath 1
|
|
|
|
|
\use_package amssymb 1
|
|
|
|
|
\use_package cancel 1
|
|
|
|
|
\use_package esint 1
|
|
|
|
|
\use_package mathdots 1
|
|
|
|
|
\use_package mathtools 1
|
|
|
|
|
\use_package mhchem 1
|
|
|
|
|
\use_package stackrel 1
|
|
|
|
|
\use_package stmaryrd 1
|
|
|
|
|
\use_package undertilde 1
|
|
|
|
|
\cite_engine biblatex
|
|
|
|
|
\cite_engine_type authoryear
|
|
|
|
|
\biblio_style plain
|
|
|
|
|
\biblatex_bibstyle ieee
|
|
|
|
|
\biblatex_citestyle ieee
|
|
|
|
|
\use_bibtopic false
|
|
|
|
|
\use_indices false
|
|
|
|
|
\paperorientation portrait
|
|
|
|
|
\suppress_date true
|
|
|
|
|
\justification true
|
|
|
|
|
\use_refstyle 1
|
|
|
|
|
\use_minted 0
|
|
|
|
|
\index Index
|
|
|
|
|
\shortcut idx
|
|
|
|
|
\color #008000
|
|
|
|
|
\end_index
|
|
|
|
|
\leftmargin 2cm
|
|
|
|
|
\topmargin 2cm
|
|
|
|
|
\rightmargin 2cm
|
|
|
|
|
\bottommargin 2cm
|
|
|
|
|
\secnumdepth 3
|
|
|
|
|
\tocdepth 3
|
|
|
|
|
\paragraph_separation indent
|
|
|
|
|
\paragraph_indentation default
|
|
|
|
|
\is_math_indent 0
|
|
|
|
|
\math_numbering_side default
|
|
|
|
|
\quotes_style english
|
|
|
|
|
\dynamic_quotes 0
|
|
|
|
|
\papercolumns 1
|
|
|
|
|
\papersides 1
|
2019-12-06 16:13:29 +00:00
|
|
|
\paperpagestyle fancy
|
2019-12-03 10:14:59 +00:00
|
|
|
\bullet 1 0 9 -1
|
|
|
|
|
\tracking_changes false
|
|
|
|
|
\output_changes false
|
|
|
|
|
\html_math_output 0
|
|
|
|
|
\html_css_as_file 0
|
|
|
|
|
\html_be_strict false
|
|
|
|
|
\end_header
|
|
|
|
|
|
|
|
|
|
\begin_body
|
|
|
|
|
|
|
|
|
|
\begin_layout Title
|
2020-01-09 09:37:42 +00:00
|
|
|
|
|
|
|
|
\size giant
|
2019-12-10 20:06:32 +00:00
|
|
|
Multi-Source Holoportation
|
2019-12-03 10:14:59 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Author
|
2020-01-09 09:37:42 +00:00
|
|
|
Andy Pack
|
2019-12-03 10:14:59 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-16 23:57:43 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
\align center
|
2020-01-09 09:37:42 +00:00
|
|
|
|
|
|
|
|
\size largest
|
2019-12-16 23:57:43 +00:00
|
|
|
Mid-Term Report
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 09:37:42 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset VSpace bigskip
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
|
|
|
|
filename ../surreylogo.png
|
|
|
|
|
lyxscale 30
|
|
|
|
|
width 60col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset VSpace vfill
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\align center
|
|
|
|
|
|
|
|
|
|
\size large
|
|
|
|
|
Department of Electrical and Electronic Engineering
|
|
|
|
|
\begin_inset Newline newline
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
Faculty of Engineering and Physical Sciences
|
|
|
|
|
\begin_inset Newline newline
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
University of Surrey
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Newpage newpage
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Abstract
|
2020-01-09 14:10:03 +00:00
|
|
|
The scope and current state of the multi-source holoportation project is
|
|
|
|
|
examined.
|
|
|
|
|
The aim is to take a suite of 3D video capture software and extend it from
|
|
|
|
|
the current capabilities of multiple sensors but a single client to handle
|
|
|
|
|
multiple groups of sensors called sources during frame collection and display.
|
|
|
|
|
Currently the display methods have been extended in line with the specification
|
|
|
|
|
in order to allow simultaneous display and arbitrary real-time placement
|
|
|
|
|
within the display space.
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Abstract
|
|
|
|
|
The future work for the project is described including the current designs
|
2020-01-09 19:43:23 +00:00
|
|
|
these endeavours.
|
2020-01-09 14:10:03 +00:00
|
|
|
The bulk of this remaining work involves developing the network capabilities
|
2020-01-09 19:43:23 +00:00
|
|
|
of the software to accommodate multiple sources.
|
2019-12-03 10:14:59 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset CommandInset toc
|
|
|
|
|
LatexCommand tableofcontents
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
2020-01-09 14:10:03 +00:00
|
|
|
\begin_inset Newpage newpage
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset FloatList figure
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\begin_inset CommandInset toc
|
|
|
|
|
LatexCommand lstlistoflistings
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
2019-12-06 16:13:29 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout List of TODOs
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Newpage newpage
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-13 13:48:47 +00:00
|
|
|
\begin_layout Right Footer
|
|
|
|
|
Andy Pack / 6420013
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Left Footer
|
|
|
|
|
January 2020
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\begin_layout Section
|
|
|
|
|
Introduction
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
The aim of this project is to develop a piece of software capable of supporting
|
|
|
|
|
multi-source holoportation (hologram teleportation) using the
|
|
|
|
|
\emph on
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan3D
|
|
|
|
|
\emph default
|
|
|
|
|
\noun default
|
|
|
|
|
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "livescan3d"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
suite of software as a base.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
2020-01-09 19:43:23 +00:00
|
|
|
As the spaces of augmented and virtual reality become more commonplace and
|
|
|
|
|
mature, the ability to capture and stream 3D renderings of objects and
|
|
|
|
|
people over the internet using consumer-grade hardware has many possible
|
|
|
|
|
applications.
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-05 15:31:33 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
This represents one of the most direct evolutions of traditional video streaming
|
|
|
|
|
when applied to this new technological space.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\begin_layout Standard
|
2020-01-09 19:43:23 +00:00
|
|
|
LiveScan3D is a suite of 3D video software capable of recording and transmitting
|
|
|
|
|
video from client to server for rendering.
|
|
|
|
|
The suite is fast and uses consumer grade hardware for capture in the form
|
|
|
|
|
of
|
2019-12-11 20:51:53 +00:00
|
|
|
\noun on
|
2020-01-09 19:43:23 +00:00
|
|
|
Xbox Kinect
|
|
|
|
|
\noun default
|
|
|
|
|
cameras, it is used in projects at various levels throughout the
|
|
|
|
|
\noun on
|
|
|
|
|
University of Surrey
|
|
|
|
|
\noun default
|
|
|
|
|
and has multiple setups in dedicated laboratory space.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan3D's
|
|
|
|
|
\noun default
|
|
|
|
|
use
|
|
|
|
|
\noun on
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\noun default
|
2020-01-09 19:43:23 +00:00
|
|
|
of
|
2019-12-11 20:51:53 +00:00
|
|
|
\noun on
|
|
|
|
|
Xbox Kinect
|
|
|
|
|
\noun default
|
2020-01-09 19:43:23 +00:00
|
|
|
cameras allows the capture and stream of 3D renders in single or multi-view
|
|
|
|
|
configurations using multiple cameras however the server is only able to
|
|
|
|
|
process and reconstruct one environment at a time.
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The capability to concurrently receive and reconstruct streams of different
|
|
|
|
|
objects further broadens the landscape of possible applications, analogous
|
|
|
|
|
to the movement from 1-to-1 phone calls to conference calling.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\begin_layout Section
|
|
|
|
|
Literature Review
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 09:37:42 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
The significance of the 3D video captured and relayed with the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
suite is closely related to the development of new technologies able to
|
|
|
|
|
immersively display such video content.
|
|
|
|
|
Therefore before discussing the specific extension that this project will
|
|
|
|
|
make to the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
software it is important to contextualise it within the space of 3D video
|
|
|
|
|
capture while also considering it's implications for AR and VR applications.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\begin_layout Subsection
|
2019-12-16 23:57:43 +00:00
|
|
|
Augmented and Virtual Reality
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
2020-01-09 09:37:42 +00:00
|
|
|
Traditional Optical 3D Reconstruction
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-17 16:34:12 +00:00
|
|
|
\begin_layout Subsection
|
2020-01-09 09:37:42 +00:00
|
|
|
Kinect and RGB-D Cameras
|
2019-12-17 16:34:12 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\begin_layout Subsection
|
2020-01-09 09:37:42 +00:00
|
|
|
Holoportation and Telepresence
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The term Holoportation is defined and exemplified in the
|
|
|
|
|
\noun on
|
|
|
|
|
Microsoft Research
|
|
|
|
|
\noun default
|
|
|
|
|
paper
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "holoportation"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
, where an end-to-end pipeline is laid out for the acquisition, transmission
|
|
|
|
|
and display of 3D video facilitating real-time AR and VR experiences.
|
|
|
|
|
The
|
|
|
|
|
\noun on
|
|
|
|
|
Microsoft Research
|
|
|
|
|
\noun default
|
|
|
|
|
paper builds on works such as
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "Immersive-telepresence"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
2 years earlier which describes attempts at achieving
|
|
|
|
|
\begin_inset Quotes eld
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
telepresence
|
|
|
|
|
\begin_inset Quotes erd
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
, a term coined by Marvin Minksy to describe the transparent and intuitive
|
|
|
|
|
remote control of robot arms as if they were the controllers own
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "marvin-minksy"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
The term was broadened by Bill Buxton
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "buxton-telepresence"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
to include the space of telecommunications to describe technology being
|
|
|
|
|
used to make someone feel present in a different environment.
|
|
|
|
|
In the context of holoportation this is through the use of 3D video reconstruct
|
|
|
|
|
ion.
|
|
|
|
|
The aforementioned
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "Immersive-telepresence"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
used 10
|
|
|
|
|
\noun on
|
|
|
|
|
Microsoft Kinect
|
|
|
|
|
\noun default
|
|
|
|
|
cameras to capture a room before virtually reconstructing the models.
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
In service of demonstrating it's applicability to achieving telepresence,
|
|
|
|
|
a figure was isolated from the surroundings and stereoscopically rear-projected
|
|
|
|
|
onto a screen for a single participant, a result of this can be seen in
|
|
|
|
|
figure
|
|
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "fig:stereoscopic"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Float figure
|
|
|
|
|
wide false
|
|
|
|
|
sideways false
|
|
|
|
|
status open
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
|
|
|
|
filename ../media/telepresence-stereoscopic.png
|
|
|
|
|
lyxscale 30
|
|
|
|
|
width 40col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\begin_inset Caption Standard
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
An example of stereoscopic projection of depth aware footage captured during
|
|
|
|
|
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "Immersive-telepresence"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "fig:stereoscopic"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The
|
|
|
|
|
\noun on
|
|
|
|
|
Microsoft Research
|
|
|
|
|
\noun default
|
|
|
|
|
paper demonstrates a system using 8 cameras surrounding a space.
|
|
|
|
|
Each camera captured both Near Infra-Red and colour images to construct
|
|
|
|
|
a colour-depth video stream, .
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
|
|
|
|
Multi-Source Holoportation
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The space of work implementing multi-source holoportation has been explored
|
|
|
|
|
in works such as
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "group-to-group-telepresence"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
in the context of shared architectural design spaces in virtual reality
|
|
|
|
|
similar to a conference call.
|
|
|
|
|
Two groups of people were captured in 3D using clusters of
|
|
|
|
|
\noun on
|
2019-12-11 20:51:53 +00:00
|
|
|
Kinect
|
2020-01-09 09:37:42 +00:00
|
|
|
\noun default
|
|
|
|
|
cameras before having these renders transmitted to the other group.
|
|
|
|
|
Each group reconstructs the other's render for display in virtual reality
|
|
|
|
|
in conjunction with their own.
|
|
|
|
|
In doing so a shared virtual space for the two groups has been created
|
|
|
|
|
and it can be seen to implement the process of holoportation.
|
|
|
|
|
The shared architectural design experience is emergent of the semantics
|
|
|
|
|
of the virtual space where a World in Miniature (WIM) metaphor is used.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsubsection
|
|
|
|
|
Worlds in Miniature
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
2020-01-09 14:10:03 +00:00
|
|
|
The Worlds in Miniature is described in
|
2020-01-09 09:37:42 +00:00
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "wim"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
as a set of interfaces between the user and the virtual space they experience
|
|
|
|
|
using tactile and visual tools.
|
|
|
|
|
The interface involves providing the user with a miniature render of the
|
2020-01-09 14:10:03 +00:00
|
|
|
world they are inhabiting that can interacted with in order to affect the
|
|
|
|
|
full scale environment around them.
|
2020-01-09 09:37:42 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
This navigation tool maps well to the architecture groupware structure of
|
|
|
|
|
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "group-to-group-telepresence"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
, an image captured during the work can be seen in figure
|
|
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "fig:World-in-Miniature-group-by-group"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Float figure
|
|
|
|
|
wide false
|
|
|
|
|
sideways false
|
|
|
|
|
status open
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
|
|
|
|
filename ../media/group-by-group.png
|
|
|
|
|
lyxscale 30
|
|
|
|
|
width 50col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\begin_inset Caption Standard
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
World in Miniature render demonstrated in a multi-source holoporation context
|
|
|
|
|
during
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "group-to-group-telepresence"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "fig:World-in-Miniature-group-by-group"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-10 20:06:32 +00:00
|
|
|
\begin_layout Section
|
|
|
|
|
LiveScan3D
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan3D
|
|
|
|
|
\noun default
|
|
|
|
|
is a suite of software developed by Marek Kowalski, Jacek Naruniec and
|
|
|
|
|
Michal Daniluk of the Warsaw University of Technology in 2015
|
|
|
|
|
\begin_inset CommandInset citation
|
|
|
|
|
LatexCommand cite
|
|
|
|
|
key "livescan3d"
|
|
|
|
|
literal "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
The suite utilises the
|
|
|
|
|
\noun on
|
|
|
|
|
Xbox Kinect
|
|
|
|
|
\noun default
|
2019-12-19 01:12:54 +00:00
|
|
|
v2 camera to record and transmit 3D renders over an IP network.
|
2020-01-05 15:31:33 +00:00
|
|
|
A server can manage multiple clients simultaneously and is responsible
|
|
|
|
|
for processing, reconstructing and displaying the renderings in real-time.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
These renderings take the form of a point cloud, a collection of 3D co-ordinates
|
|
|
|
|
indicating the position of each voxel (3D pixel) and it's associated RGB
|
|
|
|
|
colour value.
|
|
|
|
|
As a result of it's analogous nature to a traditional frame of 2D video,
|
|
|
|
|
the terms
|
|
|
|
|
\begin_inset Quotes eld
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
render
|
|
|
|
|
\begin_inset Quotes erd
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
,
|
|
|
|
|
\begin_inset Quotes eld
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
point cloud
|
|
|
|
|
\begin_inset Quotes erd
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
and
|
|
|
|
|
\begin_inset Quotes eld
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
frame
|
|
|
|
|
\begin_inset Quotes erd
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
are used interchangeably from here.
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
2019-12-19 01:12:54 +00:00
|
|
|
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
Client
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
Client is responsible for interfacing with the
|
|
|
|
|
\noun on
|
|
|
|
|
Kinect
|
|
|
|
|
\noun default
|
|
|
|
|
sensor via the
|
|
|
|
|
\noun on
|
|
|
|
|
Kinect
|
|
|
|
|
\noun default
|
|
|
|
|
v2 SDK and transmitting frames to the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
Server.
|
|
|
|
|
Body detection takes place client side, as does calibration when using
|
|
|
|
|
multiple sensors.
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
2019-12-19 01:12:54 +00:00
|
|
|
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
Server
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The server component of the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
suite is responsible for managing and receiving 3D renders from connected
|
|
|
|
|
clients.
|
2019-12-13 13:48:47 +00:00
|
|
|
These renderings are reconstructed in an
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
2019-12-19 01:12:54 +00:00
|
|
|
window, the structure of the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
server can be seen in figure
|
2019-12-13 13:48:47 +00:00
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "fig:server-structure"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Float figure
|
|
|
|
|
wide false
|
|
|
|
|
sideways false
|
|
|
|
|
status open
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
2019-12-16 23:57:43 +00:00
|
|
|
filename ../media/initial-state.png
|
2019-12-13 13:48:47 +00:00
|
|
|
lyxscale 30
|
|
|
|
|
width 50col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\begin_inset Caption Standard
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
2020-01-09 14:10:03 +00:00
|
|
|
Initial architecture of the
|
2019-12-13 13:48:47 +00:00
|
|
|
\noun on
|
|
|
|
|
LiveScan3D
|
|
|
|
|
\noun default
|
|
|
|
|
server
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "fig:server-structure"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The
|
|
|
|
|
\noun on
|
|
|
|
|
KinectServer
|
|
|
|
|
\noun default
|
|
|
|
|
is responsible for the network layer of the program, managing client connection
|
|
|
|
|
s via
|
|
|
|
|
\noun on
|
|
|
|
|
KinectSocket
|
|
|
|
|
\noun default
|
|
|
|
|
s and frame reception.
|
|
|
|
|
Received frames in the form of lists of vertices, RGB values, camera poses
|
|
|
|
|
and bodies override shared variables between the main window and the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
window.
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
2019-12-17 16:34:12 +00:00
|
|
|
Frame Geometry & Multi-View Configurations
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-19 01:12:54 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
When using a single client setup frames are transmitted in their own co-ordinate
|
|
|
|
|
space, the sensor is made the origin with the scene being rendered in front
|
|
|
|
|
of it.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
When using multiple sensors, the server would be unable to combine these
|
|
|
|
|
unique Euclidean spaces without knowledge of the sensors relative positions.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
In order to make a composite frame a calibration process is completed client
|
|
|
|
|
side following instruction by the server.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\begin_layout Section
|
|
|
|
|
Current Work
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 09:37:42 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
The required development to take the existing
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
2020-01-09 14:10:03 +00:00
|
|
|
codebase to the desired multi-source result can be split into two areas
|
|
|
|
|
of concern.
|
2020-01-09 09:37:42 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 14:10:03 +00:00
|
|
|
\begin_layout Labeling
|
|
|
|
|
\labelwidthstring 00.00.0000
|
|
|
|
|
|
|
|
|
|
\series bold
|
|
|
|
|
\emph on
|
|
|
|
|
Network
|
|
|
|
|
\series default
|
|
|
|
|
\emph default
|
|
|
|
|
The network layer of the
|
2020-01-09 09:37:42 +00:00
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
server must be updated in order to accommodate multiple clients logically
|
|
|
|
|
grouped into
|
|
|
|
|
\begin_inset Quotes eld
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
sources
|
|
|
|
|
\begin_inset Quotes erd
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
for which separate frames are collected for display.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 14:10:03 +00:00
|
|
|
\begin_layout Labeling
|
|
|
|
|
\labelwidthstring 00.00.0000
|
|
|
|
|
|
|
|
|
|
\series bold
|
|
|
|
|
\emph on
|
|
|
|
|
Display
|
|
|
|
|
\series default
|
|
|
|
|
\emph default
|
|
|
|
|
Finally the display element of the server should be extended to allow the
|
2020-01-09 09:37:42 +00:00
|
|
|
simultaneous presentation of multiple point clouds.
|
|
|
|
|
These objects should be individually arrangeable in the display space allowing
|
|
|
|
|
both movement and rotation.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
As of January 2020 the method for displaying renderings, the server's
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
window, has been modified such that it can construct and render point clouds
|
|
|
|
|
from multiple sources.
|
2020-01-09 14:10:03 +00:00
|
|
|
To do so a dynamic sub-system of geometric transformations has been written
|
|
|
|
|
in order to coherently arrange sources within the space when reconstructed.
|
2020-01-09 09:37:42 +00:00
|
|
|
The default arrangements can be overridden with keyboard controls facilitating
|
2019-12-16 23:57:43 +00:00
|
|
|
arbitrary placement and rotation of separate sources within the
|
2019-12-11 20:51:53 +00:00
|
|
|
\noun on
|
2019-12-13 13:48:47 +00:00
|
|
|
OpenGL
|
2019-12-11 20:51:53 +00:00
|
|
|
\noun default
|
2019-12-13 13:48:47 +00:00
|
|
|
window's co-ordinate space.
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
|
|
|
|
Geometric Transformations
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Within the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan3D
|
|
|
|
|
\noun default
|
2019-12-13 13:48:47 +00:00
|
|
|
server source code are utility structures and classes which were extended
|
|
|
|
|
in order to develop a wider geometric manipulation system.
|
2020-01-09 09:37:42 +00:00
|
|
|
Structures defining Cartesian coordinates in both 3D and 2D spaces called
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\noun on
|
|
|
|
|
Point3f
|
|
|
|
|
\noun default
|
|
|
|
|
and
|
|
|
|
|
\noun on
|
|
|
|
|
Point2f
|
|
|
|
|
\noun default
|
|
|
|
|
respectively are used in drawing skeletons.
|
2020-01-09 14:10:03 +00:00
|
|
|
There is also a class defining an affine transformation, the definitions
|
|
|
|
|
for all three can be seen in appendix
|
|
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "sec:Existing-Data-Structures"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
2020-01-09 09:37:42 +00:00
|
|
|
Affine transformations are a family of geometric transformations that preserve
|
2019-12-11 20:51:53 +00:00
|
|
|
parallel lines within geometric spaces.
|
|
|
|
|
Some examples of affine transformations include scaling, reflection, rotation,
|
|
|
|
|
translation and shearing.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The class definition is made up of a three-by-three transformation matrix
|
2020-01-09 09:37:42 +00:00
|
|
|
and single 3D vector for translation, within the initial code it is used
|
|
|
|
|
for both camera poses and world transformations.
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
A camera pose is the affine transformation defining the position and orientation
|
|
|
|
|
of the
|
2019-12-11 20:51:53 +00:00
|
|
|
\noun on
|
|
|
|
|
Kinect
|
|
|
|
|
\noun default
|
|
|
|
|
camera when drawn in the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
2020-01-09 09:37:42 +00:00
|
|
|
space as a green cross.
|
2019-12-11 20:51:53 +00:00
|
|
|
The world transformations are used when using multiple sensors simultaneously.
|
|
|
|
|
When completing the calibration process, the origin of the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
space shifts from being the position of the single
|
|
|
|
|
\noun on
|
|
|
|
|
Kinect
|
|
|
|
|
\noun default
|
|
|
|
|
sensor to being the calibration markers that each camera now orbits.
|
|
|
|
|
The server, however, still receives renders from each sensor defined by
|
|
|
|
|
their own Euclidean space and as such the server must transform each view
|
|
|
|
|
into a composite one.
|
|
|
|
|
The world transforms define the transformations for each sensor that correctly
|
|
|
|
|
construct a calibrated 3D render.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
When considering how each source's render would be arranged in the space
|
|
|
|
|
the use of this class definition of affine transformations was extended.
|
|
|
|
|
As the use of the class is fairly limited within the base source code,
|
|
|
|
|
some utility classes and functions were required in order to fully maximise
|
|
|
|
|
their effectiveness.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The
|
|
|
|
|
\noun on
|
|
|
|
|
Transformer
|
|
|
|
|
\noun default
|
|
|
|
|
class has static methods to apply
|
|
|
|
|
\noun on
|
|
|
|
|
AffineTransform
|
|
|
|
|
\noun default
|
|
|
|
|
s to both
|
|
|
|
|
\noun on
|
|
|
|
|
Point3f
|
|
|
|
|
\noun default
|
|
|
|
|
structures and raw vertices when received from
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
clients.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
It also has static methods to generate affine transformations for rotations
|
|
|
|
|
in each axis given an arbitrary angle.
|
|
|
|
|
This provided a foundation on which to define how the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
space would arrange separate sources within it's combined co-ordinate space.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
2019-12-16 23:57:43 +00:00
|
|
|
Separation of Network and Presentation Layer
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
During initial testing frames received from a live sensor were intercepted
|
2020-01-09 14:10:03 +00:00
|
|
|
and serialized to XML in local storage.
|
|
|
|
|
These frames were loaded into memory as the server was started and merged
|
|
|
|
|
with those received live before display.
|
2019-12-16 23:57:43 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The composite frame can be seen in figure
|
|
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "fig:Initial-composite-frame"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Float figure
|
|
|
|
|
wide false
|
|
|
|
|
sideways false
|
|
|
|
|
status open
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
|
|
|
|
filename ../media/pretransform.jpg
|
|
|
|
|
lyxscale 10
|
|
|
|
|
width 50col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\begin_inset Caption Standard
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
2020-01-09 14:10:03 +00:00
|
|
|
Initial multi-source composite testing frame
|
2019-12-16 23:57:43 +00:00
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "fig:Initial-composite-frame"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The objects can be seen to be occupying the same space due to their similar
|
2020-01-09 14:10:03 +00:00
|
|
|
positions in the frame during capture.
|
2019-12-16 23:57:43 +00:00
|
|
|
This is not a sufficient solution for displaying separate sources and so
|
2020-01-09 14:10:03 +00:00
|
|
|
geometric transformations like those mentioned above were employed to separate
|
|
|
|
|
the two.
|
|
|
|
|
The change in software structure at this stage can be seen in figure
|
2019-12-16 23:57:43 +00:00
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "fig:Initial-testing-layout"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
A rotation of 180° in the
|
|
|
|
|
\begin_inset Formula $y$
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
axis pivoted the frames such that they faced those being received live,
|
|
|
|
|
the results can be seen in figure
|
|
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "fig:180-degree-rotation"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Float figure
|
|
|
|
|
wide false
|
|
|
|
|
sideways false
|
|
|
|
|
status open
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
|
|
|
|
filename ../media/local-testing.png
|
|
|
|
|
lyxscale 30
|
|
|
|
|
width 70col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\begin_inset Caption Standard
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
Initial testing process transforming frames loaded from local storage
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "fig:Initial-testing-layout"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Float figure
|
|
|
|
|
wide false
|
|
|
|
|
sideways false
|
|
|
|
|
status open
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
|
|
|
|
filename ../media/180flip.jpg
|
|
|
|
|
lyxscale 10
|
|
|
|
|
width 50col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\begin_inset Caption Standard
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
2020-01-09 14:10:03 +00:00
|
|
|
Composite testing frame following 180° rotation of recorded source in
|
2019-12-16 23:57:43 +00:00
|
|
|
\begin_inset Formula $y$
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
axis
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "fig:180-degree-rotation"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
At this point it was noted that transforming and arranging figures within
|
|
|
|
|
the main window before passing the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
window a complete point cloud spreads responsibility for the display process
|
|
|
|
|
logic to the main window.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan3D
|
|
|
|
|
\noun default
|
|
|
|
|
is capable of supporting more display methods than just the native
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
implementation with versions available for both
|
|
|
|
|
\noun on
|
|
|
|
|
Microsoft Hololens
|
|
|
|
|
\noun default
|
|
|
|
|
and Mobile AR applications.
|
2020-01-09 14:10:03 +00:00
|
|
|
Therefore when designing the multi-source capabilities, the separation
|
|
|
|
|
of logic between the network and presentation layer is important.
|
2019-12-16 23:57:43 +00:00
|
|
|
The way in which the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
2020-01-09 14:10:03 +00:00
|
|
|
window arranges the figures in it's display space should be defined by
|
|
|
|
|
the
|
2019-12-16 23:57:43 +00:00
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
2020-01-09 14:10:03 +00:00
|
|
|
window itself.
|
2019-12-16 23:57:43 +00:00
|
|
|
The network layer should be display agnostic and not make assumptions about
|
|
|
|
|
how the display will process figures.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
In order to follow this design the transformations were moved to instead
|
|
|
|
|
occur within the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
window class.
|
|
|
|
|
To allow this the shared variables between the
|
|
|
|
|
\noun on
|
|
|
|
|
MainWindow
|
|
|
|
|
\noun default
|
|
|
|
|
and
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
were changed.
|
2020-01-09 14:10:03 +00:00
|
|
|
A Frame structure was defined to wrap an individual point cloud with a
|
|
|
|
|
client ID to allow differentiation, the definition can be seen in appendix
|
|
|
|
|
|
|
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "subsec:Frame"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
2019-12-16 23:57:43 +00:00
|
|
|
The structure holds fields for each of the lists previously shared between
|
|
|
|
|
the two objects including a list of vertices or co-ordinates and the RGB
|
|
|
|
|
values for each as well as the camera poses and bodies.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The original
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan3D
|
|
|
|
|
\noun default
|
2020-01-09 14:10:03 +00:00
|
|
|
cleared each of these variables before retrieving a new frame, when moving
|
2019-12-16 23:57:43 +00:00
|
|
|
to a multi-source architecture the ability to individually update source
|
2020-01-09 14:10:03 +00:00
|
|
|
point clouds was noted as being important.
|
|
|
|
|
This would remove blocking the entire display when unable to receive frames
|
|
|
|
|
from a specific client, other clients would still be able to have frames
|
|
|
|
|
updated promptly.
|
2019-12-16 23:57:43 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
To accomplish this a dictionary was used as the shared variable with each
|
|
|
|
|
clients frame being keyed by it's client ID.
|
|
|
|
|
In doing so only one frame per client is kept and each new frame overrides
|
|
|
|
|
the last.
|
|
|
|
|
During rendering the dictionary is iterated through and each point cloud
|
|
|
|
|
combined.
|
|
|
|
|
Before combination a client specific transformation is retrieved from an
|
|
|
|
|
instance of the
|
|
|
|
|
\noun on
|
|
|
|
|
DisplayFrameTransformer
|
|
|
|
|
\noun default
|
|
|
|
|
class.
|
|
|
|
|
This object is a member of the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
window and is responsible for defining the orientation and position of
|
|
|
|
|
each point cloud.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
|
|
|
|
DisplayFrameTransformer
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-17 16:34:12 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
The
|
|
|
|
|
\noun on
|
|
|
|
|
DisplayFrameTransformer
|
|
|
|
|
\noun default
|
|
|
|
|
is responsible for generating transformations for the sources displayed
|
|
|
|
|
within the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
2020-01-09 14:10:03 +00:00
|
|
|
window, a UML diagram for the class can be seen in figure
|
|
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "fig:UML-displayframetransformer"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Float figure
|
|
|
|
|
wide false
|
|
|
|
|
sideways false
|
|
|
|
|
status open
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
|
|
|
|
filename ../media/DisplayFrameTransformer.png
|
|
|
|
|
lyxscale 30
|
|
|
|
|
width 50col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\begin_inset Caption Standard
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
UML diagram for
|
|
|
|
|
\noun on
|
|
|
|
|
DisplayFrameTransformer
|
|
|
|
|
\noun default
|
|
|
|
|
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "fig:UML-displayframetransformer"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
2019-12-17 16:34:12 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Each client is assigned a default transformation which can be overridden
|
|
|
|
|
using keyboard controls.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
2020-01-09 14:10:03 +00:00
|
|
|
Clients are initially arranged in a circle around the origin in the center
|
2019-12-17 16:34:12 +00:00
|
|
|
of the space.
|
|
|
|
|
This is done by retrieving a transformation for a rotation in the
|
|
|
|
|
\begin_inset Formula $y$
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
axis for each client number,
|
|
|
|
|
\begin_inset Formula $n$
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
, using the below,
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Formula
|
|
|
|
|
\[
|
|
|
|
|
\alpha\left(n\right)=\frac{n}{client\:total}\cdotp360\textdegree
|
|
|
|
|
\]
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Similar to the shared variables between the
|
|
|
|
|
\noun on
|
|
|
|
|
MainWindow
|
|
|
|
|
\noun default
|
|
|
|
|
and
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
window, client transformations are stored within a dictionary indexed by
|
|
|
|
|
client ID.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The
|
|
|
|
|
\noun on
|
|
|
|
|
DisplayFrameTransformer
|
|
|
|
|
\noun default
|
|
|
|
|
also has methods to override these initial transforms with the RotateClient()
|
|
|
|
|
and TranslateClient() methods.
|
|
|
|
|
When these methods are called for the first time for a client an object
|
2020-01-09 14:10:03 +00:00
|
|
|
defining the position and rotation is populated using the default rotation.
|
|
|
|
|
From here the presence of a client override results in applied transforms
|
|
|
|
|
being defined by these values as opposed to the default orientation.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
This leaves the current architecture of the server application as described
|
|
|
|
|
in figure
|
|
|
|
|
\begin_inset CommandInset ref
|
|
|
|
|
LatexCommand ref
|
|
|
|
|
reference "fig:current-state-diagram"
|
|
|
|
|
plural "false"
|
|
|
|
|
caps "false"
|
|
|
|
|
noprefix "false"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
.
|
2019-12-17 16:34:12 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-16 23:57:43 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Float figure
|
|
|
|
|
wide false
|
|
|
|
|
sideways false
|
|
|
|
|
status open
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\align center
|
|
|
|
|
\begin_inset Graphics
|
|
|
|
|
filename ../media/december-state.png
|
|
|
|
|
lyxscale 30
|
|
|
|
|
width 60col%
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
\begin_inset Caption Standard
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
2019-12-17 16:34:12 +00:00
|
|
|
Current state of
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
server structure with
|
2019-12-16 23:57:43 +00:00
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
window-based transformer
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "fig:current-state-diagram"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Plain Layout
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
2019-12-11 20:51:53 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 14:10:03 +00:00
|
|
|
\begin_layout Subsection
|
|
|
|
|
Control Scheme
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The movement of objects within the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
space is conducted through keyboard controls.
|
|
|
|
|
While mouse control would fine-grained and intuitive, the axes of motion
|
|
|
|
|
and rotation available to objects makes defining specific keys for each
|
|
|
|
|
more flexible.
|
|
|
|
|
This additionally removes the need to redefine or overload the camera controls.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The
|
|
|
|
|
\begin_inset Quotes eld
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
I
|
|
|
|
|
\begin_inset Quotes erd
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
key is used to cycle through displayed sources, the currently selected
|
|
|
|
|
source is the subject of each of the movement actions.
|
|
|
|
|
Sources are moved across the horizontal plane (
|
|
|
|
|
\family roman
|
|
|
|
|
\series medium
|
|
|
|
|
\shape up
|
|
|
|
|
\size normal
|
|
|
|
|
\emph off
|
|
|
|
|
\bar no
|
|
|
|
|
\strikeout off
|
|
|
|
|
\xout off
|
|
|
|
|
\uuline off
|
|
|
|
|
\uwave off
|
|
|
|
|
\noun off
|
|
|
|
|
\color none
|
|
|
|
|
|
|
|
|
|
\begin_inset Formula $x$
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\family default
|
|
|
|
|
\series default
|
|
|
|
|
\shape default
|
|
|
|
|
\size default
|
|
|
|
|
\emph default
|
|
|
|
|
\bar default
|
|
|
|
|
\strikeout default
|
|
|
|
|
\xout default
|
|
|
|
|
\uuline default
|
|
|
|
|
\uwave default
|
|
|
|
|
\noun default
|
|
|
|
|
\color inherit
|
|
|
|
|
and
|
|
|
|
|
\family roman
|
|
|
|
|
\series medium
|
|
|
|
|
\shape up
|
|
|
|
|
\size normal
|
|
|
|
|
\emph off
|
|
|
|
|
\bar no
|
|
|
|
|
\strikeout off
|
|
|
|
|
\xout off
|
|
|
|
|
\uuline off
|
|
|
|
|
\uwave off
|
|
|
|
|
\noun off
|
|
|
|
|
\color none
|
|
|
|
|
|
|
|
|
|
\begin_inset Formula $z$
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\family default
|
|
|
|
|
\series default
|
|
|
|
|
\shape default
|
|
|
|
|
\size default
|
|
|
|
|
\emph default
|
|
|
|
|
\bar default
|
|
|
|
|
\strikeout default
|
|
|
|
|
\xout default
|
|
|
|
|
\uuline default
|
|
|
|
|
\uwave default
|
|
|
|
|
\noun default
|
|
|
|
|
\color inherit
|
|
|
|
|
axis) of the display space using a WASD-esque layout of the UHJK keys.
|
|
|
|
|
Objects can be rotated about the vertical (
|
|
|
|
|
\begin_inset Formula $y$
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
) axis using the B and N keys.
|
|
|
|
|
Finally the placement of an object can be reset to default using the R
|
|
|
|
|
key, the addition of the shift modifier resets all clients.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Worth noting is that this represents arbitrary placement of sources in two
|
|
|
|
|
axes of position and one of rotation.
|
|
|
|
|
This is a result of these being the most common and intuitive axes with
|
|
|
|
|
which sources will need to be manipulated.
|
|
|
|
|
The ability to allow movement in all degrees would require only binding
|
|
|
|
|
these actions to keys.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 19:43:23 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
There is room to improve these controls, however, as the directions of movement
|
|
|
|
|
for selected objects are in relation to the fixed axes of the display space
|
|
|
|
|
as opposed to the view of the viewpoint camera.
|
|
|
|
|
In practice this means that when moving objects in the display space the
|
|
|
|
|
orientation of the space must be considered in order to identify which
|
|
|
|
|
direction the object should be moved.
|
|
|
|
|
This is less intuitive than could be expected in other areas where such
|
|
|
|
|
a control scheme is used such as video games or modelling software.
|
|
|
|
|
In such implementations when moving objects the directions are typically
|
|
|
|
|
taken from the camera's frame of reference and as the feasibility of employing
|
|
|
|
|
a similar control philosophy should be considered.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\begin_layout Section
|
|
|
|
|
Future Work
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 09:37:42 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
Following the extension of the
|
|
|
|
|
\noun on
|
|
|
|
|
OpenGL
|
|
|
|
|
\noun default
|
|
|
|
|
window, the network layer of the
|
|
|
|
|
\noun on
|
|
|
|
|
KinectServer
|
|
|
|
|
\noun default
|
2020-01-09 14:10:03 +00:00
|
|
|
can now be developed with the advantage of a fully functional display method
|
|
|
|
|
for debugging.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The aim of this aspect of the project will be to alter the the
|
|
|
|
|
\noun on
|
|
|
|
|
KinectServer
|
|
|
|
|
\noun default
|
|
|
|
|
in order to allow the logical grouping of connected clients into sources
|
|
|
|
|
for separate display.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
When integrated together the server as a whole will then be able to collect
|
|
|
|
|
discrete point clouds from different sources and coherently display them
|
|
|
|
|
separately in the space, in doing so achieving the objectives for this
|
|
|
|
|
project3.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
|
|
|
|
Network Layer Design Considerations
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Some thought as to the design for the network layer has been undertaken.
|
|
|
|
|
Although this has not yielded a final design for implementation, it has
|
|
|
|
|
made apparent some of the conditions and constraints which must be considered.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
When considering the initial steps for the project, it was thought that
|
|
|
|
|
the network layer should be developed first.
|
|
|
|
|
The design would involve separating much of the logic contained within
|
|
|
|
|
the
|
|
|
|
|
\noun on
|
|
|
|
|
KinectServer
|
|
|
|
|
\noun default
|
|
|
|
|
object into a new
|
|
|
|
|
\noun on
|
|
|
|
|
KinectSource
|
|
|
|
|
\noun default
|
|
|
|
|
object which would represent a group of clients acting as a single source.
|
|
|
|
|
It would function as a group of
|
|
|
|
|
\noun on
|
|
|
|
|
KinectSocket
|
|
|
|
|
\noun default
|
|
|
|
|
s that could be individually polled for new frames using the same interface
|
|
|
|
|
currently being used by the
|
|
|
|
|
\noun on
|
|
|
|
|
KinectServer
|
|
|
|
|
\noun default
|
|
|
|
|
.
|
|
|
|
|
The
|
|
|
|
|
\noun on
|
|
|
|
|
KinectServer
|
|
|
|
|
\noun default
|
|
|
|
|
object itself would be reduced to simply managing these
|
|
|
|
|
\noun on
|
|
|
|
|
KinectSource
|
|
|
|
|
\noun default
|
|
|
|
|
s.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
2020-01-09 19:43:23 +00:00
|
|
|
An advantage of this approach would be that it could also contain the additional
|
|
|
|
|
information which should exist per source such as the calibration data
|
|
|
|
|
and settings.
|
2020-01-09 14:10:03 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
2020-01-09 19:43:23 +00:00
|
|
|
However it would have also represented a significant architecture change
|
2020-01-09 14:10:03 +00:00
|
|
|
in the entire server application and without a functioning display method
|
2020-01-09 19:43:23 +00:00
|
|
|
it would have been challenging to debug.
|
2020-01-09 14:10:03 +00:00
|
|
|
As a result, it was instead decided to work on the display method first.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Coming back to the network design following this work, a different method
|
|
|
|
|
has been considered.
|
|
|
|
|
A separate piece of work currently being undertaken is investigating the
|
2020-01-09 19:43:23 +00:00
|
|
|
network behaviour of the suite with focus on unblocking the sockets and
|
2020-01-09 14:10:03 +00:00
|
|
|
aid in the parallel operation of the server.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
2020-01-09 19:43:23 +00:00
|
|
|
Considerations from this work in combination with an emphasis on simplicity
|
|
|
|
|
has suggested a new approach.
|
2020-01-09 14:10:03 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsubsection
|
|
|
|
|
Socket Handshake
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
A handshake process has been suggested for when new clients connect to the
|
|
|
|
|
|
|
|
|
|
\noun on
|
|
|
|
|
KinectServer
|
|
|
|
|
\noun default
|
|
|
|
|
.
|
|
|
|
|
The aim is to implement the method by which clients are grouped into sources
|
|
|
|
|
but also to solve how clients identify themselves consistently when communicati
|
|
|
|
|
ng over multiple sockets.
|
|
|
|
|
Multiple sockets can be used by clients in order to make simultaneous connectio
|
|
|
|
|
ns to the server and increase bandwidth.
|
|
|
|
|
However when doing so it is important to be able to identify which sockets
|
|
|
|
|
represent which client.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The proposed handshake would be initiated by the client when connecting
|
|
|
|
|
to the server, at this point they include which source they should be grouped
|
|
|
|
|
with using an integer ID.
|
|
|
|
|
The server groups the socket as such and, if one has not been received,
|
|
|
|
|
responds with a random identifier string that should be used across all
|
|
|
|
|
sockets to identify the client.
|
|
|
|
|
Should the newly connected socket be for a client that is already connected
|
|
|
|
|
then the client will respond with it's existing identifier to inform the
|
|
|
|
|
server that this ID has been ignored.
|
|
|
|
|
In doing so the client now has a method of identifying itself agnostic
|
|
|
|
|
of socket, and the server has a way of identifying the source which is
|
|
|
|
|
frame is representing.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
|
|
|
|
Deliverables and Additional Goals
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
At this point in the project it is worth considering the viability of the
|
|
|
|
|
final deliverables with relation to the time remaining.
|
|
|
|
|
Based on the work completed so far the original objectives of multi-source
|
|
|
|
|
holoportation remain viable with a round of complete testing defined and
|
|
|
|
|
employed.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
This testing suite is yet to be defined but will comprise performance evaluation
|
|
|
|
|
for both the network and display aspects of the software.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Should the original specification be delivered and evaluated with time remaining
|
|
|
|
|
, additional goals and investigations should be examined.
|
2020-01-09 19:43:23 +00:00
|
|
|
Initially, aspects already completed should be investigated for further
|
|
|
|
|
refinement, namely the control scheme as mentioned above.
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Additionally, when considering the design principle of network and presentation
|
|
|
|
|
separation in combination with the relevance of the technology to the spaces
|
|
|
|
|
of AR and VR, an interesting analysis could be made into the applicability
|
|
|
|
|
of multi-source network developments to additional display methods.
|
2020-01-09 14:10:03 +00:00
|
|
|
Mobile AR and
|
|
|
|
|
\noun on
|
|
|
|
|
Hololens
|
|
|
|
|
\noun default
|
|
|
|
|
display for
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
have both been demonstrated and either could prove interesting when considered
|
|
|
|
|
in a multi-source context.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
|
|
|
|
Final Report Structure
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
2020-01-09 19:43:23 +00:00
|
|
|
The final report for the project will, to an extent, be based off of this
|
2020-01-09 14:10:03 +00:00
|
|
|
piece in terms of structure and content.
|
|
|
|
|
The literature review will be expanded to include an investigation of simultane
|
|
|
|
|
ous video streaming in order to contextualise the coming work on the network
|
|
|
|
|
layer of the application suite.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
A results section will describe the quantitative and qualitative results
|
|
|
|
|
of the final product, these will then be evaluated and discussed in the
|
|
|
|
|
discussion section.
|
|
|
|
|
At this point a discussion can also be had about further developments that
|
|
|
|
|
can be made, both to the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
2020-01-09 19:43:23 +00:00
|
|
|
codebase but also to the multi-source version presented by this project.
|
2020-01-09 14:10:03 +00:00
|
|
|
The structure will be as follows,
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Title Page
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Abstract
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Table of Contents
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_deeper
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Including lists of figures, code listings
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_deeper
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Acknowledgements
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Introduction
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Literature Review
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Methodology and Developments
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Results
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Discussion
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_deeper
|
|
|
|
|
\begin_layout Itemize
|
2020-01-09 19:43:23 +00:00
|
|
|
A description of possible further developments
|
2020-01-09 14:10:03 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_deeper
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Conclusion
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
References
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Itemize
|
|
|
|
|
Appendices
|
2020-01-09 09:37:42 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\begin_layout Section
|
|
|
|
|
Summary
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 19:43:23 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
Within this piece the process of extending the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan3D
|
|
|
|
|
\noun default
|
|
|
|
|
software to include multi-source holoportation has been introduced.
|
|
|
|
|
The use of such a system has many applications from uses inherited from
|
|
|
|
|
it's 2D video basis such as conference calls to new utilisations that are
|
|
|
|
|
wholly unique to the environment.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The literature review contextualises the
|
|
|
|
|
\noun on
|
|
|
|
|
LiveScan
|
|
|
|
|
\noun default
|
|
|
|
|
suite and the wider spaces of AR, VR, 3D video and multi-source holoportation
|
|
|
|
|
itself.
|
|
|
|
|
Previous examples of holoportation are described and their aims of achieving
|
|
|
|
|
telepresence are discussed.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The current state of the project is laid out showing good progress through
|
|
|
|
|
the required areas of development.
|
|
|
|
|
Of the display and network layers requiring development to fulfil the project,
|
|
|
|
|
the display element has been extended in order to allow the rendering of
|
|
|
|
|
multiple environments simultaneously with a dynamic sub-system of geometric
|
|
|
|
|
transformations.
|
|
|
|
|
The transforms are responsive to user input allowing arbitrary placement
|
|
|
|
|
and orientation of individual sources within the display space.
|
|
|
|
|
While this control interface allows free movement in the most naturally
|
|
|
|
|
traversed axes it could use some additional tuning to make it feel more
|
|
|
|
|
intuitive.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
The next steps for the project leading up to its completion are presented,
|
|
|
|
|
the initial and current plans for the remaining work is described and additiona
|
|
|
|
|
l stretch goals are defined for any additional time.
|
|
|
|
|
How the work will be presented in a final report is also described.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\begin_layout Section
|
|
|
|
|
Conclusions
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 14:10:03 +00:00
|
|
|
\begin_layout Standard
|
2020-01-09 19:43:23 +00:00
|
|
|
At roughly halfway through the time allowed for the project the native display
|
|
|
|
|
methods have successfully been extended to meet the deliverable specification.
|
|
|
|
|
This involves allowing the display of multiple sources while also allowing
|
|
|
|
|
arbitrary placement and orientation within the display space.
|
2020-01-09 14:10:03 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
From this point the network layer of the suite will be developed to also
|
2020-01-09 19:43:23 +00:00
|
|
|
match the specification, allowing connected clients to be grouped into
|
|
|
|
|
sources for polling and processing.
|
2020-01-09 14:10:03 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
Following the development of the two, testing methodologies will be defined
|
|
|
|
|
and carried out to gather quantitative results for the final product.
|
|
|
|
|
A final report on the results will be available in May 2020.
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-06 16:28:10 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset Newpage pagebreak
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset CommandInset bibtex
|
|
|
|
|
LatexCommand bibtex
|
|
|
|
|
btprint "btPrintCited"
|
|
|
|
|
bibfiles "references"
|
|
|
|
|
options "bibtotoc"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
2020-01-09 09:37:42 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
2020-01-09 14:10:03 +00:00
|
|
|
\begin_layout Section
|
2020-01-09 09:37:42 +00:00
|
|
|
\start_of_appendix
|
2020-01-09 14:10:03 +00:00
|
|
|
Existing Data Structures
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "sec:Existing-Data-Structures"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset CommandInset include
|
|
|
|
|
LatexCommand lstinputlisting
|
|
|
|
|
filename "../snippets/point2f.cs"
|
|
|
|
|
lstparams "language={[Sharp]C},keywordstyle={\\color{blue}},commentstyle={\\color{magenta}\\itshape},emphstyle={\\color{red}},basicstyle={\\ttfamily},stringstyle={\\color{green}},identifierstyle={\\color{cyan}},caption={Cartesian coordinate in 2 dimensions}"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset CommandInset include
|
|
|
|
|
LatexCommand lstinputlisting
|
|
|
|
|
filename "../snippets/point3f.cs"
|
|
|
|
|
lstparams "language={[Sharp]C},keywordstyle={\\color{blue}},commentstyle={\\color{magenta}\\itshape},emphstyle={\\color{red}},basicstyle={\\ttfamily},stringstyle={\\color{green}},identifierstyle={\\color{cyan}},caption={Cartesian coordinate in 3 dimensions}"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset CommandInset include
|
|
|
|
|
LatexCommand lstinputlisting
|
|
|
|
|
filename "/home/andy/uni/dissertation/snippets/affinetransform.cs"
|
|
|
|
|
lstparams "language={[Sharp]C},keywordstyle={\\color{blue}},commentstyle={\\color{magenta}\\itshape},emphstyle={\\color{red}},basicstyle={\\ttfamily},stringstyle={\\color{green}},identifierstyle={\\color{cyan}},caption={Affine transformation matrix with translation}"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Section
|
|
|
|
|
New Data Structures
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Subsection
|
|
|
|
|
Frame
|
|
|
|
|
\begin_inset CommandInset label
|
|
|
|
|
LatexCommand label
|
|
|
|
|
name "subsec:Frame"
|
|
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\begin_layout Standard
|
|
|
|
|
\begin_inset CommandInset include
|
|
|
|
|
LatexCommand lstinputlisting
|
|
|
|
|
filename "../snippets/frame.cs"
|
|
|
|
|
lstparams "language={[Sharp]C},keywordstyle={\\color{blue}},commentstyle={\\color{magenta}\\itshape},emphstyle={\\color{red}},basicstyle={\\ttfamily},stringstyle={\\color{green}},identifierstyle={\\color{cyan}},caption={Point cloud with Client ID}"
|
2020-01-09 09:37:42 +00:00
|
|
|
|
|
|
|
|
\end_inset
|
|
|
|
|
|
|
|
|
|
|
2019-12-03 10:14:59 +00:00
|
|
|
\end_layout
|
|
|
|
|
|
|
|
|
|
\end_body
|
|
|
|
|
\end_document
|
