-
Notifications
You must be signed in to change notification settings - Fork 0
/
point_layout.html
441 lines (396 loc) · 44 KB
/
point_layout.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
<!DOCTYPE HTML>
<html lang="en" class="sidebar-visible no-js light">
<head>
<!-- Book generated using mdBook -->
<meta charset="UTF-8">
<title>Point layout in detail - pasture</title>
<!-- Custom HTML head -->
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#ffffff" />
<link rel="icon" href="favicon.svg">
<link rel="shortcut icon" href="favicon.png">
<link rel="stylesheet" href="css/variables.css">
<link rel="stylesheet" href="css/general.css">
<link rel="stylesheet" href="css/chrome.css">
<link rel="stylesheet" href="css/print.css" media="print">
<!-- Fonts -->
<link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
<link rel="stylesheet" href="fonts/fonts.css">
<!-- Highlight.js Stylesheets -->
<link rel="stylesheet" href="highlight.css">
<link rel="stylesheet" href="tomorrow-night.css">
<link rel="stylesheet" href="ayu-highlight.css">
<!-- Custom theme stylesheets -->
<link rel="stylesheet" href="./mdbook-admonish.css">
</head>
<body>
<!-- Provide site root to javascript -->
<script type="text/javascript">
var path_to_root = "";
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
</script>
<!-- Work around some values being stored in localStorage wrapped in quotes -->
<script type="text/javascript">
try {
var theme = localStorage.getItem('mdbook-theme');
var sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') && theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
</script>
<!-- Set the theme before any content is loaded, prevents flash -->
<script type="text/javascript">
var theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
var html = document.querySelector('html');
html.classList.remove('no-js')
html.classList.remove('light')
html.classList.add(theme);
html.classList.add('js');
</script>
<!-- Hide / unhide sidebar before it is displayed -->
<script type="text/javascript">
var html = document.querySelector('html');
var sidebar = 'hidden';
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
}
html.classList.remove('sidebar-visible');
html.classList.add("sidebar-" + sidebar);
</script>
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
<div class="sidebar-scrollbox">
<ol class="chapter"><li class="chapter-item expanded "><a href="overview.html"><strong aria-hidden="true">1.</strong> Overview</a></li><li class="chapter-item expanded "><a href="using_pasture.html"><strong aria-hidden="true">2.</strong> Using pasture</a></li><li class="chapter-item expanded "><a href="data_model.html"><strong aria-hidden="true">3.</strong> pasture data model</a></li><li class="chapter-item expanded "><a href="point_layout.html" class="active"><strong aria-hidden="true">4.</strong> Point layout in detail</a></li><li class="chapter-item expanded "><a href="point_buffers.html"><strong aria-hidden="true">5.</strong> Point buffers in detail</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="point_buffers/builtin_types.html"><strong aria-hidden="true">5.1.</strong> The built-in point buffer types</a></li><li class="chapter-item expanded "><a href="point_buffers/buffer_traits.html"><strong aria-hidden="true">5.2.</strong> The buffer traits</a></li><li class="chapter-item expanded "><div><strong aria-hidden="true">5.3.</strong> Implementing your own point buffer type</div></li><li class="chapter-item expanded "><a href="point_buffers/buffer_slices.html"><strong aria-hidden="true">5.4.</strong> Buffer slices</a></li></ol></li><li class="chapter-item expanded "><a href="examples.html"><strong aria-hidden="true">6.</strong> Examples</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="examples/basic_usage.html"><strong aria-hidden="true">6.1.</strong> Basic usage</a></li><li class="chapter-item expanded "><a href="examples/point_io.html"><strong aria-hidden="true">6.2.</strong> Reading and writing general point cloud files</a></li><li class="chapter-item expanded "><a href="examples/reading_las.html"><strong aria-hidden="true">6.3.</strong> Reading LAS files</a></li></ol></li></ol>
</div>
<div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
</nav>
<div id="page-wrapper" class="page-wrapper">
<div class="page">
<div id="menu-bar-hover-placeholder"></div>
<div id="menu-bar" class="menu-bar sticky bordered">
<div class="left-buttons">
<button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
<i class="fa fa-bars"></i>
</button>
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
<i class="fa fa-paint-brush"></i>
</button>
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
<li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li>
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
</ul>
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
<i class="fa fa-search"></i>
</button>
</div>
<h1 class="menu-title">pasture</h1>
<div class="right-buttons">
<a href="print.html" title="Print this book" aria-label="Print this book">
<i id="print-button" class="fa fa-print"></i>
</a>
</div>
</div>
<div id="search-wrapper" class="hidden">
<form id="searchbar-outer" class="searchbar-outer">
<input type="search" name="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
</form>
<div id="searchresults-outer" class="searchresults-outer hidden">
<div id="searchresults-header" class="searchresults-header"></div>
<ul id="searchresults">
</ul>
</div>
</div>
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
<script type="text/javascript">
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
</script>
<div id="content" class="content">
<main>
<h1 id="understanding-the-pointlayout-type"><a class="header" href="#understanding-the-pointlayout-type">Understanding the <code>PointLayout</code> type</a></h1>
<p>In this section you will learn all about the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointLayout.html"><code>PointLayout</code></a> type in <code>pasture</code>. You will learn how it is used to represent the structure of the points in your point clouds, how it relates to individual point attributes, and how you can create your own <code>PointLayout</code> either from hand or from an existing <code>struct</code> definition. Additionally you will learn about all the built-in point attribute definitions that <code>pasture</code> provides, and how you can create your own point attributes.</p>
<h2 id="point-attributes"><a class="header" href="#point-attributes">Point attributes</a></h2>
<p>In the <a href="data_model.html">data model section</a> of this tutorial we learned that <code>pasture</code> models a point cloud as a collection of tuples of attributes. A point attribute is a uniquely identifiable piece of data, which <code>pasture</code> represents using two types: <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointAttributeDefinition.html"><code>PointAttributeDefinition</code></a> and <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointAttributeMember.html"><code>PointAttributeMember</code></a>. Understanding how attributes combine to form a <code>PointLayout</code> will also answer the question why there are two distinct attribute types in <code>pasture</code>.</p>
<p>When we talk about specific point attributes, we often refer to them by their <em>name</em>, which is why <code>pasture</code> uses strings as the unique identifiers for point attributes. To get a feel for the different types of point attributes that are commonly used, we can look at the <a href="https://www.asprs.org/wp-content/uploads/2019/07/LAS_1_4_r15.pdf">LAS file specification</a>, specifically under the section that defines the <em>point data record formats</em>. Looking at point data record format 0---the simplest format that LAS provides---we see the following list of point attributes:</p>
<div class="table-wrapper"><table><thead><tr><th>Item</th><th>Format</th><th>Size</th><th>Required</th></tr></thead><tbody>
<tr><td>X</td><td>long</td><td>4 bytes</td><td>yes</td></tr>
<tr><td>Y</td><td>long</td><td>4 bytes</td><td>yes</td></tr>
<tr><td>Z</td><td>long</td><td>4 bytes</td><td>yes</td></tr>
<tr><td>Intensity</td><td>unsigned short</td><td>2 bytes</td><td>no</td></tr>
<tr><td>Return Number</td><td>3 bits (bits 0-2)</td><td>3 bits</td><td>yes</td></tr>
<tr><td>Number of Returns (Given Pulse)</td><td>3 bits (bits 3-5)</td><td>3 bits</td><td>yes</td></tr>
<tr><td>Scan Direction Flag</td><td>1 bit (bit 6)</td><td>1 bit</td><td>yes</td></tr>
<tr><td>Edge of Flight Line</td><td>1 bit (bit 7)</td><td>1 bit</td><td>yes</td></tr>
<tr><td>Classification</td><td>unsigned char</td><td>1 byte</td><td>yes</td></tr>
<tr><td>Scan Angle Rank (-90 to +90) – Left Side</td><td>signed char</td><td>1 byte</td><td>yes</td></tr>
<tr><td>User Data</td><td>unsigned char</td><td>1 byte</td><td>no</td></tr>
<tr><td>Point Source ID</td><td>unsigned short</td><td>2 bytes</td><td>yes</td></tr>
</tbody></table>
</div>
<p>We see the name of each attribute (the <code>Item</code> column), the data type used to represent the attribute (the <code>Format</code>) column, how many bits or bytes a single value of that attribute requires (the <code>Size</code> column), and whether or not it is required to include this value in the LAS point record (the <code>Required</code> column). We don't really care about the last column, but the first three columns are interesting. Not only do they explain what types of data a single point might contain, they also give information of the <em>memory representation</em> of the point data. <code>pasture</code> deals with efficient in-memory representations of point clouds, so the mapping of memory ranges to the actual point attributes is a central part of what <code>pasture</code> does. It does this by using precisely the information we saw in the table:</p>
<p>Each point attribute in <code>pasture</code> has a unique name, a data type and a size! If you look at the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointAttributeDefinition.html"><code>PointAttributeDefinition</code></a> type, you will see three corresponding accessors:</p>
<ul>
<li><a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointAttributeDefinition.html#method.name"><code>fn name(&self) -> &str</code></a> for accessing the name of the attribute</li>
<li><a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointAttributeDefinition.html#method.datatype"><code>fn datatype(&self) -> PointAttributeDataType</code></a> for accessing the data type of the attribute</li>
<li><a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointAttributeDefinition.html#method.size"><code>fn size(&self) -> u64</code></a> for accessing the size in bytes of the attribute</li>
</ul>
<p>If we look into the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/attributes/index.html"><code>layout::attributes</code> module</a> we see all the predefined point attributes that <code>pasture</code> provides:</p>
<div class="table-wrapper"><table><thead><tr><th>Name</th><th>Description</th></tr></thead><tbody>
<tr><td>CLASSIFICATION</td><td>Attribute definition for a classification. Default datatype is U8</td></tr>
<tr><td>CLASSIFICATION_FLAGS</td><td>Attribute definition for the classification flags. Default datatype is U8</td></tr>
<tr><td>COLOR_RGB</td><td>Attribute definition for an RGB color. Default datatype is Vec3u16</td></tr>
<tr><td>EDGE_OF_FLIGHT_LINE</td><td>Attribute definition for an edge of flight line flag. Default datatype is Bool</td></tr>
<tr><td>GPS_TIME</td><td>Attribute definition for a GPS timestamp. Default datatype is F64</td></tr>
<tr><td>INTENSITY</td><td>Attribute definition for an intensity value. Default datatype is U16</td></tr>
<tr><td>NIR</td><td>Attribute definition for near-infrared records (NIR). Default datatype is U16</td></tr>
<tr><td>NORMAL</td><td>Attribute definition for a 3D point normal. Default datatype is Vec3f32</td></tr>
<tr><td>NUMBER_OF_RETURNS</td><td>Attribute definition for the number of returns. Default datatype is U8</td></tr>
<tr><td>POINT_ID</td><td>Attribute definition for a point ID. Default datatype is U64</td></tr>
<tr><td>POINT_SOURCE_ID</td><td>Attribute definition for a point source ID. Default datatype is U16</td></tr>
<tr><td>POSITION_3D</td><td>Attribute definition for a 3D position. Default datatype is Vec3f64</td></tr>
<tr><td>RETURN_NUMBER</td><td>Attribute definition for a return number. Default datatype is U8</td></tr>
<tr><td>RETURN_POINT_WAVEFORM_LOCATION</td><td>Attribute definition for the return point waveform location in the LAS format. Default datatype is F32</td></tr>
<tr><td>SCANNER_CHANNEL</td><td>Attribute definition for the scanner channel. Default datatype is U8</td></tr>
<tr><td>SCAN_ANGLE</td><td>Attribute definition for a scan angle with extended precision (like in LAS format 1.4). Default datatype is I16</td></tr>
<tr><td>SCAN_ANGLE_RANK</td><td>Attribute definition for a scan angle rank. Default datatype is I8</td></tr>
<tr><td>SCAN_DIRECTION_FLAG</td><td>Attribute definition for a scan direction flag. Default datatype is Bool</td></tr>
<tr><td>USER_DATA</td><td>Attribute definition for a user data field. Default datatype is U8</td></tr>
<tr><td>WAVEFORM_DATA_OFFSET</td><td>Attribute definition for the offset to the waveform data in the LAS format. Default datatype is U64</td></tr>
<tr><td>WAVEFORM_PACKET_SIZE</td><td>Attribute definition for the size of a waveform data packet in the LAS format. Default datatype is U32</td></tr>
<tr><td>WAVEFORM_PARAMETERS</td><td>Attribute definition for the waveform parameters in the LAS format. Default datatype is Vector3</td></tr>
<tr><td>WAVE_PACKET_DESCRIPTOR_INDEX</td><td>Attribute definition for the wave packet descriptor index in the LAS format. Default datatype is U8</td></tr>
</tbody></table>
</div>
<h2 id="point-attribute-data-types"><a class="header" href="#point-attribute-data-types">Point attribute data types</a></h2>
<p>The description of each built-in attribute includes a statement of the form "The default datatype is X". <code>pasture</code> makes an assumption what a good default datatype is for each attribute. Here are some examples:</p>
<ul>
<li><code>POSITION_3D</code> -> <code>Vec3f64</code></li>
<li><code>CLASSIFICATION</code> -> <code>U8</code></li>
<li><code>COLOR_RGB</code> -> <code>Vec3u16</code></li>
</ul>
<p>These datatypes roughly correspond to primitive types that Rust supports, plus several vector types provided by <code>nalgebra</code>. <code>pasture</code> defines the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/enum.PointAttributeDataType.html"><code>PointAttributeDataType</code></a> enum, which constrains the valid datatypes for point attributes. It includes all integer and floating-point types up to and including 64 bits, some three- and four-component vector types (themselves of integers and floating-point values), as well as two special types <code>ByteArray(u64)</code> and <code>Custom{ ... }</code>. Each of these datatypes has a well-known binary representation and corresponds to a specific Rust type, which is what allows conversions between untyped memory (<code>[u8]</code>) and strongly typed attribute data. This correspondence is established using the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/trait.PrimitiveType.html"><code>PrimitiveType</code></a> trait, which is a trait that <code>pasture</code> implements for the Rust integer and floating-point primitive types and some of the <code>nalgebra</code> vector types. It allows code like this:</p>
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
</span><span class="boring">fn main() {
</span>let datatype: PointAttributeDataType = f64::data_type();
<span class="boring">}</span></code></pre></pre>
<p>For memory safety, <code>pasture</code> uses the <a href="https://docs.rs/bytemuck/latest/bytemuck/"><code>bytemuck</code></a> crate, which ensures that all memory transmutations from and to untyped memory are safe. This is done by requiring that all <code>pasture</code> primitive types have to implement <a href="https://docs.rs/bytemuck/latest/bytemuck/trait.Pod.html"><code>bytemuck::Pod</code></a>. This has some interesting but also limiting side-effects, in particular it prevents <code>pasture</code> from supporting the Rust primitive type <code>bool</code>, which is not valid for any bit pattern (i.e. it is represented using 8 bits, but only the bit patterns <code>0</code> and <code>1</code> are valid). We will shortly see that this also has implications for which types of <code>struct</code>s you are allowed to use as point representations.</p>
<h2 id="custom-attributes"><a class="header" href="#custom-attributes">Custom attributes</a></h2>
<p>Besides the built-in attribute definitions, you can easily create your own point attributes using <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointAttributeDefinition.html#method.custom"><code>PointAttributeDefinition::custom</code></a>:</p>
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
</span><span class="boring">fn main() {
</span>let custom_attribute = PointAttributeDefinition::custom(Cow::Borrowed("Custom"), PointAttributeDataType::F32);
<span class="boring">}</span></code></pre></pre>
<p>It is a <code>const fn</code>, so you can create compile-time constant point attributes, which is precisely what all the built-in attribute definitions are! <code>pasture-io</code> uses the same mechanism to add LAS-specific attribute definitions, for example the <a href="https://docs.rs/pasture-io/0.4.0/pasture_io/las/constant.ATTRIBUTE_LOCAL_LAS_POSITION.html"><code>ATTRIBUTE_LOCAL_LAS_POSITION</code></a> which corresponds to a point position in the local space of a LAS file, represented using 32-bit signed integers:</p>
<pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)]
</span><span class="boring">fn main() {
</span>pub const ATTRIBUTE_LOCAL_LAS_POSITION: PointAttributeDefinition = PointAttributeDefinition::custom(
Cow::Borrowed("LASLocalPosition"),
PointAttributeDataType::Vec3i32,
);
<span class="boring">}</span></code></pre></pre>
<h2 id="from-attributes-to-pointlayout"><a class="header" href="#from-attributes-to-pointlayout">From attributes to <code>PointLayout</code></a></h2>
<p>So far we saw how <code>pasture</code> represents a single point attribute. Now we will look at combining multiple point attributes into the full description of a point cloud, i.e. the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointLayout.html"><code>PointLayout</code></a> type! This will also answer the question why there are two types for point attributes in <code>pasture</code>!</p>
<p>Recall the custom <code>Point</code> type that we wrote in the <a href="data_model.html">Data Model section</a>:</p>
<pre><pre class="playground"><code class="language-rust editable">use nalgebra::Vector3;
type Position = Vector3<f64>;
type Classification = u8;
type Color = Vector3<u8>;
type NumberOfReturnPulses = u8;
type Point = (Position, Color, Classification, NumberOfReturnPulses);</code></pre></pre>
<p>Let's restate it using <code>struct</code> syntax instead of tuple syntax:</p>
<pre><pre class="playground"><code class="language-rust editable">struct Point {
pub position: Position, //Vector3<f64>
pub color: Color, //Vector3<u8>
pub classification: Classification, //u8
pub number_of_returns: NumberOfReturnPulses, //u8
}</code></pre></pre>
<p>To describe this point type in <code>pasture</code>, we first need one <code>PointAttributeDefinition</code> for each of the members:</p>
<pre><pre class="playground"><code class="language-rust editable">use pasture_core::layout::attributes::*;
let position_attribute = POSITION_3D;
let color_attribute = COLOR_RGB.with_custom_datatype(PointAttributeDataType::Vec3u8);
let classification_attribute = CLASSIFICATION;
let number_of_returns_attribute = NUMBER_OF_RETURNS;</code></pre></pre>
<p>We can use the built-in attribute definitions for every attribute, which is convenient. The only change is that <code>pasture</code> assumes RGB colors to be represented using the <code>Vec3u16</code> datatype (a convention taken from the LAS file format), but here we want 8-bit unsigned integers instead, so we modify the datatype to be <code>Vec3u8</code> instead. Note that this does not affect the global <code>COLOR_RGB</code> constant but instead creates a new <code>PointAttributeDefinition</code> instance.</p>
<p>In order to combine these attributes into a <code>PointLayout</code>, we have to understand what the purpose of the <code>PointLayout</code> structure is: The <code>PointLayout</code> provides the information necessary to go from untyped memory to strongly typed attribute and point values! This means that it has to include all the information about the exact memory layout of each attribute as well as their relationship to each other, for example the order of the attributes within a point record. So if you write the following code, <code>pasture</code> needs to know how to execute it and whether it is correct or would violate memory safety:</p>
<pre><pre class="playground"><code class="language-rust editable">let buffer: VectorBuffer = ...; // A buffer that stores `Point` values, obtained from somewhere
// Get the color of the first point:
let color = buffer.view_attribute::<Vector3<u8>>(&color_attribute).at(0);</code></pre></pre>
<p>Since <code>VectorBuffer</code> stores untyped memory internally (<code>Vec<u8></code>), here is what <code>pasture</code> has to do:</p>
<p><img src="figures/untyped_to_typed_memory.svg" alt="Picture illustrating how pasture goes from untyped memory to typed attributes" /></p>
<p>The call <code>.at(0)</code> in the code example above translates to the instruction "Read 3 bytes at offset 24 and cast to <code>Vector3<u8></code>" in the figure. It contains three pieces of information that are only known at runtime and which are included in the <code>PointLayout</code> type<label for="sn-0" class="margin-toggle sidenote-number"></label><input type="checkbox" id="sn-0" class="margin-toggle"></input><span class="sidenote">The necessary information is actually calculated not only from the <code>PointLayout</code>, but also based on the memory layout of the point buffer, since offsets are calculated differently in interleaved memory layout vs. columnar memory layout. We will learn all about that in the section on <a href="point_buffers/memory_layout.html">buffer memory layouts</a>.</span>:</p>
<ul>
<li>The <em>read offset</em> (24 bytes)</li>
<li>The <em>read size</em> (3 bytes)</li>
<li>The static type that the memory should be interpreted<label for="sn-1" class="margin-toggle sidenote-number"></label><input type="checkbox" id="sn-1" class="margin-toggle"></input><span class="sidenote">Interpreted is the correct term here, because that is what all statically typed programming languages do: Static types provide the semantics for raw memory and hence define how memory regions are interpreted, from which follow the correct assembly instructions for reading from or writing to memory. If we write <code>let m = variable.member;</code>, it is the job of the compiler to figure out the correct instructions for reading the memory corresponding to <code>member</code> from the instance <code>variable</code> based on the static type of <code>variable</code>. The compiler does this at compile time, <code>pasture</code> does something similar at runtime!</span> as (<code>Vector3<u8></code>)</li>
</ul>
<p>Two of the three properties are already included in the <code>PointAttributeDefinition</code> type: The read size is simply the size of the attribute datatype, and the static type is given by the user (<code>Vector3<u8></code>) and has to correspond to the attribute datatype. <code>pasture</code> performs a runtime check in <code>view_attribute</code> to make sure that these two types match! The offset however is not known given only a <code>PointAttributeDefinition</code>. It depends on the <em>order of attributes</em> within the point type. Take a look at the following two point type definitions and try to figure out in which sense they are identical and in which sense they differ:</p>
<pre><pre class="playground"><code class="language-rust editable">struct Point1 {
pub position: Position, //Vector3<f64>
pub color: Color, //Vector3<u8>
pub classification: Classification, //u8
pub number_of_returns: NumberOfReturnPulses, //u8
}
struct Point2 {
pub classification: Classification, //u8
pub color: Color, //Vector3<u8>
pub number_of_returns: NumberOfReturnPulses, //u8
pub position: Position, //Vector3<f64>
}</code></pre></pre>
<p>Both types have the exact same point attributes but in a different order. Even for attributes at the same index, such as the <code>color</code> attribute, their memory locations can differ. The actual offset is determined by the Rust compiler when calculating the <a href="https://doc.rust-lang.org/reference/type-layout.html#representations"><em>representation</em></a> of the type. The default algorithm is implementation-defined, but assuming a simple algorithm the offset of <code>Point1::color</code> might be 24 bytes (the size of the <code>position</code> field), whereas the offset of <code>Point2::color</code> might be 1 byte (the size of the <code>classification</code> field).</p>
<p>If you want to know all the details about point type memory layout and how to manually build a <code>PointLayout</code>, read on! If you only care about something that works, skip to <a href="#using-the-derivepointtype-macro">"Using the <code>#[derive(PointType)]</code> macro"</a>.</p>
<h2 id="details-of-point-memory-layouts"><a class="header" href="#details-of-point-memory-layouts">Details of point memory layouts</a></h2>
<p>Getting the memory layout of a point type right is one of the more challenging aspects that <code>pasture</code> has to deal with. It matters most when going from untyped memory to user-defined point types, such as the <code>Point1</code> and <code>Point2</code> types from the previous example. Note that if you only ever access point data through the attribute views and never use the point views (which require a strongly typed point type such as <code>Point1</code>), most of these details will not concern you!</p>
<p>We will now build a <code>PointLayout</code> from scratch for the <code>Point2</code> type and learn all about type representations and alignment requirements, as well as some details regarding memory transmutations. Time to have some fun!</p>
<p>To make things a bit easier, we will use <code>f32</code> positions instead of <code>f64</code> so that the overall type has less bytes. This makes some of the figures easier to follow. Here is the updated <code>Point2</code> type:</p>
<pre><pre class="playground"><code class="language-rust editable">struct Point2 {
pub classification: Classification, //u8
pub color: Color, //Vector3<u8>
pub number_of_returns: NumberOfReturnPulses, //u8
pub position: Position, //Vector3<f32> !!
}</code></pre></pre>
<p>First we have to figure out the size of each attribute, which we can then sum to get the total size of a single <code>Point2</code> instance in memory:</p>
<ul>
<li><code>classification</code> -> <code>u8</code> -> 1 byte</li>
<li><code>color</code> -> <code>Vector3<u8></code> -> semantically equivalent to <code>[u8; 3]</code> -> 3 bytes</li>
<li><code>number_of_returns</code> -> <code>u8</code> -> 1 byte</li>
<li><code>position</code> -> <code>Vector3<f32></code> -> semantically equivalent to <code>[f32; 3]</code> -> 4*3 = 12 bytes</li>
</ul>
<p>If we sum everything together, we get 17 bytes. Let's see what <code>println!("{}", std::mem::size_of::<Point2>())</code> returns:</p>
<pre><code>fn main() {
println!("{}", std::mem::size_of::<Point2>());
}
// Output: 20
</code></pre>
<p>The difference (which might vary depending on the compiler version and target platform) is due to the <a href="https://doc.rust-lang.org/reference/type-layout.html#representations">Rust rules for type representations</a>. Unfortunately, not only is the default type representation (the <code>Rust</code> representation) implementation-defined, there is not even a stabilized API for querying the offset of a variable within a type<label for="sn-2" class="margin-toggle sidenote-number"></label><input type="checkbox" id="sn-2" class="margin-toggle"></input><span class="sidenote">There is <a href="https://doc.rust-lang.org/std/mem/macro.offset_of.html"><code>std::mem::offset_of!</code></a>, but it is nightly-only at the moment, and it is somewhat limited.</span>! For this reason, <code>pasture</code> does not support creating a <code>PointLayout</code> for a <code>struct</code> that uses the default <code>Rust</code> representation! Instead, all types that want to implement the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/trait.PointType.html"><code>PointType</code></a> trait---a trait required for the type to work with the point view function <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/containers/trait.BorrowedBuffer.html#method.view"><code>view::<T></code></a>---should use the <code>C</code> representation instead.</p>
<div id="admonition-warning" class="admonition admonish-warning">
<div class="admonition-title">
<p>Warning</p>
<p><a class="admonition-anchor-link" href="#admonition-warning"></a></p>
</div>
<div>
<p><code>pasture</code> does not enforce implementors of the <code>PointType</code> trait to use the <code>C</code> representation as the Rust language does not support such constraints. Correctly implementing the <code>PointLayout</code> for a type using the <code>Rust</code> representation will be impractical and is strongly discouraged. Even if you manage to do so, types with the <code>Rust</code> representation may contain padding bytes, which breaks the guarantees made by <code>bytemuck::NoUninit</code>!</p>
</div>
</div>
<p>The <code>C</code> representation has a predictable algorithm for calculating the offsets of members within a <code>struct</code> (explained in detail <a href="https://doc.rust-lang.org/reference/type-layout.html#reprc-structs">here</a>) which we can use to figure out the correct offsets to all our point attributes. Unfortunately even with the <code>C</code> representation, the compiler is allowed to insert <em>padding bytes</em> which would break the requirements that <code>pasture</code> has which enable memory transmutations. To get rid of padding bytes, we have to use <code>#[repr(packed)]</code>, which removes any padding bytes. So we have to change our definition of the <code>Point2</code> type:</p>
<pre><pre class="playground"><code class="language-rust editable">#[repr(C, packed)]
struct Point2 {
pub classification: Classification, //u8
pub color: Color, //Vector3<u8>
pub number_of_returns: NumberOfReturnPulses, //u8
pub position: Position, //Vector3<f32>
}</code></pre></pre>
<p>Now <code>std::mem::size_of::<Point2>()</code> returns 17, as would be expected. The offset for each member can now be easily calculated as the size of all preceding members. Luckily, we don't have to do this by hand: The <code>PointLayout</code> type has a <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointLayout.html#method.add_attribute"><code>add_attribute</code></a> function! So building a <code>PointLayout</code> for <code>Point2</code> works like this:</p>
<pre><pre class="playground"><code class="language-rust editable">use pasture_core::layout::*;
use pasture_core::layout::attributes::*;
let mut layout = PointLayout::default();
layout.add_attribute(CLASSIFICATION, FieldAlignment::Packed(1));
layout.add_attribute(
COLOR_RGB.with_custom_datatype(PointAttributeDataType::Vec3u8),
FieldAlignment::Packed(1));
layout.add_attribute(NUMBER_OF_RETURNS, FieldAlignment::Packed(1));
layout.add_attribute(
POSITION_3D.with_custom_datatype(PointAttributeDataType::Vec3f32),
FieldAlignment::Packed(1));</code></pre></pre>
<p>We use <code>FieldAlignment::Packed(1)</code> to prevent any padding bytes from being added<label for="sn-3" class="margin-toggle sidenote-number"></label><input type="checkbox" id="sn-3" class="margin-toggle"></input><span class="sidenote">Why does <code>pasture</code> support padding bytes in a <code>PointLayout</code> when manually adding attributes, if it is not supported to have a <code>PointType</code> that includes padding bytes? Because you are allowed to access individual attributes even if there is padding, since padding only affects the point view type (obtained by calling <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/containers/trait.BorrowedBuffer.html#method.view"><code>view::<T></code></a> on the point buffer) but not the attribute views (obtained by calling <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/containers/trait.BorrowedBuffer.html#method.view_attribute"><code>view_attribute::<T></code></a> on the buffer).</span> and end up with a <code>PointLayout</code> that correctly represents the memory representation of the <code>Point2</code> type. The calculated offset of each point attribute is stored within the <code>PointLayout</code> together with the <code>PointAttributeDefinition</code> using the second attribute type: <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/struct.PointAttributeMember.html"><code>PointAttributeMember</code></a>. A <code>PointAttributeMember</code> is therefore a specific instance of a point attribute within the memory layout of a point type. This distinction allows using the <code>PointAttributeDefinition</code> as a general identifier for attributes independent of any actual point type. Take a look at the two original point types <code>Point1</code> and <code>Point2</code> again:</p>
<pre><pre class="playground"><code class="language-rust editable">struct Point1 {
pub position: Position, //Vector3<f64>
pub color: Color, //Vector3<u8>
pub classification: Classification, //u8
pub number_of_returns: NumberOfReturnPulses, //u8
}
struct Point2 {
pub classification: Classification, //u8
pub color: Color, //Vector3<u8>
pub number_of_returns: NumberOfReturnPulses, //u8
pub position: Position, //Vector3<f64>
}</code></pre></pre>
<p>Both types contain the same attributes but with different offsets. If we ask the question "Does type <code>X</code> have the point attribute <code>CLASSIFICATION</code>?", then we want the answer to be affirmative for both <code>Point1</code> and <code>Point2</code>. <code>PointAttributeDefinition</code> is the type for checking such a property. If we instead ask "At which byte offset lies the <code>CLASSIFICATION</code> attribute in type <code>X</code>?", we need the information encoded in the <code>PointAttributeMember</code>. You will rarely interact with this type directly, most of it is done by <code>pasture</code> internally.</p>
<h2 id="using-the-derivepointtype-macro"><a class="header" href="#using-the-derivepointtype-macro">Using the <code>#[derive(PointType)]</code> macro</a></h2>
<p>You don't have to create the correct <code>PointLayout</code> for your desired point type by hand. <code>pasture</code> supports a <code>#[derive(PointType)]</code> macro which auto-generates the correct <code>PointLayout</code> based on the memory layout of your <code>struct</code>. Let's look at <a href="https://github.com/Mortano/pasture/blob/main/pasture-core/examples/custom_point_type.rs">an example</a>:</p>
<pre><pre class="playground"><code class="language-rust editable">#[derive(PointType, Clone, Copy, bytemuck::NoUninit, bytemuck::AnyBitPattern)]
#[repr(C, packed)]
struct CustomPointType {
#[pasture(BUILTIN_INTENSITY)]
pub intensity: u16,
#[pasture(BUILTIN_POSITION_3D)]
pub position: Vector3<f32>,
#[pasture(attribute = "CUSTOM_ATTRIBUTE")]
pub custom_attribute: f32,
}</code></pre></pre>
<p>The <code>CustomPointType</code> structure has three point attributes, each corresponding to one of the fields. To tell <code>pasture</code> which field represents which attribute, we have to annotate the fields with <code>#[pasture(X)]</code>, where <code>X</code> is either the name of a built-in attribute, or the special syntax <code>attribute = "name"</code> for custom attributes. The example shows both usage patterns. Built-in attribute names follow the structure <code>BUILTIN_X</code>, where <code>X</code> is the name of the corresponding attribute definition constant as found in the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/attributes/index.html"><code>layout::attributes</code> module</a>.</p>
<p>Then we have to derive a lot of stuff. First, we derive <code>PointType</code>, which will implement the <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/trait.PointType.html"><code>PointType</code></a> trait for us. This trait has a method <code>fn layout() -> PointLayout</code> that returns the <code>PointLayout</code> for the type it is implemented on. In order to use the derive macro, our <code>struct</code> must adhere to the <code>pasture</code> requirements as described in the previous section:</p>
<ul>
<li>It must use the <code>C</code> representation (<code>#[repr(C)]</code>)</li>
<li>It must not include any padding bytes (<code>#[repr(packed)]</code>)</li>
<li>It must implement <code>bytemuck::NoUninit</code> and <code>bytemuck::AnyBitPattern</code>, which have derive macros of their own
<ul>
<li>This in turn requires that the type implements <code>Copy</code></li>
<li><code>bytemuck::NoUninit</code> disallows padding bytes</li>
<li><code>bytemuck::AnyBitPattern</code> disallows <code>bool</code>, <code>enum</code>s and any types that are not valid for all possible bit patterns</li>
</ul>
</li>
<li>The data type of each field must be one of the supported <a href="https://docs.rs/pasture-core/0.4.0/pasture_core/layout/trait.PrimitiveType.html">primitive types</a></li>
</ul>
<p>These restrictions are quite severe but are necessary to allow safe memory transmutations in <code>pasture</code>. The resulting <code>PointLayout</code> looks like this:</p>
<pre><code>PointLayout {
[Intensity;U16 @ offset 0]
[Position3D;Vec3<f32> @ offset 2]
[CUSTOM_ATTRIBUTE;F32 @ offset 14]
}
</code></pre>
</main>
<nav class="nav-wrapper" aria-label="Page navigation">
<!-- Mobile navigation buttons -->
<a rel="prev" href="data_model.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next" href="point_buffers.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
<div style="clear: both"></div>
</nav>
</div>
</div>
<nav class="nav-wide-wrapper" aria-label="Page navigation">
<a rel="prev" href="data_model.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next" href="point_buffers.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
</nav>
</div>
<script type="text/javascript">
window.playground_line_numbers = true;
</script>
<script type="text/javascript">
window.playground_copyable = true;
</script>
<script src="ace.js" type="text/javascript" charset="utf-8"></script>
<script src="editor.js" type="text/javascript" charset="utf-8"></script>
<script src="mode-rust.js" type="text/javascript" charset="utf-8"></script>
<script src="theme-dawn.js" type="text/javascript" charset="utf-8"></script>
<script src="theme-tomorrow_night.js" type="text/javascript" charset="utf-8"></script>
<script src="elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
<script src="mark.min.js" type="text/javascript" charset="utf-8"></script>
<script src="searcher.js" type="text/javascript" charset="utf-8"></script>
<script src="clipboard.min.js" type="text/javascript" charset="utf-8"></script>
<script src="highlight.js" type="text/javascript" charset="utf-8"></script>
<script src="book.js" type="text/javascript" charset="utf-8"></script>
<!-- Custom JS scripts -->
</body>
</html>