Project
Loading...
Searching...
No Matches
test_Decoder.cxx
Go to the documentation of this file.
1// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#include "AODNative/Decoder.h"
13#include "AODNative/Errors.h"
14#include "RootFraming.h"
15#include <catch_amalgamated.hpp>
16#include <bit>
17
18using namespace o2::aod::native;
19using namespace o2::aod::native::test;
20
21namespace
22{
23// Serializes values big-endian, as ROOT stores them.
24template <typename T>
25std::vector<uint8_t> bigEndian(std::vector<T> const& values)
26{
28 for (auto v : values) {
29 w.put(std::bit_cast<std::conditional_t<sizeof(T) == 4, uint32_t, std::conditional_t<sizeof(T) == 8, uint64_t, std::conditional_t<sizeof(T) == 2, uint16_t, uint8_t>>>>(v));
30 }
31 return w.bytes;
32}
33
35struct BranchBuilder {
36 std::vector<uint8_t> file = std::vector<uint8_t>(100, 0); // some leading bytes, so seeks are non-zero
37 BranchInfo branch;
38
39 BranchBuilder(char leafClass, int fixedLen = 1)
40 {
41 branch.name = "fX";
42 branch.leafClass = leafClass;
43 branch.isUnsigned = false;
44 branch.fixedLen = fixedLen;
45 branch.entries = branch.totBytes = branch.zipBytes = 0;
46 }
47
48 void add(Algo algo, std::vector<uint8_t> const& values, int32_t nEntries, size_t tail = 0, size_t chunkSize = 0xffffff, bool ioBits = false)
49 {
50 auto basket = makeBasket(algo, values, nEntries, tail, chunkSize, ioBits);
51 branch.baskets.push_back(BasketRef{file.size(), (uint32_t)basket.size(), branch.entries, nEntries, false, {}});
52 file.insert(file.end(), basket.begin(), basket.end());
53 branch.entries += nEntries;
54 }
55
56 void addEmbedded(std::vector<uint8_t> const& values, int32_t nEntries)
57 {
58 branch.baskets.push_back(BasketRef{0, 0, branch.entries, nEntries, true, values});
59 branch.entries += nEntries;
60 }
61};
62} // namespace
63
64TEST_CASE("byteswapInPlace")
65{
66 std::vector<uint8_t> v{1, 2, 3, 4, 5, 6, 7, 8};
67 byteswapInPlace(v.data(), 2, 4);
68 CHECK(v == std::vector<uint8_t>{4, 3, 2, 1, 8, 7, 6, 5});
69 byteswapInPlace(v.data(), 1, 8);
70 CHECK(v == std::vector<uint8_t>{5, 6, 7, 8, 1, 2, 3, 4});
71 byteswapInPlace(v.data(), 4, 2);
72 CHECK(v == std::vector<uint8_t>{6, 5, 8, 7, 2, 1, 4, 3});
73}
74
75TEST_CASE("float values across codecs, uncompressed and embedded baskets")
76{
77 BranchBuilder b{'F'};
78 std::vector<float> expected;
79 int n = 0;
80 for (auto algo : {Algo::ZSTD, Algo::None, Algo::ZLIB, Algo::LZMA, Algo::LZ4}) {
81 std::vector<float> values;
82 for (int i = 0; i < 1000; ++i) {
83 values.push_back(n++ * 0.5f);
84 }
85 // Entry-offset tail after the values must never reach the destination.
86 b.add(algo, bigEndian(values), values.size(), 4004);
87 expected.insert(expected.end(), values.begin(), values.end());
88 }
89 std::vector<float> tail{-1.f, -2.f, -3.f};
90 b.addEmbedded(bigEndian(tail), tail.size());
91 expected.insert(expected.end(), tail.begin(), tail.end());
92
93 for (bool mappable : {true, false}) {
94 for (int threads : {1, 4}) {
95 MemoryByteSource source{b.file, mappable};
96 Options options;
97 options.threads = threads;
98 BranchDecoder decoder{source, options};
99 std::vector<float> out(expected.size(), 42.f);
100 decoder.decodeValues(b.branch, (uint8_t*)out.data(), out.size() * sizeof(float));
101 CHECK(out == expected);
102 }
103 }
104}
105
106TEST_CASE("baskets with fIOBits in their header decode like the others")
107{
108 BranchBuilder b{'F'};
109 std::vector<float> expected;
110 for (auto [algo, ioBits] : {std::pair{Algo::ZSTD, true}, std::pair{Algo::None, false}, std::pair{Algo::LZ4, true}}) {
111 std::vector<float> values;
112 for (int i = 0; i < 100; ++i) {
113 values.push_back(expected.size() + i);
114 }
115 b.add(algo, bigEndian(values), values.size(), 404, 0xffffff, ioBits);
116 expected.insert(expected.end(), values.begin(), values.end());
117 }
119 BranchDecoder decoder{source, Options{}};
120 std::vector<float> out(expected.size(), 42.f);
121 decoder.decodeValues(b.branch, (uint8_t*)out.data(), out.size() * sizeof(float));
122 CHECK(out == expected);
123}
124
125TEST_CASE("decoding stops exactly at the value region for every codec")
126{
127 // A poisoned entry-offset tail after the values: nothing past fLast - fKeylen
128 // may reach the destination, which is followed by guard bytes. Sizes span
129 // several zstd blocks and compression chunks.
130 for (auto algo : {Algo::ZSTD, Algo::ZLIB, Algo::LZMA, Algo::LZ4, Algo::None}) {
131 for (size_t chunkSize : {size_t(100000), size_t(0xffffff)}) {
132 INFO("algorithm " << (int)algo << ", chunk size " << chunkSize);
133 BranchBuilder b{'D'};
134 std::vector<double> values(70001);
135 for (size_t i = 0; i < values.size(); ++i) {
136 values[i] = i * 0.25 - 7;
137 }
138 b.add(algo, bigEndian(values), values.size(), 8 * values.size() + 4, chunkSize);
140 BranchDecoder decoder{source, Options{}};
141 std::vector<double> out(values.size() + 16, 1e300);
142 decoder.decodeValues(b.branch, (uint8_t*)out.data(), values.size() * sizeof(double));
143 CHECK(std::equal(values.begin(), values.end(), out.begin()));
144 CHECK(std::all_of(out.begin() + values.size(), out.end(), [](double v) { return v == 1e300; }));
145 }
146 }
147}
148
149TEST_CASE("fixed-size arrays and wrong destination sizes")
150{
151 BranchBuilder b{'S', 3};
152 std::vector<int16_t> values{1, 2, 3, -4, -5, -6};
153 b.add(Algo::ZSTD, bigEndian(values), 2);
155 BranchDecoder decoder{source, Options{}};
156 std::vector<int16_t> out(6);
157 decoder.decodeValues(b.branch, (uint8_t*)out.data(), 12);
158 CHECK(out == values);
159 CHECK_THROWS_AS(decoder.decodeValues(b.branch, (uint8_t*)out.data(), 10), FormatError);
160}
161
162TEST_CASE("booleans are packed across basket and window boundaries")
163{
164 // Basket sizes chosen so that boundaries fall inside bitmap bytes, and one
165 // basket is larger than the 64 KiB window.
166 BranchBuilder b{'O'};
167 std::vector<bool> expected;
168 for (int size : {3, 13, (int)BranchDecoder::kBooleanWindow + 5, 1, 7}) {
169 std::vector<uint8_t> bytes;
170 for (int i = 0; i < size; ++i) {
171 bool v = ((expected.size() * 7) % 3) == 0;
172 expected.push_back(v);
173 // Any non-zero byte is true.
174 bytes.push_back(v ? (i % 2 ? 1 : 0x80) : 0);
175 }
176 b.add(Algo::ZSTD, bytes, size, 8, 30000);
177 }
179 BranchDecoder decoder{source, Options{}};
180 std::vector<uint8_t> bitmap((expected.size() + 7) / 8, 0xFF);
181 decoder.decodeBooleans(b.branch, bitmap.data(), bitmap.size());
182 for (size_t i = 0; i < expected.size(); ++i) {
183 INFO("value " << i);
184 CHECK(((bitmap[i / 8] >> (i % 8)) & 1) == expected[i]);
185 }
186 // Padding bits are zero.
187 for (size_t i = expected.size(); i < bitmap.size() * 8; ++i) {
188 CHECK(((bitmap[i / 8] >> (i % 8)) & 1) == 0);
189 }
190}
191
192TEST_CASE("variable-length arrays")
193{
194 BranchBuilder sizes{'I'};
195 sizes.branch.name = "fY_size";
196 std::vector<int32_t> s1{2, 0, 3}, s2{1};
197 sizes.add(Algo::ZLIB, bigEndian(s1), 3);
198 sizes.add(Algo::None, bigEndian(s2), 1);
199
200 // Values baskets do not need to align with size baskets.
201 BranchBuilder values{'D'};
202 values.branch.name = "fY";
203 values.branch.countBranch = "fY_size";
204 std::vector<double> v1{1.5, 2.5}, v2{3.5, 4.5, 5.5, 6.5};
205 values.add(Algo::LZ4, bigEndian(v1), 2);
206 values.add(Algo::ZSTD, bigEndian(v2), 2);
207
208 std::vector<uint8_t> file = sizes.file;
209 // Place the values baskets after the size baskets.
210 auto shift = file.size() - 100;
211 file.insert(file.end(), values.file.begin() + 100, values.file.end());
212 for (auto& basket : values.branch.baskets) {
213 basket.seek += shift;
214 }
215
217 BranchDecoder decoder{source, Options{}};
218 auto offsets = decoder.decodeOffsets(sizes.branch);
219 CHECK(offsets == std::vector<int32_t>{0, 2, 2, 5, 6});
220 std::vector<double> out(6);
221 decoder.decodeVLAValues(values.branch, offsets, (uint8_t*)out.data(), out.size() * sizeof(double));
222 CHECK(out == std::vector<double>{1.5, 2.5, 3.5, 4.5, 5.5, 6.5});
223}
224
225TEST_CASE("inconsistent baskets raise CorruptData")
226{
227 BranchBuilder b{'I'};
228 std::vector<int32_t> values{1, 2, 3};
229 b.add(Algo::ZSTD, bigEndian(values), 3);
231 BranchDecoder decoder{source, Options{}};
232 std::vector<int32_t> out(3);
233
234 SECTION("entry count in the header differs from the TBranch")
235 {
236 auto branch = b.branch;
237 branch.baskets[0].nEntries = 2;
238 branch.entries = 2;
239 CHECK_THROWS_AS(decoder.decodeValues(branch, (uint8_t*)out.data(), 8), CorruptData);
240 }
241 SECTION("baskets do not cover the entries")
242 {
243 auto branch = b.branch;
244 branch.entries = 4;
245 std::vector<int32_t> big(4);
246 CHECK_THROWS_AS(decoder.decodeValues(branch, (uint8_t*)big.data(), 16), CorruptData);
247 }
248 SECTION("basket size differs from the TBranch")
249 {
250 auto branch = b.branch;
251 branch.baskets[0].nbytes -= 1;
252 CHECK_THROWS_AS(decoder.decodeValues(branch, (uint8_t*)out.data(), 12), CorruptData);
253 }
254 SECTION("basket location past the end of the file")
255 {
256 auto branch = b.branch;
257 branch.baskets[0].seek = b.file.size();
258 MemoryByteSource unmapped{b.file, false};
259 BranchDecoder d2{unmapped, Options{}};
260 CHECK_THROWS_AS(d2.decodeValues(branch, (uint8_t*)out.data(), 12), CorruptData);
261 }
262}
@ INFO
Definition FairLogger.h:32
unsigned short int uint16_t
int32_t i
configStandalone Light OpenGL display display_light Apply smoothing to points Enable Z buffer Highlight links Highlight seeds Highlight sector tracks Highlight final tracks Show individual Show related Highlight grid Show only individual collision Color clusters belonging to track objects Set line Enable drawing TPC data Enable drawing TOF data Enable drawing magnetic field Set field line step count Invert colors configStandalone Heavy OpenGL display display_heavy Show tracks with ITS contribution Show tracks with TOF contribution Enable propagation of loopers Show clusters at nominal x of pad row for early transformed data Split CE tracks when they cross the central Mark clusters Mark adjacent clusters Hide rejected tracks Apply filter on tracks to be displayed Separate increase X h configStandalone Camera window renderer OpenGL display display_camera Orient the camera such that the y axis is always upwards Full Screen MultiSample Anti Downsampling Anti Enable Vertical Sync Target frame or if the backend cannot limit to the refresh Use OpenGL indirect draws to reduce number of draw calls Display h configStandalone Vulkan display display_vulkan Have uniform buffers in host accessible device memory h configStandalone Internal display display_internal Number of updates the Qt GUI may send ahead of the last one the display has before it waits for the display Opacity of the console background Number of console lines the history is shortened to this when it reaches twice the size Time in seconds an info message fades out after setInfoTime h OpenGL display display Font size Do not use Freetype for font Renderer for event Frontend to use for event ROOT macros used as track filter Console script file
int32_t nEntries
Definition Objects.cxx:40
size_t bytes
static constexpr size_t kBooleanWindow
Size of the boolean window, the only decode-side scratch memory.
Definition Decoder.h:35
A file made of concatenated in-memory buffers.
#define CHECK
GLdouble n
Definition glcorearb.h:1982
GLsizeiptr size
Definition glcorearb.h:659
GLuint GLsizei const GLuint const GLintptr * offsets
Definition glcorearb.h:2595
const GLdouble * v
Definition glcorearb.h:832
GLuint GLsizei const GLuint const GLintptr const GLsizeiptr * sizes
Definition glcorearb.h:2595
GLuint GLfloat GLfloat GLfloat GLfloat GLfloat GLfloat GLfloat GLfloat s1
Definition glcorearb.h:5034
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei GLsizei GLchar * source
Definition glcorearb.h:798
GLenum GLsizei GLsizei GLint * values
Definition glcorearb.h:1576
GLfloat GLfloat v1
Definition glcorearb.h:812
GLubyte GLubyte GLubyte GLubyte w
Definition glcorearb.h:852
GLfloat GLfloat GLfloat v2
Definition glcorearb.h:813
std::vector< uint8_t > makeBasket(Algo algo, std::span< const uint8_t > values, int32_t nevbuf, size_t tailBytes=0, size_t chunkSize=0xffffff, bool ioBits=false)
void byteswapInPlace(uint8_t *data, size_t count, size_t typeSize)
Byteswaps count big-endian values of typeSize bytes in place.
Definition Decoder.cxx:44
int32_t const char * file
std::vector< BasketRef > baskets
Definition Objects.h:46
std::map< std::string, ID > expected
TEST_CASE("byteswapInPlace")