Project
Loading...
Searching...
No Matches
RootFraming.h
Go to the documentation of this file.
1// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#ifndef O2_FRAMEWORK_AODNATIVE_TEST_ROOTFRAMING_H_
13#define O2_FRAMEWORK_AODNATIVE_TEST_ROOTFRAMING_H_
14
15// Helpers to build ROOT-framed compressed payloads and TBasket keys in memory,
16// so that codecs and the decoder can be tested without ROOT or a file.
17
19#include "AODNative/Codecs.h"
20#include "AODNative/Errors.h"
21#include <cstring>
22#include <lz4.h>
23#include <lzma.h>
24#include <stdexcept>
25#include <string>
26#include <vector>
27#include <zlib.h>
28#include <zstd.h>
29
31{
32
33enum struct Algo { ZSTD,
34 ZLIB,
35 LZMA,
36 LZ4,
37 None };
38
39inline std::vector<uint8_t> compressChunk(Algo algo, std::span<const uint8_t> in)
40{
41 std::vector<uint8_t> out;
42 switch (algo) {
43 case Algo::ZSTD: {
44 out.resize(ZSTD_compressBound(in.size()));
45 out.resize(ZSTD_compress(out.data(), out.size(), in.data(), in.size(), 5));
46 } break;
47 case Algo::ZLIB: {
48 uLongf n = compressBound(in.size());
49 out.resize(n);
50 compress2(out.data(), &n, in.data(), in.size(), 6);
51 out.resize(n);
52 } break;
53 case Algo::LZMA: {
54 out.resize(lzma_stream_buffer_bound(in.size()));
55 size_t pos = 0;
56 lzma_easy_buffer_encode(6, LZMA_CHECK_CRC32, nullptr, in.data(), in.size(), out.data(), &pos, out.size());
57 out.resize(pos);
58 } break;
59 case Algo::LZ4: {
60 std::vector<uint8_t> block(LZ4_compressBound(in.size()));
61 block.resize(LZ4_compress_default((char const*)in.data(), (char*)block.data(), in.size(), block.size()));
62 auto checksum = xxh64(block);
63 for (int i = 7; i >= 0; --i) {
64 out.push_back((checksum >> (8 * i)) & 0xff);
65 }
66 out.insert(out.end(), block.begin(), block.end());
67 } break;
68 case Algo::None:
69 throw std::logic_error("no chunk for uncompressed data");
70 }
71 return out;
72}
73
76inline std::vector<uint8_t> rootCompress(Algo algo, std::span<const uint8_t> in, size_t chunkSize = 0xffffff)
77{
78 static const char* tags[] = {"ZS", "ZL", "XZ", "L4"};
79 std::vector<uint8_t> out;
80 for (size_t pos = 0; pos < in.size(); pos += chunkSize) {
81 auto piece = in.subspan(pos, std::min(chunkSize, in.size() - pos));
82 auto c = compressChunk(algo, piece);
83 out.push_back(tags[(int)algo][0]);
84 out.push_back(tags[(int)algo][1]);
85 out.push_back(algo == Algo::ZLIB ? 8 : 1);
86 for (auto v : {c.size(), piece.size()}) {
87 out.push_back(v & 0xff);
88 out.push_back((v >> 8) & 0xff);
89 out.push_back((v >> 16) & 0xff);
90 }
91 out.insert(out.end(), c.begin(), c.end());
92 }
93 return out;
94}
95
97 std::vector<uint8_t> bytes;
98 template <typename T>
99 void put(T v)
100 {
101 for (int i = sizeof(T) - 1; i >= 0; --i) {
102 bytes.push_back((uint64_t(v) >> (8 * i)) & 0xff);
103 }
104 }
105 void string(std::string const& s)
106 {
107 bytes.push_back(s.size());
108 bytes.insert(bytes.end(), s.begin(), s.end());
109 }
110};
111
116inline std::vector<uint8_t> makeBasket(Algo algo, std::span<const uint8_t> values, int32_t nevbuf, size_t tailBytes = 0, size_t chunkSize = 0xffffff, bool ioBits = false)
117{
118 std::vector<uint8_t> raw(values.begin(), values.end());
119 // The tail is poisoned: a decoder that reads past the value region sees it.
120 raw.resize(values.size() + tailBytes, 0xEE);
121 std::vector<uint8_t> payload = algo == Algo::None ? raw : rootCompress(algo, raw, chunkSize);
122
124 auto build = [&](uint16_t keylen) {
125 h.bytes.clear();
126 h.put<uint32_t>(keylen + payload.size()); // fNbytes
127 h.put<int16_t>(4); // key version
128 h.put<uint32_t>(raw.size()); // fObjlen
129 h.put<uint32_t>(0); // fDatime
130 h.put<uint16_t>(keylen);
131 h.put<uint16_t>(1); // fCycle
132 h.put<uint32_t>(0); // fSeekKey
133 h.put<uint32_t>(0); // fSeekPdir
134 h.string("TBasket"); // class, name, title
135 h.string("fX");
136 h.string("O2track");
137 h.put<uint16_t>(3); // fVersion
138 h.put<int32_t>(32000); // fBufferSize
139 h.put<int32_t>(ioBits ? -4 : 0); // fNevBufSize
140 if (ioBits) {
141 h.put<uint8_t>(0); // fIOBits
142 }
143 h.put<int32_t>(nevbuf); // fNevBuf
144 h.put<int32_t>(keylen + values.size()); // fLast
145 h.put<uint8_t>(tailBytes ? 1 : 2); // flag
146 };
147 build(0);
148 auto keylen = (uint16_t)h.bytes.size();
149 build(keylen);
150 h.bytes.insert(h.bytes.end(), payload.begin(), payload.end());
151 return h.bytes;
152}
153
156{
157 public:
158 explicit MemoryByteSource(std::vector<uint8_t> data, bool mappable = true) : mData(std::move(data)), mMappable(mappable) {}
159 uint64_t size() const override { return mData.size(); }
160 void readInto(uint64_t off, size_t len, void* dst) override
161 {
162 if (off > mData.size() || len > mData.size() - off) {
163 throw CorruptData("memory", off, "read past the end");
164 }
165 if (len == 0) {
166 return;
167 }
168 std::memcpy(dst, mData.data() + off, len);
169 ++mCalls;
170 mBytes += len;
171 }
172 void readvInto(std::span<ReadRequest> requests) override
173 {
174 for (auto& r : requests) {
175 readInto(r.offset, r.length, r.dst);
176 }
177 }
178 std::span<const uint8_t> tryMap(uint64_t off, size_t len) override
179 {
180 if (!mMappable) {
181 return {};
182 }
183 if (off > mData.size() || len > mData.size() - off) {
184 throw CorruptData("memory", off, "map past the end");
185 }
186 return std::span<const uint8_t>(mData).subspan(off, len);
187 }
188 uint64_t bytesRead() const override { return mBytes; }
189 uint64_t readCalls() const override { return mCalls; }
190 std::string url() const override { return "memory"; }
191
192 private:
193 std::vector<uint8_t> mData;
194 bool mMappable;
195 uint64_t mBytes = 0, mCalls = 0;
196};
197
198} // namespace o2::aod::native::test
199
200#endif // O2_FRAMEWORK_AODNATIVE_TEST_ROOTFRAMING_H_
unsigned short int uint16_t
int32_t i
o2::raw::RawFileWriter * raw
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h configStandalone Processing proc_rtctech Use a file lock to serialize access to the cache folder int32_t
uint16_t pos
Definition RawData.h:3
uint32_t c
Definition RawData.h:2
Class for time synchronization of RawReader instances.
A file made of concatenated in-memory buffers.
std::span< const uint8_t > tryMap(uint64_t off, size_t len) override
A view of the bytes without copying them, if the backend can provide it.
void readInto(uint64_t off, size_t len, void *dst) override
Throws IoError, or CorruptData when the range is past the end of the file.
MemoryByteSource(std::vector< uint8_t > data, bool mappable=true)
uint64_t bytesRead() const override
uint64_t readCalls() const override
std::string url() const override
void readvInto(std::span< ReadRequest > requests) override
GLdouble n
Definition glcorearb.h:1982
const GLdouble * v
Definition glcorearb.h:832
GLenum GLenum dst
Definition glcorearb.h:1767
GLenum GLsizei GLsizei GLint * values
Definition glcorearb.h:1576
GLenum GLenum GLsizei len
Definition glcorearb.h:4232
std::vector< uint8_t > compressChunk(Algo algo, std::span< const uint8_t > in)
Definition RootFraming.h:39
std::vector< uint8_t > makeBasket(Algo algo, std::span< const uint8_t > values, int32_t nevbuf, size_t tailBytes=0, size_t chunkSize=0xffffff, bool ioBits=false)
std::vector< uint8_t > rootCompress(Algo algo, std::span< const uint8_t > in, size_t chunkSize=0xffffff)
Definition RootFraming.h:76
uint64_t xxh64(std::span< const uint8_t > data)
XXH64 with seed 0, as used for the LZ4 chunk checksum.
Definition Codecs.cxx:319
void string(std::string const &s)
Marks an empty item in the context.