22#define ZSTD_STATIC_LINKING_ONLY
30constexpr size_t kChunkHeaderSize = 9;
31constexpr size_t kLZ4ChecksumSize = 8;
33uint32_t le24(
uint8_t const* p) {
return p[0] | (p[1] << 8) | (p[2] << 16); }
41 lzma_stream
lzma = LZMA_STREAM_INIT;
60 : mSrc(
src), mObjLen(objLen), mFile(
file), mFileOffset(fileOffset), mCodec(
std::make_unique<
Codec>())
66void PayloadDecompressor::nextChunk()
68 auto fail = [
this](std::string
const& what) {
throw CorruptData(mFile, mFileOffset + mSrcPos, what); };
69 if (mSrc.size() - mSrcPos < kChunkHeaderSize) {
72 auto*
h = mSrc.data() + mSrcPos;
75 size_t cSize = le24(
h + 3);
76 mChunkOut = le24(
h + 6);
78 if (cSize > mSrc.size() - mSrcPos - kChunkHeaderSize) {
79 fail(
"compressed chunk of " +
std::to_string(cSize) +
" bytes past the end of the payload");
81 if (mChunkOut == 0 || mChunkOut > mObjLen - mProduced) {
82 fail(
"chunk of " +
std::to_string(mChunkOut) +
" uncompressed bytes does not fit the object length");
84 mChunkIn = mSrc.subspan(mSrcPos + kChunkHeaderSize, cSize);
85 auto& codec = *mCodec;
87 if (mTag[0] ==
'Z' && mTag[1] ==
'S') {
89 codec.zstd = ZSTD_createDCtx();
91 ZSTD_DCtx_reset(codec.zstd, ZSTD_reset_session_only);
92 }
else if (mTag[0] ==
'Z' && mTag[1] ==
'L') {
94 inflateReset(&codec.zlib);
96 if (inflateInit(&codec.zlib) != Z_OK) {
97 fail(
"cannot initialise zlib");
99 codec.zlibInit =
true;
101 codec.zlib.next_in = (Bytef*)mChunkIn.data();
102 codec.zlib.avail_in = mChunkIn.size();
103 }
else if (mTag[0] ==
'X' && mTag[1] ==
'Z') {
104 if (codec.lzmaInit) {
105 lzma_end(&codec.lzma);
107 codec.lzma = LZMA_STREAM_INIT;
108 if (lzma_stream_decoder(&codec.lzma, UINT64_MAX, 0) != LZMA_OK) {
109 fail(
"cannot initialise liblzma");
111 codec.lzmaInit =
true;
112 codec.lzma.next_in = mChunkIn.data();
113 codec.lzma.avail_in = mChunkIn.size();
114 }
else if (mTag[0] ==
'L' && mTag[1] ==
'4') {
115 if (cSize < kLZ4ChecksumSize) {
116 fail(
"LZ4 chunk too small for its checksum");
119 for (
size_t i = 0;
i < kLZ4ChecksumSize; ++
i) {
122 mChunkIn = mChunkIn.subspan(kLZ4ChecksumSize);
124 fail(
"LZ4 checksum mismatch");
126 codec.lz4Partial =
false;
128 throw UnsupportedLayout(std::string(
"compression algorithm \"") + mTag[0] + mTag[1] +
"\" in " + mFile);
130 mSrcPos += kChunkHeaderSize + cSize;
133size_t PayloadDecompressor::produceFromChunk(
uint8_t*
dst,
size_t n)
135 auto fail = [
this](std::string
const& what) {
throw CorruptData(mFile, mFileOffset + mSrcPos, what); };
136 auto& codec = *mCodec;
137 bool wholeChunk = mChunkDone == 0 &&
n == mChunkOut;
138 if (mTag[0] ==
'Z' && mTag[1] ==
'S') {
140 auto r = ZSTD_decompressDCtx(codec.zstd,
dst,
n, mChunkIn.data(), mChunkIn.size());
141 if (ZSTD_isError(
r) ||
r !=
n) {
142 fail(std::string(
"zstd: ") + (ZSTD_isError(
r) ? ZSTD_getErrorName(
r) :
"size mismatch"));
146 if (mPrefixOnly && mChunkDone == 0) {
147 return zstdPrefix(
dst,
n);
151 ZSTD_outBuffer out{
dst,
n, 0};
152 while (out.pos < out.size) {
153 ZSTD_inBuffer in{mChunkIn.data(), mChunkIn.size(), 0};
154 auto r = ZSTD_decompressStream(codec.zstd, &out, &in);
155 mChunkIn = mChunkIn.subspan(in.pos);
156 if (ZSTD_isError(
r)) {
157 fail(std::string(
"zstd: ") + ZSTD_getErrorName(
r));
159 if (out.pos < out.size && (
r == 0 || (in.pos == 0 && mChunkIn.empty()))) {
160 fail(
"zstd chunk ended early");
165 if (mTag[0] ==
'Z' && mTag[1] ==
'L') {
166 auto&
z = codec.zlib;
169 while (
z.avail_out > 0) {
170 auto r = inflate(&
z, Z_NO_FLUSH);
171 if (
r == Z_STREAM_END &&
z.avail_out > 0) {
172 fail(
"zlib chunk ended early");
174 if (
r != Z_OK &&
r != Z_STREAM_END) {
180 if (mTag[0] ==
'X' && mTag[1] ==
'Z') {
181 auto&
s = codec.lzma;
184 while (
s.avail_out > 0) {
185 auto r = lzma_code(&
s, LZMA_RUN);
186 if (
r == LZMA_STREAM_END &&
s.avail_out > 0) {
187 fail(
"xz chunk ended early");
189 if (
r != LZMA_OK &&
r != LZMA_STREAM_END) {
201 auto r = LZ4_decompress_safe((
char const*)mChunkIn.data(), (
char*)
dst, mChunkIn.size(),
n);
202 if (
r < 0 || (
size_t)
r !=
n) {
203 fail(
"LZ4 chunk size mismatch");
207 if (mScratch.empty() && mChunkDone == 0) {
208 auto r = LZ4_decompress_safe_partial((
char const*)mChunkIn.data(), (
char*)
dst, mChunkIn.size(),
n,
n);
209 if (
r < 0 || (
size_t)
r !=
n) {
210 fail(
"LZ4 decompression failed");
212 codec.lz4Partial =
true;
215 if (mScratch.empty()) {
216 mScratch.resize(mChunkOut);
217 auto r = LZ4_decompress_safe((
char const*)mChunkIn.data(), (
char*)mScratch.data(), mChunkIn.size(), mChunkOut);
218 if (
r < 0 || (
size_t)
r != mChunkOut) {
219 fail(
"LZ4 decompression failed");
222 std::memcpy(
dst, mScratch.data() + mChunkDone,
n);
231size_t PayloadDecompressor::zstdPrefix(
uint8_t*
dst,
size_t n)
233 auto fail = [
this](std::string
const& what) {
throw CorruptData(mFile, mFileOffset + mSrcPos, what); };
234 auto* dctx = mCodec->zstd;
235 if (ZSTD_isError(ZSTD_decompressBegin(dctx))) {
236 fail(
"cannot initialise zstd");
240 bool inScratch =
false;
241 size_t scratchUsed = 0;
242 while (written <
n) {
243 auto need = ZSTD_nextSrcSizeToDecompress(dctx);
245 fail(
"zstd chunk ended early");
247 if (need > in.size()) {
248 fail(
"truncated zstd chunk");
251 if (!inScratch &&
n - written >= ZSTD_BLOCKSIZE_MAX) {
252 r = ZSTD_decompressContinue(dctx,
dst + written,
n - written, in.data(), need);
253 if (!ZSTD_isError(
r)) {
258 mScratch.resize(2 * ZSTD_BLOCKSIZE_MAX);
261 r = ZSTD_decompressContinue(dctx, mScratch.data() + scratchUsed, mScratch.size() - scratchUsed, in.data(), need);
262 if (!ZSTD_isError(
r)) {
264 if (written + scratchUsed >=
n) {
265 std::memcpy(
dst + written, mScratch.data(),
n - written);
270 if (ZSTD_isError(
r)) {
271 fail(std::string(
"zstd: ") + ZSTD_getErrorName(
r));
273 in = in.subspan(need);
276 mScratch.shrink_to_fit();
282 if (mProduced != 0) {
283 throw std::logic_error(
"readPrefix must be the first read of a payload");
291 if (mPrefixOnly && mProduced != 0) {
292 throw std::logic_error(
"no read can follow readPrefix");
294 if (
n > mObjLen - mProduced) {
298 if (mChunkDone == mChunkOut) {
301 auto k = std::min(
n, mChunkOut - mChunkDone);
302 produceFromChunk(
dst, k);
313 auto n = std::min<size_t>(objLen, maxOut);
315 d.readPrefix(
dst,
n);
321 constexpr uint64_t P1 = 11400714785074694791ULL;
322 constexpr uint64_t P2 = 14029467366897019727ULL;
323 constexpr uint64_t P3 = 1609587929392839161ULL;
324 constexpr uint64_t P4 = 9650029242287828579ULL;
325 constexpr uint64_t P5 = 2870177450012600261ULL;
326 auto read64 = [](
uint8_t const* p) {
328 std::memcpy(&
v, p, 8);
329 return std::endian::native == std::endian::little ?
v :
bswap(
v);
331 auto read32 = [](
uint8_t const* p) {
333 std::memcpy(&
v, p, 4);
334 return std::endian::native == std::endian::little ?
v :
bswap(
v);
338 return std::rotl(acc, 31) * P1;
342 return acc * P1 + P4;
345 auto const* p =
data.data();
346 auto const*
end = p +
data.size();
348 if (
data.size() >= 32) {
350 for (;
end - p >= 32; p += 32) {
351 v1 = round(
v1, read64(p));
352 v2 = round(
v2, read64(p + 8));
353 v3 = round(
v3, read64(p + 16));
354 v4 = round(v4, read64(p + 24));
356 h = std::rotl(
v1, 1) + std::rotl(
v2, 7) + std::rotl(
v3, 12) + std::rotl(v4, 18);
362 for (;
end - p >= 8; p += 8) {
363 h ^= round(0, read64(p));
364 h = std::rotl(
h, 27) * P1 + P4;
368 h = std::rotl(
h, 23) * P2 + P3;
371 for (; p <
end; ++p) {
373 h = std::rotl(
h, 11) * P1;
configStandalone Light OpenGL display display_light Apply smoothing to points Enable Z buffer Highlight links Highlight seeds Highlight sector tracks Highlight final tracks Show individual Show related Highlight grid Show only individual collision Color clusters belonging to track objects Set line Enable drawing TPC data Enable drawing TOF data Enable drawing magnetic field Set field line step count Invert colors configStandalone Heavy OpenGL display display_heavy Show tracks with ITS contribution Show tracks with TOF contribution Enable propagation of loopers Show clusters at nominal x of pad row for early transformed data Split CE tracks when they cross the central Mark clusters Mark adjacent clusters Hide rejected tracks Apply filter on tracks to be displayed Separate increase X h configStandalone Camera window renderer OpenGL display display_camera Orient the camera such that the y axis is always upwards Full Screen MultiSample Anti Downsampling Anti Enable Vertical Sync Target frame or if the backend cannot limit to the refresh Use OpenGL indirect draws to reduce number of draw calls Display h configStandalone Vulkan display display_vulkan Have uniform buffers in host accessible device memory h configStandalone Internal display display_internal Number of updates the Qt GUI may send ahead of the last one the display has before it waits for the display Opacity of the console background Number of console lines the history is shortened to this when it reaches twice the size Time in seconds an info message fades out after setInfoTime h OpenGL display display Font size Do not use Freetype for font Renderer for event Frontend to use for event ROOT macros used as track filter Console script file
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events d
Class for time synchronization of RawReader instances.
PayloadDecompressor(std::span< const uint8_t > src, uint32_t objLen, std::string const &file, uint64_t fileOffset)
void read(uint8_t *dst, size_t n)
void readPrefix(uint8_t *dst, size_t n)
GLfloat GLfloat GLfloat GLfloat v3
GLfloat GLfloat GLfloat v2
GLdouble GLdouble GLdouble z
size_t decompressBounded(std::span< const uint8_t > src, uint32_t objLen, uint8_t *dst, size_t maxOut, std::string const &file="<memory>", uint64_t fileOffset=0)
uint64_t xxh64(std::span< const uint8_t > data)
XXH64 with seed 0, as used for the LZ4 chunk checksum.
std::string to_string(gsl::span< T, Size > span)
std::map< std::string, ID > expected