Project
Loading...
Searching...
No Matches
Container.cxx
Go to the documentation of this file.
1// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#include "AODNative/Container.h"
13#include "AODNative/Codecs.h"
14#include "AODNative/Errors.h"
15#include "Cursor.h"
16#include <algorithm>
17#include <regex>
18#include <unordered_set>
19
20namespace o2::aod::native
21{
22
23namespace
24{
25// Big enough for the file header and for a TDirectory record (including its UUID).
26constexpr size_t kHeaderReadSize = 128;
27constexpr size_t kDirectoryReadSize = 64;
28// Largest gap between the TTrees of a DF still read together with them.
29constexpr uint64_t kMetadataGap = 4 * 1024;
30// Smallest possible key header: 32-bit seeks and three empty strings.
31constexpr size_t kMinKeySize = 29;
32// Metadata objects (trees, maps) in AO2Ds are a few MB at most. A larger
33// object length can only come from a corrupted key, so refuse to allocate it.
34constexpr uint32_t kMaxMetadataSize = 256 * 1024 * 1024;
35
36std::vector<uint8_t> readRange(ByteSource& source, uint64_t off, size_t len)
37{
38 // Check before allocating: the length comes from the file.
39 if (off > source.size() || len > source.size() - off) {
40 throw CorruptData(source.url(), off, "range of " + std::to_string(len) + " bytes past the end of the file");
41 }
42 std::vector<uint8_t> buf(len);
43 source.readInto(off, len, buf.data());
44 return buf;
45}
46} // namespace
47
48RootFile::RootFile(std::shared_ptr<ByteSource> source)
49 : mSource(std::move(source)),
50 mUrl(mSource->url())
51{
52 auto buf = readRange(*mSource, 0, std::min<uint64_t>(kHeaderReadSize, mSource->size()));
53 Cursor c{buf, mUrl};
54 auto magic = c.bytes(4);
55 if (std::memcmp(magic.data(), "root", 4) != 0) {
56 throw FormatError(mUrl + " is not a ROOT file");
57 }
58 auto& h = mHeader;
59 h.version = c.i32();
60 h.begin = c.i32();
61 bool big = h.version >= 1000000;
62 h.end = big ? c.u64() : c.u32();
63 h.seekFree = big ? c.u64() : c.u32();
64 h.nbytesFree = c.i32();
65 h.nfree = c.i32();
66 h.nbytesName = c.i32();
67 h.units = c.u8();
68 h.compress = c.i32();
69 h.seekInfo = big ? c.u64() : c.u32();
70 h.nbytesInfo = c.i32();
71 if (h.begin <= 0 || h.nbytesName <= 0) {
72 c.fail("invalid file header");
73 }
74}
75
76KeyInfo RootFile::parseKey(std::span<const uint8_t> buf, uint64_t fileOffset)
77{
78 Cursor c{buf, mUrl, fileOffset};
79 KeyInfo k;
80 k.nbytes = c.u32();
81 k.version = c.i16();
82 k.objlen = c.u32();
83 c.u32(); // fDatime
84 k.keylen = c.u16();
85 k.cycle = c.u16();
86 bool big = k.version > 1000;
87 k.seekKey = big ? c.u64() : c.u32();
88 k.seekPdir = big ? c.u64() : c.u32();
89 k.className = c.string();
90 k.name = c.string();
91 k.title = c.string();
92 if (k.keylen > k.nbytes || c.pos() > k.keylen) {
93 c.fail("inconsistent key header for " + k.name);
94 }
95 return k;
96}
97
98DirectoryInfo RootFile::readDirectory(uint64_t offset)
99{
100 if (offset >= mSource->size()) {
101 throw CorruptData(mUrl, offset, "directory record past the end of the file");
102 }
103 auto buf = readRange(*mSource, offset, std::min<uint64_t>(kDirectoryReadSize, mSource->size() - offset));
104 Cursor c{buf, mUrl, offset};
105 DirectoryInfo d;
106 d.version = c.i16();
107 c.u32(); // fDatimeC
108 c.u32(); // fDatimeM
109 d.nbytesKeys = c.i32();
110 d.nbytesName = c.i32();
111 bool big = d.version > 1000;
112 d.seekDir = big ? c.u64() : c.u32();
113 d.seekParent = big ? c.u64() : c.u32();
114 d.seekKeys = big ? c.u64() : c.u32();
115 return d;
116}
117
118std::vector<KeyInfo> RootFile::readKeyList(DirectoryInfo const& dir)
119{
120 if (dir.seekKeys == 0 || dir.nbytesKeys <= 0) {
121 return {};
122 }
123 auto buf = readRange(*mSource, dir.seekKeys, dir.nbytesKeys);
124 auto header = parseKey(buf, dir.seekKeys);
125 Cursor c{buf, mUrl, dir.seekKeys};
126 c.seek(header.keylen);
127 auto nkeys = c.i32();
128 if (nkeys < 0 || (size_t)nkeys > c.remaining() / kMinKeySize) {
129 c.fail("invalid number of keys " + std::to_string(nkeys));
130 }
131 std::vector<KeyInfo> keys;
132 keys.reserve(nkeys);
133 for (int i = 0; i < nkeys; ++i) {
134 auto start = c.pos();
135 // The key length is only known after parsing, so parse from here to the end
136 // of the list and advance by the key length.
137 keys.push_back(parseKey(c.buffer().subspan(start), dir.seekKeys + start));
138 c.seek(start + keys.back().keylen);
139 }
140 return keys;
141}
142
143const std::vector<KeyInfo>& RootFile::topKeys()
144{
145 if (!mTopKeys) {
146 auto dir = readDirectory((uint64_t)mHeader.begin + mHeader.nbytesName);
147 mTopKeys = readKeyList(dir);
148 }
149 return *mTopKeys;
150}
151
152std::vector<KeyInfo> highestCycles(std::vector<KeyInfo> const& keys)
153{
154 std::vector<KeyInfo> result;
155 for (auto& k : keys) {
156 auto it = std::find_if(result.begin(), result.end(), [&k](KeyInfo const& r) { return r.name == k.name; });
157 if (it == result.end()) {
158 result.push_back(k);
159 } else if (k.cycle > it->cycle) {
160 *it = k;
161 }
162 }
163 return result;
164}
165
166std::optional<KeyInfo> RootFile::findTopKey(std::string const& name)
167{
168 std::optional<KeyInfo> result;
169 for (auto& k : topKeys()) {
170 if (k.name == name && (!result || k.cycle > result->cycle)) {
171 result = k;
172 }
173 }
174 return result;
175}
176
177std::vector<std::string> RootFile::dataFrames()
178{
179 static const std::regex tfRegex("/?DF_([0-9]+)(|-.*)$");
180 std::vector<uint64_t> numbers;
181 std::unordered_set<uint64_t> seen;
182 for (auto& k : topKeys()) {
183 std::smatch m;
184 if (std::regex_match(k.name, m, tfRegex)) {
185 if (m[1].length() > 19) {
186 throw CorruptData(mUrl, k.seekKey, "DF number of " + k.name + " does not fit 64 bits");
187 }
188 auto n = std::stoull(m[1].str());
189 if (seen.insert(n).second) {
190 numbers.push_back(n);
191 }
192 }
193 }
194 std::sort(numbers.begin(), numbers.end());
195 auto parents = readStringMap("parentFiles");
196 if (!parents.empty()) {
197 // Process DFs grouped by parent file, to avoid swapping between files.
198 // DataInputDirector uses an unstable sort here; ties are kept in numeric order.
199 auto parentOf = [&parents](uint64_t n) -> std::string const& {
200 static const std::string none;
201 auto it = parents.find("DF_" + std::to_string(n));
202 return it == parents.end() ? none : it->second;
203 };
204 std::stable_sort(numbers.begin(), numbers.end(), [&](uint64_t a, uint64_t b) { return parentOf(a) < parentOf(b); });
205 }
206 std::vector<std::string> result;
207 result.reserve(numbers.size());
208 for (auto n : numbers) {
209 result.push_back("DF_" + std::to_string(n));
210 }
211 return result;
212}
213
214std::vector<KeyInfo> RootFile::keysOf(const std::string& df)
215{
216 if (auto it = mKeysOf.find(df); it != mKeysOf.end()) {
217 return it->second;
218 }
219 auto key = findTopKey(df);
220 if (!key) {
221 for (auto& k : topKeys()) {
222 if (k.name.starts_with(df + "-")) {
223 throw UnsupportedLayout("flat " + df + "-<table> layout (" + k.className + ") in " + mUrl);
224 }
225 }
226 throw FormatError("no directory " + df + " in " + mUrl);
227 }
228 if (key->className != "TDirectoryFile" && key->className != "TDirectory") {
229 throw UnsupportedLayout(df + " in " + mUrl + " is a " + key->className + ", not a TDirectoryFile");
230 }
231 if (key->compressed()) {
232 throw UnsupportedLayout("compressed directory record for " + df + " in " + mUrl);
233 }
234 auto dir = readDirectory(key->seekKey + key->keylen);
235 auto keys = highestCycles(readKeyList(dir));
236
237 // The TTrees of a DF are written next to each other: ranges less than
238 // kMetadataGap apart are read as one.
239 struct Wanted {
240 uint64_t offset, length, seekKey;
241 };
242 std::vector<Wanted> wanted;
243 for (auto const& k : keys) {
244 auto stored = (uint64_t)k.nbytes - k.keylen;
245 auto offset = k.seekKey + k.keylen;
246 bool listed = mTreesToPrefetch.empty() || std::ranges::find(mTreesToPrefetch, k.name) != mTreesToPrefetch.end();
247 // Anything odd is left to readPayload(), which reports it.
248 if (mSource->canMap() || !listed || k.className != "TTree" || k.nbytes < k.keylen || stored > kMaxMetadataSize ||
249 offset > mSource->size() || stored > mSource->size() - offset) {
250 continue;
251 }
252 wanted.push_back({offset, stored, k.seekKey});
253 }
254 std::ranges::sort(wanted, {}, &Wanted::offset);
255 std::vector<std::pair<uint64_t, std::vector<uint8_t>>> spans;
256 for (auto const& w : wanted) {
257 if (!spans.empty()) {
258 auto& [start, bytes] = spans.back();
259 auto end = start + bytes.size();
260 if (w.offset <= end + kMetadataGap && w.offset + w.length - start <= kMaxMetadataSize) {
261 bytes.resize(std::max(end, w.offset + w.length) - start);
262 continue;
263 }
264 }
265 spans.emplace_back(w.offset, std::vector<uint8_t>(w.length));
266 }
267 std::vector<ReadRequest> requests;
268 for (auto& [start, bytes] : spans) {
269 requests.push_back({start, bytes.size(), bytes.data()});
270 }
271 mPrefetched.clear();
272 if (!requests.empty()) {
273 mSource->readvInto(requests);
274 }
275 size_t span = 0;
276 for (auto const& w : wanted) {
277 while (spans[span].first + spans[span].second.size() < w.offset + w.length) {
278 ++span;
279 }
280 auto const* at = spans[span].second.data() + (w.offset - spans[span].first);
281 mPrefetched[w.seekKey].assign(at, at + w.length);
282 }
283 mKeysOf.emplace(df, keys);
284 return keys;
285}
286
287std::vector<uint8_t> RootFile::readStored(const KeyInfo& key)
288{
289 auto stored = key.nbytes - key.keylen;
290 if (auto it = mPrefetched.find(key.seekKey); it != mPrefetched.end() && it->second.size() == stored) {
291 auto bytes = std::move(it->second);
292 mPrefetched.erase(it);
293 return bytes;
294 }
295 return readRange(*mSource, key.seekKey + key.keylen, stored);
296}
297
298std::vector<uint8_t> RootFile::readPayload(const KeyInfo& key)
299{
300 auto offset = key.seekKey + key.keylen;
301 auto stored = key.nbytes - key.keylen;
302 if (key.objlen > kMaxMetadataSize || stored > kMaxMetadataSize) {
303 throw CorruptData(mUrl, key.seekKey, "object " + key.name + " of " + std::to_string(key.objlen) + " bytes is too large for metadata");
304 }
305 if (!key.compressed()) {
306 return readStored(key);
307 }
308 auto compressed = readStored(key);
309 std::vector<uint8_t> result(key.objlen);
310 decompressBounded(compressed, key.objlen, result.data(), result.size(), mUrl, offset);
311 return result;
312}
313
315{
316 if (key.className != "TTree") {
317 throw UnsupportedLayout(key.name + " in " + mUrl + " is a " + key.className + ", not a TTree");
318 }
319 auto payload = readPayload(key);
320 auto tree = readTreePayload(payload, key.keylen, mUrl, key.seekKey + key.keylen);
321 // When fBasketEntry has no slot past the last normal basket, its entry count
322 // is only in the basket header (fNevBuf).
323 for (auto& branch : tree.branches) {
324 for (auto& basket : branch.baskets) {
325 if (basket.nEntries >= 0) {
326 continue;
327 }
328 auto buf = readRange(*mSource, basket.seek, std::min<uint64_t>(basket.nbytes, kHeaderReadSize * 4));
329 Cursor c{buf, mUrl, basket.seek};
330 basket.nEntries = readBasketHeader(c).nevbuf;
331 }
332 }
333 return tree;
334}
335
336std::map<std::string, std::string> RootFile::readStringMap(const std::string& name)
337{
338 if (auto it = mStringMaps.find(name); it != mStringMaps.end()) {
339 return it->second;
340 }
341 auto key = findTopKey(name);
342 if (!key) {
343 return mStringMaps[name];
344 }
345 if (key->className != "TMap") {
346 throw UnsupportedLayout(name + " in " + mUrl + " is a " + key->className + ", not a TMap");
347 }
348 auto payload = readPayload(*key);
349 return mStringMaps[name] = readStringMapPayload(payload, key->keylen, mUrl, key->seekKey + key->keylen);
350}
351
352} // namespace o2::aod::native
std::string url
unsigned int uint32_t
std::vector< std::string > keys
int32_t i
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events d
uint32_t c
Definition RawData.h:2
size_t bytes
StringRef key
Class for time synchronization of RawReader instances.
RootFile(std::shared_ptr< ByteSource >)
Definition Container.cxx:48
const std::vector< KeyInfo > & topKeys()
All the keys of the top directory, every cycle, in on-disk order.
TreeInfo readTree(const KeyInfo &)
std::vector< std::string > dataFrames()
std::map< std::string, std::string > readStringMap(const std::string &key)
Empty map if the key does not exist. Read once per key.
FileHeader const & header() const
Definition Container.h:60
std::vector< uint8_t > readPayload(const KeyInfo &)
The uncompressed payload of a key.
std::vector< KeyInfo > keysOf(const std::string &df)
GLdouble n
Definition glcorearb.h:1982
const GLfloat * m
Definition glcorearb.h:4066
GLuint64EXT * result
Definition glcorearb.h:5662
GLuint GLuint end
Definition glcorearb.h:469
GLuint const GLchar * name
Definition glcorearb.h:781
GLint first
Definition glcorearb.h:399
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei GLsizei GLchar * source
Definition glcorearb.h:798
GLintptr offset
Definition glcorearb.h:660
GLuint GLsizei GLsizei * length
Definition glcorearb.h:790
GLuint start
Definition glcorearb.h:469
GLenum GLenum GLsizei len
Definition glcorearb.h:4232
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
GLenum GLuint GLenum GLsizei const GLchar * buf
Definition glcorearb.h:2514
GLubyte GLubyte GLubyte GLubyte w
Definition glcorearb.h:852
TreeInfo readTreePayload(std::span< const uint8_t > payload, uint16_t keylen, std::string const &file, uint64_t fileOffset)
Definition Objects.cxx:534
BasketHeader readBasketHeader(Cursor &c)
Definition Cursor.h:144
size_t decompressBounded(std::span< const uint8_t > src, uint32_t objLen, uint8_t *dst, size_t maxOut, std::string const &file="<memory>", uint64_t fileOffset=0)
Definition Codecs.cxx:310
std::vector< KeyInfo > highestCycles(std::vector< KeyInfo > const &keys)
Keeps only the highest cycle of each name, preserving first-seen order.
std::map< std::string, std::string > readStringMapPayload(std::span< const uint8_t > payload, uint16_t keylen, std::string const &file, uint64_t fileOffset)
Deserializes a TMap of TObjString to TObjString (metaData, parentFiles).
Definition Objects.cxx:540
std::string to_string(gsl::span< T, Size > span)
Definition common.h:52
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))
const std::string str