18#include <unordered_set>
26constexpr size_t kHeaderReadSize = 128;
27constexpr size_t kDirectoryReadSize = 64;
29constexpr uint64_t kMetadataGap = 4 * 1024;
31constexpr size_t kMinKeySize = 29;
34constexpr uint32_t kMaxMetadataSize = 256 * 1024 * 1024;
42 std::vector<uint8_t>
buf(
len);
52 auto buf = readRange(*mSource, 0, std::min<uint64_t>(kHeaderReadSize, mSource->size()));
54 auto magic =
c.bytes(4);
55 if (std::memcmp(magic.data(),
"root", 4) != 0) {
61 bool big =
h.version >= 1000000;
62 h.end = big ?
c.u64() :
c.u32();
63 h.seekFree = big ?
c.u64() :
c.u32();
64 h.nbytesFree =
c.i32();
66 h.nbytesName =
c.i32();
69 h.seekInfo = big ?
c.u64() :
c.u32();
70 h.nbytesInfo =
c.i32();
71 if (
h.begin <= 0 ||
h.nbytesName <= 0) {
72 c.fail(
"invalid file header");
93 c.fail(
"inconsistent key header for " + k.
name);
100 if (
offset >= mSource->size()) {
101 throw CorruptData(mUrl,
offset,
"directory record past the end of the file");
103 auto buf = readRange(*mSource,
offset, std::min<uint64_t>(kDirectoryReadSize, mSource->size() -
offset));
109 d.nbytesKeys =
c.i32();
110 d.nbytesName =
c.i32();
111 bool big =
d.version > 1000;
112 d.seekDir = big ?
c.u64() :
c.u32();
113 d.seekParent = big ?
c.u64() :
c.u32();
114 d.seekKeys = big ?
c.u64() :
c.u32();
118std::vector<KeyInfo> RootFile::readKeyList(DirectoryInfo
const& dir)
120 if (dir.seekKeys == 0 || dir.nbytesKeys <= 0) {
123 auto buf = readRange(*mSource, dir.seekKeys, dir.nbytesKeys);
124 auto header = parseKey(
buf, dir.seekKeys);
125 Cursor
c{
buf, mUrl, dir.seekKeys};
127 auto nkeys =
c.i32();
128 if (nkeys < 0 || (
size_t)nkeys >
c.remaining() / kMinKeySize) {
131 std::vector<KeyInfo>
keys;
133 for (
int i = 0;
i < nkeys; ++
i) {
137 keys.push_back(parseKey(
c.buffer().subspan(
start), dir.seekKeys +
start));
147 mTopKeys = readKeyList(dir);
154 std::vector<KeyInfo>
result;
155 for (
auto& k :
keys) {
156 auto it = std::find_if(
result.begin(),
result.end(), [&k](
KeyInfo const&
r) { return r.name == k.name; });
159 }
else if (k.
cycle > it->cycle) {
166std::optional<KeyInfo> RootFile::findTopKey(std::string
const&
name)
168 std::optional<KeyInfo>
result;
179 static const std::regex tfRegex(
"/?DF_([0-9]+)(|-.*)$");
180 std::vector<uint64_t> numbers;
181 std::unordered_set<uint64_t> seen;
184 if (std::regex_match(k.
name,
m, tfRegex)) {
188 auto n = std::stoull(
m[1].
str());
189 if (seen.insert(
n).second) {
190 numbers.push_back(
n);
194 std::sort(numbers.begin(), numbers.end());
196 if (!parents.empty()) {
199 auto parentOf = [&parents](
uint64_t n) -> std::string
const& {
200 static const std::string none;
202 return it == parents.end() ? none : it->second;
204 std::stable_sort(numbers.begin(), numbers.end(), [&](
uint64_t a,
uint64_t b) { return parentOf(a) < parentOf(b); });
206 std::vector<std::string>
result;
207 result.reserve(numbers.size());
208 for (
auto n : numbers) {
216 if (
auto it = mKeysOf.find(df); it != mKeysOf.end()) {
219 auto key = findTopKey(df);
222 if (k.
name.starts_with(df +
"-")) {
226 throw FormatError(
"no directory " + df +
" in " + mUrl);
228 if (
key->className !=
"TDirectoryFile" &&
key->className !=
"TDirectory") {
229 throw UnsupportedLayout(df +
" in " + mUrl +
" is a " +
key->className +
", not a TDirectoryFile");
231 if (
key->compressed()) {
232 throw UnsupportedLayout(
"compressed directory record for " + df +
" in " + mUrl);
234 auto dir = readDirectory(
key->seekKey +
key->keylen);
242 std::vector<Wanted> wanted;
243 for (
auto const& k :
keys) {
246 bool listed = mTreesToPrefetch.empty() || std::ranges::find(mTreesToPrefetch, k.
name) != mTreesToPrefetch.end();
248 if (mSource->canMap() || !listed || k.
className !=
"TTree" || k.
nbytes < k.
keylen || stored > kMaxMetadataSize ||
249 offset > mSource->size() || stored > mSource->size() -
offset) {
254 std::ranges::sort(wanted, {}, &Wanted::offset);
255 std::vector<std::pair<uint64_t, std::vector<uint8_t>>> spans;
256 for (
auto const&
w : wanted) {
257 if (!spans.empty()) {
260 if (
w.offset <=
end + kMetadataGap &&
w.offset +
w.length -
start <= kMaxMetadataSize) {
265 spans.emplace_back(
w.offset, std::vector<uint8_t>(
w.length));
267 std::vector<ReadRequest> requests;
272 if (!requests.empty()) {
273 mSource->readvInto(requests);
276 for (
auto const&
w : wanted) {
277 while (spans[span].
first + spans[span].second.size() <
w.offset +
w.length) {
280 auto const* at = spans[span].second.data() + (
w.offset - spans[span].first);
281 mPrefetched[
w.seekKey].assign(at, at +
w.length);
283 mKeysOf.emplace(df,
keys);
287std::vector<uint8_t> RootFile::readStored(
const KeyInfo&
key)
289 auto stored =
key.nbytes -
key.keylen;
290 if (
auto it = mPrefetched.find(
key.seekKey); it != mPrefetched.end() && it->second.size() == stored) {
291 auto bytes = std::move(it->second);
292 mPrefetched.erase(it);
295 return readRange(*mSource,
key.seekKey +
key.keylen, stored);
301 auto stored =
key.nbytes -
key.keylen;
302 if (
key.objlen > kMaxMetadataSize || stored > kMaxMetadataSize) {
305 if (!
key.compressed()) {
306 return readStored(
key);
308 auto compressed = readStored(
key);
316 if (
key.className !=
"TTree") {
323 for (
auto& branch :
tree.branches) {
324 for (
auto& basket : branch.baskets) {
325 if (basket.nEntries >= 0) {
328 auto buf = readRange(*mSource, basket.seek, std::min<uint64_t>(basket.nbytes, kHeaderReadSize * 4));
338 if (
auto it = mStringMaps.find(
name); it != mStringMaps.end()) {
343 return mStringMaps[
name];
345 if (
key->className !=
"TMap") {
std::vector< std::string > keys
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events d
Class for time synchronization of RawReader instances.
RootFile(std::shared_ptr< ByteSource >)
const std::vector< KeyInfo > & topKeys()
All the keys of the top directory, every cycle, in on-disk order.
TreeInfo readTree(const KeyInfo &)
std::vector< std::string > dataFrames()
std::map< std::string, std::string > readStringMap(const std::string &key)
Empty map if the key does not exist. Read once per key.
FileHeader const & header() const
std::vector< uint8_t > readPayload(const KeyInfo &)
The uncompressed payload of a key.
std::vector< KeyInfo > keysOf(const std::string &df)
GLuint const GLchar * name
GLboolean GLboolean GLboolean b
GLsizei GLsizei GLchar * source
GLuint GLsizei GLsizei * length
GLenum GLenum GLsizei len
GLboolean GLboolean GLboolean GLboolean a
GLenum GLuint GLenum GLsizei const GLchar * buf
GLubyte GLubyte GLubyte GLubyte w
TreeInfo readTreePayload(std::span< const uint8_t > payload, uint16_t keylen, std::string const &file, uint64_t fileOffset)
BasketHeader readBasketHeader(Cursor &c)
size_t decompressBounded(std::span< const uint8_t > src, uint32_t objLen, uint8_t *dst, size_t maxOut, std::string const &file="<memory>", uint64_t fileOffset=0)
std::vector< KeyInfo > highestCycles(std::vector< KeyInfo > const &keys)
Keeps only the highest cycle of each name, preserving first-seen order.
std::map< std::string, std::string > readStringMapPayload(std::span< const uint8_t > payload, uint16_t keylen, std::string const &file, uint64_t fileOffset)
Deserializes a TMap of TObjString to TObjString (metaData, parentFiles).
std::string to_string(gsl::span< T, Size > span)
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))