Project
Loading...
Searching...
No Matches
ArrowTable.cxx
Go to the documentation of this file.
1// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
13#include "AODNative/Errors.h"
14#include <arrow/array/array_nested.h>
15#include <arrow/array/util.h>
16#include <arrow/buffer.h>
17#include <cstdio>
18#include <cstring>
19
20namespace o2::aod::native
21{
22
23namespace
24{
25std::shared_ptr<arrow::DataType> valueType(BranchInfo const& b)
26{
27 switch (b.leafClass) {
28 case 'O':
29 return arrow::boolean();
30 case 'B':
31 return b.isUnsigned ? arrow::uint8() : arrow::int8();
32 case 'S':
33 return b.isUnsigned ? arrow::uint16() : arrow::int16();
34 case 'I':
35 return b.isUnsigned ? arrow::uint32() : arrow::int32();
36 case 'L':
37 return b.isUnsigned ? arrow::uint64() : arrow::int64();
38 case 'F':
39 return arrow::float32();
40 case 'D':
41 return arrow::float64();
42 }
43 throw UnsupportedLayout(std::string("leaf class TLeaf") + b.leafClass + " of branch " + b.name);
44}
45
46BranchInfo const& branchNamed(TreeInfo const& tree, std::string const& name)
47{
48 for (auto& b : tree.branches) {
49 if (b.name == name) {
50 return b;
51 }
52 }
53 throw FormatError("tree " + tree.name + " has no branch " + name);
54}
55
56std::string format(char const* fmt, auto... args)
57{
58 auto n = std::snprintf(nullptr, 0, fmt, args...);
59 std::string s(n, '\0');
60 std::snprintf(s.data(), n + 1, fmt, args...);
61 return s;
62}
63} // namespace
64
65std::shared_ptr<arrow::Schema> physicalSchema(TreeInfo const& tree)
66{
67 std::vector<std::shared_ptr<arrow::Field>> fields;
68 bool prevIsSize = false;
69 for (auto& b : tree.branches) {
70 if (prevIsSize && fields.back()->name() != b.name + "_size") {
71 throw UnsupportedLayout(format("Unexpected layout for VLA container %s.", b.name.c_str()));
72 }
73 if (b.name.ends_with("_size")) {
74 fields.push_back(std::make_shared<arrow::Field>(b.name, arrow::int32()));
75 prevIsSize = true;
76 continue;
77 }
78 if (prevIsSize) {
79 fields.push_back(std::make_shared<arrow::Field>(b.name, arrow::list(valueType(b))));
80 } else if (!b.countBranch.empty()) {
81 throw UnsupportedLayout("variable-length branch " + b.name + " is not preceded by its size branch");
82 } else if (b.fixedLen > 1) {
83 fields.push_back(std::make_shared<arrow::Field>(b.name, arrow::fixed_size_list(valueType(b), b.fixedLen)));
84 } else {
85 fields.push_back(std::make_shared<arrow::Field>(b.name, valueType(b)));
86 }
87 prevIsSize = false;
88 }
89 if (!fields.empty() && fields.back()->name().ends_with("_size")) {
90 throw UnsupportedLayout(format("Missing values for VLA indices %s.", fields.back()->name().c_str()));
91 }
92 return std::make_shared<arrow::Schema>(fields);
93}
94
95std::shared_ptr<arrow::Schema> datasetSchema(std::shared_ptr<arrow::Schema> const& physical)
96{
97 std::vector<std::shared_ptr<arrow::Field>> fields;
98 for (auto& f : physical->fields()) {
99 if (!f->name().ends_with("_size")) {
100 fields.push_back(f);
101 }
102 }
103 return std::make_shared<arrow::Schema>(fields);
104}
105
106std::vector<ColumnOp> planColumns(TreeInfo const& tree, std::shared_ptr<arrow::Schema> const& schema,
107 BranchDecoder& decoder, arrow::MemoryPool* pool, int64_t& rows)
108{
109 std::vector<ColumnOp> ops;
110 rows = -1;
111 for (int fi = 0; fi < schema->num_fields(); ++fi) {
112 auto const& field = schema->field(fi);
113 auto const& branch = branchNamed(tree, field->name());
114 auto type = field->type();
115 ColumnOp op{ColumnOp::Kind::Values, &branch, fi, 0, nullptr};
116 int64_t opRows = branch.entries;
117 if (type->id() == arrow::Type::LIST) {
118 auto const& sizeBranch = branchNamed(tree, branch.name + "_size");
119 if (branch.entries != sizeBranch.entries) {
120 throw CorruptData(decoder.url(), 0, format("Branch %s has %lld entries, but its size branch %s has %lld.", branch.name.c_str(), (long long)branch.entries, sizeBranch.name.c_str(), (long long)sizeBranch.entries));
121 }
122 auto offsets = decoder.decodeOffsets(sizeBranch);
123 auto buffer = arrow::AllocateBuffer(offsets.size() * sizeof(int32_t), pool).ValueOrDie();
124 std::memcpy(buffer->mutable_data(), offsets.data(), offsets.size() * sizeof(int32_t));
125 op.offsets = std::move(buffer);
126 op.kind = ColumnOp::Kind::VLA;
127 op.byteSize = (size_t)offsets.back() * leafElementSize(branch.leafClass);
128 opRows = sizeBranch.entries;
129 } else if (type->id() == arrow::Type::BOOL) {
131 op.byteSize = (branch.entries + 7) / 8;
132 } else if (type->id() == arrow::Type::FIXED_SIZE_LIST && type->field(0)->type()->id() == arrow::Type::BOOL) {
133 // Not the minimal size, but the TTree plugin's, for byte identity.
135 op.byteSize = (branch.entries * branch.fixedLen) / 8 + 1;
136 } else {
137 op.byteSize = branch.entries * branch.fixedLen * leafElementSize(branch.leafClass);
138 }
139 if (rows == -1) {
140 rows = opRows;
141 } else if (rows != opRows) {
142 throw CorruptData(decoder.url(), 0, format("Unmatching number of rows for branch %s. Expected %lli, found %lli", branch.name.c_str(), (long long)rows, (long long)opRows));
143 }
144 ops.push_back(std::move(op));
145 }
146 if (rows == -1) {
147 rows = 0;
148 }
149 return ops;
150}
151
153{
154 switch (op.kind) {
156 decoder.decodeValues(*op.branch, dst, op.byteSize);
157 break;
159 decoder.decodeBooleans(*op.branch, dst, op.byteSize);
160 break;
161 case ColumnOp::Kind::VLA: {
162 auto const* offsets = (int32_t const*)op.offsets->data();
163 decoder.decodeVLAValues(*op.branch, {offsets, (size_t)op.branch->entries + 1}, dst, op.byteSize);
164 } break;
165 }
166}
167
168std::shared_ptr<arrow::RecordBatch> makeBatch(std::shared_ptr<arrow::Schema> const& schema, std::vector<ColumnOp> const& ops,
169 std::vector<std::shared_ptr<arrow::Buffer>> const& buffers, int64_t rows)
170{
171 std::vector<std::shared_ptr<arrow::Array>> columns;
172 for (size_t i = 0; i < ops.size(); ++i) {
173 auto const& op = ops[i];
174 auto type = schema->field(op.datasetField)->type();
175 auto entries = op.branch->entries;
176 if (type->id() == arrow::Type::FIXED_SIZE_LIST) {
177 auto vdata = std::make_shared<arrow::ArrayData>(type->field(0)->type(), entries * op.branch->fixedLen,
178 std::vector<std::shared_ptr<arrow::Buffer>>{nullptr, buffers[i]});
179 columns.push_back(std::make_shared<arrow::FixedSizeListArray>(type, entries, arrow::MakeArray(vdata)));
180 } else if (op.kind == ColumnOp::Kind::VLA) {
181 auto total = ((int32_t const*)op.offsets->data())[entries];
182 auto vdata = std::make_shared<arrow::ArrayData>(type->field(0)->type(), total,
183 std::vector<std::shared_ptr<arrow::Buffer>>{nullptr, buffers[i]});
184 columns.push_back(std::make_shared<arrow::ListArray>(type, entries, op.offsets, arrow::MakeArray(vdata)));
185 } else {
186 auto data = std::make_shared<arrow::ArrayData>(type, entries, std::vector<std::shared_ptr<arrow::Buffer>>{nullptr, buffers[i]});
187 columns.push_back(arrow::MakeArray(data));
188 }
189 }
190 return arrow::RecordBatch::Make(schema, rows, columns);
191}
192
193arrow::Result<std::shared_ptr<arrow::RecordBatch>> readTree(ByteSource& source, TreeInfo const& tree, Options const& options, arrow::MemoryPool* pool)
194{
196 BranchDecoder decoder{source, options};
198 auto ops = planColumns(tree, schema, decoder, pool, rows);
199 std::vector<std::shared_ptr<arrow::Buffer>> buffers;
200 for (auto& op : ops) {
201 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Buffer> buffer, arrow::AllocateBuffer(op.byteSize, pool));
202 executeColumnOp(decoder, op, buffer->mutable_data());
203 buffers.push_back(std::move(buffer));
204 }
205 return makeBatch(schema, ops, buffers, rows);
206}
207
208} // namespace o2::aod::native
std::shared_ptr< arrow::Schema > schema
std::vector< std::shared_ptr< arrow::Field > > fields
std::vector< long > entries
int32_t i
uint32_t op
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h configStandalone Processing proc_rtctech Use a file lock to serialize access to the cache folder int32_t
void decodeVLAValues(BranchInfo const &branch, std::span< const int32_t > offsets, uint8_t *dst, size_t nbytes)
Definition Decoder.cxx:215
std::string const & url() const
Definition Decoder.h:38
std::vector< int32_t > decodeOffsets(BranchInfo const &sizeBranch)
Decodes a <col>_size branch into entries + 1 int32 offsets.
Definition Decoder.cxx:265
void decodeValues(BranchInfo const &branch, uint8_t *dst, size_t nbytes)
Definition Decoder.cxx:198
void decodeBooleans(BranchInfo const &branch, uint8_t *bitmap, size_t nbytes)
Definition Decoder.cxx:239
GLdouble n
Definition glcorearb.h:1982
GLuint GLsizei const GLuint const GLintptr * offsets
Definition glcorearb.h:2595
GLuint const GLchar * name
Definition glcorearb.h:781
GLdouble f
Definition glcorearb.h:310
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei GLsizei GLchar * source
Definition glcorearb.h:798
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
GLenum GLenum dst
Definition glcorearb.h:1767
GLint GLint GLsizei GLint GLenum format
Definition glcorearb.h:275
const GLuint * buffers
Definition glcorearb.h:656
std::shared_ptr< arrow::Schema > datasetSchema(std::shared_ptr< arrow::Schema > const &physical)
The physical schema without the *_size fields, as DataInputDirector derives it.
size_t leafElementSize(char leafClass)
Size in bytes of one value of a leaf class on disk.
Definition Objects.cxx:493
std::shared_ptr< arrow::RecordBatch > makeBatch(std::shared_ptr< arrow::Schema > const &schema, std::vector< ColumnOp > const &ops, std::vector< std::shared_ptr< arrow::Buffer > > const &buffers, int64_t rows)
Wraps the buffers of ops (one per op, in order) into a record batch.
arrow::Result< std::shared_ptr< arrow::RecordBatch > > readTree(ByteSource &source, TreeInfo const &tree, Options const &options, arrow::MemoryPool *pool=arrow::default_memory_pool())
Reads a whole tree into heap memory: plan, execute, wrap.
std::shared_ptr< arrow::Schema > physicalSchema(TreeInfo const &tree)
std::vector< ColumnOp > planColumns(TreeInfo const &tree, std::shared_ptr< arrow::Schema > const &schema, BranchDecoder &decoder, arrow::MemoryPool *pool, int64_t &rows)
void executeColumnOp(BranchDecoder &decoder, ColumnOp const &op, uint8_t *dst)
Fills the buffer of op, which is exactly op.byteSize bytes.
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))
std::vector< ReadoutWindowData > rows