Project
Loading...
Searching...
No Matches
ASoA.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#include "Framework/ASoA.h"
13#include "ArrowDebugHelpers.h"
15#include <arrow/util/key_value_metadata.h>
16#include <arrow/util/config.h>
17
18namespace o2::soa
19{
20void accessingInvalidIndexFor(const char* getter)
21{
22 throw o2::framework::runtime_error_f("Accessing invalid index for %s", getter);
23}
24void dereferenceWithWrongType(const char* getter, const char* target)
25{
26 throw o2::framework::runtime_error_f("Trying to dereference index with a wrong type in %s_as<T> for base target \"%s\". Note that if you have several compatible index targets in your process() signature, the last one will be the one actually bound.", getter, target);
27}
28void missingFilterDeclaration(int hash, int ai)
29{
30 throw o2::framework::runtime_error_f("Null selection for %d (arg %d), missing Filter declaration?", hash, ai);
31}
32
33void getterNotFound(const char* targetColumnLabel)
34{
35 throw o2::framework::runtime_error_f("Getter for \"%s\" not found", targetColumnLabel);
36}
37
39{
40 throw framework::runtime_error("columnLabel: must not be empty");
41}
42
44{
46 rows.resize(sel->GetNumSlots());
47 for (auto i = 0; i < sel->GetNumSlots(); ++i) {
48 rows[i] = sel->GetIndex(i);
49 }
50 return rows;
51}
52
53SelectionVector sliceSelection(gsl::span<int64_t const> const& mSelectedRows, int64_t nrows, uint64_t offset)
54{
55 auto start = offset;
56 auto end = start + nrows;
57 auto start_iterator = std::lower_bound(mSelectedRows.begin(), mSelectedRows.end(), start);
58 auto stop_iterator = std::lower_bound(start_iterator, mSelectedRows.end(), end);
59 SelectionVector slicedSelection{start_iterator, stop_iterator};
60 std::transform(slicedSelection.begin(), slicedSelection.end(), slicedSelection.begin(),
61 [&start](int64_t idx) {
62 return idx - static_cast<int64_t>(start);
63 });
64 return slicedSelection;
65}
66
67std::shared_ptr<arrow::Table> ArrowHelpers::joinTables(std::vector<std::shared_ptr<arrow::Table>>&& tables, std::span<const char* const> labels)
68{
69 if (tables.size() == 1) {
70 return tables[0];
71 }
72 for (auto i = 0U; i < tables.size() - 1; ++i) {
73 if (tables[i]->num_rows() != tables[i + 1]->num_rows()) {
74 throw o2::framework::runtime_error_f("Tables %s and %s have different sizes (%d vs %d) and cannot be joined!",
75 labels[i], labels[i + 1], tables[i]->num_rows(), tables[i + 1]->num_rows());
76 }
77 }
78 std::vector<std::shared_ptr<arrow::Field>> fields;
79 std::vector<std::shared_ptr<arrow::ChunkedArray>> columns;
80
81 for (auto& t : tables) {
82 auto tf = t->fields();
83 std::copy(tf.begin(), tf.end(), std::back_inserter(fields));
84 }
85
86 auto schema = std::make_shared<arrow::Schema>(fields);
87
88 if (tables[0]->num_rows() != 0) {
89 for (auto& t : tables) {
90 auto tc = t->columns();
91 std::copy(tc.begin(), tc.end(), std::back_inserter(columns));
92 }
93 }
94 return arrow::Table::Make(schema, columns);
95}
96
97std::shared_ptr<arrow::Table> ArrowHelpers::concatTables(std::vector<std::shared_ptr<arrow::Table>>&& tables)
98{
99 if (tables.size() == 1) {
100 return tables[0];
101 }
102 std::vector<std::shared_ptr<arrow::ChunkedArray>> columns;
103 assert(tables.size() > 1);
104 std::vector<std::shared_ptr<arrow::Field>> resultFields = tables[0]->schema()->fields();
105 auto compareFields = [](std::shared_ptr<arrow::Field> const& f1, std::shared_ptr<arrow::Field> const& f2) {
106 // Let's do this with stable sorting.
107 return (!f1->Equals(f2)) && (f1->name() < f2->name());
108 };
109 for (size_t i = 1; i < tables.size(); ++i) {
110 auto& fields = tables[i]->schema()->fields();
111 std::vector<std::shared_ptr<arrow::Field>> intersection;
112
113 std::set_intersection(resultFields.begin(), resultFields.end(),
114 fields.begin(), fields.end(),
115 std::back_inserter(intersection), compareFields);
116 resultFields.swap(intersection);
117 }
118
119 for (auto& field : resultFields) {
120 arrow::ArrayVector chunks;
121 for (auto& table : tables) {
122 auto ci = table->schema()->GetFieldIndex(field->name());
123 if (ci == -1) {
124 throw std::runtime_error("Unable to find field " + field->name());
125 }
126 auto column = table->column(ci);
127 auto otherChunks = column->chunks();
128 chunks.insert(chunks.end(), otherChunks.begin(), otherChunks.end());
129 }
130 columns.push_back(std::make_shared<arrow::ChunkedArray>(chunks));
131 }
132
133 auto result = arrow::Table::Make(std::make_shared<arrow::Schema>(resultFields), columns);
134 return result;
135}
136
137arrow::ChunkedArray* getIndexFromLabel(arrow::Table* table, std::string_view label)
138{
139 auto field = std::find_if(table->schema()->fields().begin(), table->schema()->fields().end(), [&](std::shared_ptr<arrow::Field> const& f) {
140 auto caseInsensitiveCompare = [](const std::string_view& str1, const std::string& str2) {
141 return std::ranges::equal(
142 str1, str2,
143 [](char c1, char c2) {
144 return std::tolower(static_cast<unsigned char>(c1)) ==
145 std::tolower(static_cast<unsigned char>(c2));
146 });
147 };
148
149 return caseInsensitiveCompare(label, f->name());
150 });
151 if (field == table->schema()->fields().end()) {
152 o2::framework::throw_error(o2::framework::runtime_error_f("Unable to find column with label %s", label));
153 }
154 auto index = std::distance(table->schema()->fields().begin(), field);
155 return table->column(index).get();
156}
157
158void notBoundTable(const char* tableName)
159{
160 throw o2::framework::runtime_error_f("Index pointing to %s is not bound! Did you subscribe to the table?", tableName);
161}
162
163void notFoundColumn(const char* label, const char* key)
164{
165 throw o2::framework::runtime_error_f(R"(Preslice not valid: table "%s" (or join based on it) does not have column "%s")", label, key);
166}
167
168void missingOptionalPreslice(const char* label, const char* key)
169{
170 throw o2::framework::runtime_error_f(R"(Optional Preslice with missing binding used: table "%s" (or join based on it) does not have column "%s")", label, key);
171}
172
173} // namespace o2::soa
174
175namespace o2::framework
176{
177std::string cutString(std::string&& str)
178{
179 auto pos = str.find('_');
180 if (pos != std::string::npos) {
181 str.erase(pos);
182 }
183 return str;
184}
185
186std::string strToUpper(std::string&& str)
187{
188 std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) { return std::toupper(c); });
189 return str;
190}
191
193{
194 return binding == "[MISSING]";
195}
196
198{
199 return bindingKey;
200}
201
206
211
212std::shared_ptr<arrow::Table> PreslicePolicySorted::getSliceFor(int value, std::shared_ptr<arrow::Table> const& input, uint64_t& offset) const
213{
214 auto [offset_, count] = this->sliceInfo.getSliceFor(value);
215 auto output = input->Slice(offset_, count);
216 offset = static_cast<int64_t>(offset_);
217 return output;
218}
219
220gsl::span<const int64_t> PreslicePolicyGeneral::getSliceFor(int value) const
221{
222 return this->sliceInfo.getSliceFor(value);
223}
224} // namespace o2::framework
int32_t i
void output(const std::map< std::string, ChannelStat > &channels)
Definition rawdump.cxx:197
uint16_t pos
Definition RawData.h:3
uint32_t c
Definition RawData.h:2
StringRef key
GLint GLsizei count
Definition glcorearb.h:399
GLuint64EXT * result
Definition glcorearb.h:5662
GLuint GLuint end
Definition glcorearb.h:469
GLuint index
Definition glcorearb.h:781
GLdouble f
Definition glcorearb.h:310
GLsizei const GLfloat * value
Definition glcorearb.h:819
GLenum target
Definition glcorearb.h:1641
GLintptr offset
Definition glcorearb.h:660
GLuint GLsizei const GLchar * label
Definition glcorearb.h:2519
GLuint start
Definition glcorearb.h:469
std::shared_ptr< gandiva::SelectionVector > Selection
Definition Expressions.h:46
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
RuntimeErrorRef runtime_error(const char *)
void throw_error(RuntimeErrorRef)
std::string strToUpper(std::string &&str)
Definition ASoA.cxx:186
RuntimeErrorRef runtime_error_f(const char *,...)
std::string cutString(std::string &&str)
Definition ASoA.cxx:177
SelectionVector selectionToVector(gandiva::Selection const &sel)
Definition ASoA.cxx:43
SelectionVector sliceSelection(gsl::span< int64_t const > const &mSelectedRows, int64_t nrows, uint64_t offset)
Definition ASoA.cxx:53
void notBoundTable(const char *tableName)
Definition ASoA.cxx:158
std::vector< int64_t > SelectionVector
Definition ASoA.h:412
void missingFilterDeclaration(int hash, int ai)
Definition ASoA.cxx:28
void accessingInvalidIndexFor(const char *getter)
Definition ASoA.cxx:20
void dereferenceWithWrongType(const char *getter, const char *target)
Definition ASoA.cxx:24
void emptyColumnLabel()
Definition ASoA.cxx:38
void getterNotFound(const char *targetColumnLabel)
Definition ASoA.cxx:33
void missingOptionalPreslice(const char *label, const char *key)
Definition ASoA.cxx:168
arrow::ChunkedArray * getIndexFromLabel(arrow::Table *table, std::string_view label)
Definition ASoA.cxx:137
void notFoundColumn(const char *label, const char *key)
Definition ASoA.cxx:163
std::unique_ptr< GPUReconstructionTimeframe > tf
const std::string binding
Definition ASoA.h:1405
Entry const & getBindingKey() const
Definition ASoA.cxx:197
SliceInfoUnsortedPtr sliceInfo
Definition ASoA.h:1422
gsl::span< const int64_t > getSliceFor(int value) const
Definition ASoA.cxx:220
void updateSliceInfo(SliceInfoUnsortedPtr &&si)
Definition ASoA.cxx:207
void updateSliceInfo(SliceInfoPtr &&si)
Definition ASoA.cxx:202
std::shared_ptr< arrow::Table > getSliceFor(int value, std::shared_ptr< arrow::Table > const &input, uint64_t &offset) const
Definition ASoA.cxx:212
std::pair< int64_t, int64_t > getSliceFor(int value) const
gsl::span< int64_t const > getSliceFor(int value) const
static std::shared_ptr< arrow::Table > joinTables(std::vector< std::shared_ptr< arrow::Table > > &&tables, std::span< const char *const > labels)
Definition ASoA.cxx:67
static std::shared_ptr< arrow::Table > concatTables(std::vector< std::shared_ptr< arrow::Table > > &&tables)
Definition ASoA.cxx:97
std::vector< ReadoutWindowData > rows
const std::string str