Project
Loading...
Searching...
No Matches
TableBuilder.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#ifndef O2_FRAMEWORK_TABLEBUILDER_H_
13#define O2_FRAMEWORK_TABLEBUILDER_H_
14
15#include "Framework/ASoA.h"
18#include "arrow/type_traits.h"
19
20// Apparently needs to be on top of the arrow includes.
21#include <sstream>
22
23#include <arrow/chunked_array.h>
24#include <arrow/status.h>
25#include <arrow/memory_pool.h>
26#include <arrow/stl.h>
27#include <arrow/type_traits.h>
28#include <arrow/table.h>
29#include <arrow/builder.h>
30
31#include <vector>
32#include <string>
33#include <memory>
34#include <tuple>
35#include <type_traits>
36#include <concepts>
37
38namespace arrow
39{
40class ArrayBuilder;
41class Table;
42class Array;
43} // namespace arrow
44
45extern template class arrow::NumericBuilder<arrow::UInt8Type>;
46extern template class arrow::NumericBuilder<arrow::UInt32Type>;
47extern template class arrow::NumericBuilder<arrow::FloatType>;
48extern template class arrow::NumericBuilder<arrow::Int32Type>;
49extern template class arrow::NumericBuilder<arrow::Int8Type>;
50
51namespace o2::framework
52{
53namespace detail
54{
56template <typename T>
58};
59
60template <typename T, int N>
61struct ConversionTraits<T (&)[N]> {
62 using ArrowType = ::arrow::FixedSizeListType;
63};
64
65template <typename T, int N>
66struct ConversionTraits<T[N]> {
67 using ArrowType = ::arrow::FixedSizeListType;
68};
69
70template <typename T, int N>
71struct ConversionTraits<std::array<T, N>> {
72 using ArrowType = ::arrow::FixedSizeListType;
73};
74
75template <typename T>
76struct ConversionTraits<std::vector<T>> {
77 using ArrowType = ::arrow::ListType;
78};
79
80#define O2_ARROW_STL_CONVERSION(c_type, ArrowType_) \
81 template <> \
82 struct ConversionTraits<c_type> { \
83 using ArrowType = ::arrow::ArrowType_; \
84 };
85
86// FIXME: for now we use Int8 to store booleans
87O2_ARROW_STL_CONVERSION(bool, BooleanType)
88O2_ARROW_STL_CONVERSION(int8_t, Int8Type)
89O2_ARROW_STL_CONVERSION(int16_t, Int16Type)
90O2_ARROW_STL_CONVERSION(int32_t, Int32Type)
91O2_ARROW_STL_CONVERSION(long long, Int64Type)
92O2_ARROW_STL_CONVERSION(long, Int64Type)
93O2_ARROW_STL_CONVERSION(uint8_t, UInt8Type)
94O2_ARROW_STL_CONVERSION(uint16_t, UInt16Type)
95O2_ARROW_STL_CONVERSION(uint32_t, UInt32Type)
96O2_ARROW_STL_CONVERSION(long long unsigned, UInt64Type)
97O2_ARROW_STL_CONVERSION(long unsigned, UInt64Type)
98O2_ARROW_STL_CONVERSION(float, FloatType)
99O2_ARROW_STL_CONVERSION(double, DoubleType)
100O2_ARROW_STL_CONVERSION(std::string, StringType)
101} // namespace detail
102
103void addLabelToSchema(std::shared_ptr<arrow::Schema>& schema, const char* label);
104
106 template <typename T>
107 static arrow::Status appendToList(std::unique_ptr<arrow::FixedSizeListBuilder>& builder, T* data, int size = 1)
108 {
109 using ArrowType = typename detail::ConversionTraits<std::decay_t<T>>::ArrowType;
110 using BuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
111 size_t numElements = static_cast<const arrow::FixedSizeListType*>(builder->type().get())->list_size();
112
113 auto status = builder->AppendValues(size);
114 auto ValueBuilder = static_cast<BuilderType*>(builder->value_builder());
115 status &= ValueBuilder->AppendValues(data, numElements * size, nullptr);
116
117 return status;
118 }
119
120 template <typename HolderType, typename T>
121 static arrow::Status append(HolderType& holder, T value)
122 {
123 return static_cast<typename HolderType::Policy&>(holder).append(holder.builder, value);
124 }
125
126 template <typename HolderType>
127 static arrow::Status flush(HolderType& holder)
128 {
129 return static_cast<typename HolderType::Policy&>(holder).flush(holder.builder);
130 }
131
135 template <typename HolderType, typename T>
136 static arrow::Status append(HolderType& holder, T* data)
137 {
138 if constexpr (std::is_same_v<decltype(holder.builder), std::unique_ptr<arrow::FixedSizeListBuilder>>) {
139 return appendToList<T>(holder.builder, data);
140 } else {
141 return holder.builder->Append(reinterpret_cast<const uint8_t*>(data));
142 }
143 }
145 template <typename HolderType, typename T, int N>
146 static arrow::Status append(HolderType& holder, T (&data)[N])
147 {
148 return holder.builder->Append(reinterpret_cast<const uint8_t*>(data));
149 }
150
152 template <typename HolderType, typename T, int N>
153 static arrow::Status append(HolderType& holder, std::array<T, N> const& data)
154 {
155 return holder.builder->Append(reinterpret_cast<const uint8_t*>(data.data()));
156 }
157
159 template <typename HolderType, typename T>
160 static arrow::Status append(HolderType& holder, std::vector<T> const& data)
161 {
162 using ArrowType = typename detail::ConversionTraits<T>::ArrowType;
163 using ValueBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
164 auto status = holder.builder->Reserve(data.size());
165 status &= holder.builder->Append();
166 auto vbuilder = static_cast<ValueBuilderType*>(holder.builder->value_builder());
167 status &= vbuilder->AppendValues(data.begin(), data.end());
168
169 return status;
170 }
171
172 template <typename HolderType, typename T>
173 static void unsafeAppend(HolderType& holder, std::vector<T> const& value)
174 {
175 auto status = append(holder, value);
176 if (!status.ok()) {
177 throw runtime_error("Unable to append to column");
178 }
179 }
180
181 template <typename HolderType, typename T>
182 static void unsafeAppend(HolderType& holder, T value)
183 {
184 return holder.builder->UnsafeAppend(value);
185 }
186
187 template <typename HolderType, typename T>
188 static void unsafeAppend(HolderType& holder, T* value)
189 {
190 if constexpr (std::is_same_v<decltype(holder.builder), std::unique_ptr<arrow::FixedSizeListBuilder>>) {
191 auto status = appendToList<T>(holder.builder, value);
192 } else {
193 return holder.builder->UnsafeAppend(reinterpret_cast<const uint8_t*>(value));
194 }
195 }
196
197 template <typename HolderType, typename ITERATOR>
198 static arrow::Status append(HolderType& holder, std::pair<ITERATOR, ITERATOR> ip)
199 {
201 using ValueBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
202 // FIXME: for the moment we do not fill things.
203 auto status = holder.builder->Append();
204 auto valueBuilder = reinterpret_cast<ValueBuilderType*>(holder.builder->value_builder());
205 return status & valueBuilder->AppendValues(&*ip.first, std::distance(ip.first, ip.second));
206 }
207
208 // Lists do not have UnsafeAppend so we need to use the slow path in any case.
209 template <typename HolderType, typename ITERATOR>
210 static void unsafeAppend(HolderType& holder, std::pair<ITERATOR, ITERATOR> ip)
211 {
213 using ValueBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
214 // FIXME: for the moment we do not fill things.
215 auto status = holder.builder->Append();
216 auto valueBuilder = reinterpret_cast<ValueBuilderType*>(holder.builder->value_builder());
217 status &= valueBuilder->AppendValues(&*ip.first, std::distance(ip.first, ip.second));
218 if (!status.ok()) {
219 throw runtime_error("Unable to append values to valueBuilder!");
220 }
221 return;
222 }
223};
224
225template <typename T>
227 using FillType = T;
228 using STLValueType = T;
230 using BuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
231
232 static std::unique_ptr<BuilderType> make(arrow::MemoryPool* pool)
233 {
234 return std::make_unique<BuilderType>(pool);
235 }
236
237 static std::shared_ptr<arrow::DataType> make_datatype()
238 {
239 return arrow::TypeTraits<ArrowType>::type_singleton();
240 }
241
242 static arrow::Status append(BuilderType& builder, T value)
243 {
244 return builder.Append(value);
245 }
246
247 template <int N>
248 static arrow::Status append(BuilderType& builder, std::array<T, N>& value)
249 {
250 return builder.Append(value);
251 }
252};
253
254template <>
255struct BuilderMaker<bool> {
256 using FillType = bool;
257 using STLValueType = bool;
259 using BuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
260
261 static std::unique_ptr<BuilderType> make(arrow::MemoryPool* pool)
262 {
263 return std::make_unique<BuilderType>(pool);
264 }
265
266 static std::shared_ptr<arrow::DataType> make_datatype()
267 {
268 return arrow::TypeTraits<ArrowType>::type_singleton();
269 }
270
271 static arrow::Status append(BuilderType& builder, bool value)
272 {
273 return builder.Append(value);
274 }
275};
276
277template <typename ITERATOR>
278struct BuilderMaker<std::pair<ITERATOR, ITERATOR>> {
279 using FillType = std::pair<ITERATOR, ITERATOR>;
280 using STLValueType = typename ITERATOR::value_type;
281 using ArrowType = arrow::ListType;
283 using BuilderType = arrow::ListBuilder;
284 using ValueBuilder = typename arrow::TypeTraits<ValueType>::BuilderType;
285
286 static std::unique_ptr<BuilderType> make(arrow::MemoryPool* pool)
287 {
288 auto valueBuilder = std::make_shared<ValueBuilder>(pool);
289 return std::make_unique<arrow::ListBuilder>(pool, valueBuilder);
290 }
291
292 static std::shared_ptr<arrow::DataType> make_datatype()
293 {
294 return arrow::list(arrow::TypeTraits<ValueType>::type_singleton());
295 }
296};
297
298template <typename T, int N>
299struct BuilderMaker<T (&)[N]> {
300 using FillType = T*;
301 using STLValueType = T;
302 using BuilderType = arrow::FixedSizeListBuilder;
303 using ArrowType = arrow::FixedSizeListType;
305
306 static std::unique_ptr<BuilderType> make(arrow::MemoryPool* pool)
307 {
308 std::unique_ptr<arrow::ArrayBuilder> valueBuilder;
309 auto status =
310 arrow::MakeBuilder(pool, arrow::TypeTraits<ElementType>::type_singleton(), &valueBuilder);
311 return std::make_unique<BuilderType>(pool, std::move(valueBuilder), N);
312 }
313
314 static std::shared_ptr<arrow::DataType> make_datatype()
315 {
316 return arrow::fixed_size_list(arrow::TypeTraits<ElementType>::type_singleton(), N);
317 }
318};
319
320template <typename T, int N>
321struct BuilderMaker<T[N]> {
322 using FillType = T*;
323 using BuilderType = arrow::FixedSizeListBuilder;
324 using ArrowType = arrow::FixedSizeListType;
326
327 static std::unique_ptr<BuilderType> make(arrow::MemoryPool* pool)
328 {
329 std::unique_ptr<arrow::ArrayBuilder> valueBuilder;
330 auto status =
331 arrow::MakeBuilder(pool, arrow::TypeTraits<ElementType>::type_singleton(), &valueBuilder);
332 return std::make_unique<BuilderType>(pool, std::move(valueBuilder), N);
333 }
334
335 static std::shared_ptr<arrow::DataType> make_datatype()
336 {
337 return arrow::fixed_size_list(arrow::TypeTraits<ElementType>::type_singleton(), N);
338 }
339};
340
341template <typename T, int N>
342struct BuilderMaker<std::array<T, N>> {
343 using FillType = T*;
344 using BuilderType = arrow::FixedSizeListBuilder;
345 using ArrowType = arrow::FixedSizeListType;
347
348 static std::unique_ptr<BuilderType> make(arrow::MemoryPool* pool)
349 {
350 std::unique_ptr<arrow::ArrayBuilder> valueBuilder;
351 auto status =
352 arrow::MakeBuilder(pool, arrow::TypeTraits<ElementType>::type_singleton(), &valueBuilder);
353 return std::make_unique<BuilderType>(pool, std::move(valueBuilder), N);
354 }
355
356 static std::shared_ptr<arrow::DataType> make_datatype()
357 {
358 return arrow::fixed_size_list(arrow::TypeTraits<ElementType>::type_singleton(), N);
359 }
360};
361
362template <typename T>
363struct BuilderMaker<std::vector<T>> {
364 using FillType = std::vector<T>;
365 using BuilderType = arrow::ListBuilder;
366 using ArrowType = arrow::ListType;
368
369 static std::unique_ptr<BuilderType> make(arrow::MemoryPool* pool)
370 {
371 std::unique_ptr<arrow::ArrayBuilder> valueBuilder;
372 auto status =
373 arrow::MakeBuilder(pool, arrow::TypeTraits<ElementType>::type_singleton(), &valueBuilder);
374 return std::make_unique<BuilderType>(pool, std::move(valueBuilder));
375 }
376
377 static std::shared_ptr<arrow::DataType> make_datatype()
378 {
379 return arrow::list(arrow::TypeTraits<ElementType>::type_singleton());
380 }
381};
382
383template <typename... ARGS>
385{
386 return std::make_tuple(std::make_unique<ARGS>()...);
387}
388
389template <typename T>
392 using BuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
393};
394
395// Support for building tables where each entry is an iterator pair.
396// We map them to an arrow::list for now.
397template <typename ITERATOR>
398struct BuilderTraits<std::pair<ITERATOR, ITERATOR>> {
399 using ArrowType = arrow::ListType;
400 using BuilderType = arrow::ListBuilder;
401};
402
403// Support for building array columns
404// FIXME: move to use FixedSizeList<T> once we move to 0.16.1
405template <typename T, int N>
406struct BuilderTraits<T[N]> {
407 using ArrowType = arrow::FixedSizeListType;
408 using BuilderType = arrow::FixedSizeListBuilder;
409};
410
411template <typename T>
412struct BuilderTraits<std::vector<T>> {
413 using ArrowType = arrow::ListType;
414 using BuilderType = arrow::ListBuilder;
415};
416
417template <typename T>
419 template <typename BUILDER>
420 arrow::Status append(BUILDER& builder, T value)
421 {
422 return builder->Append(value);
423 }
424
425 template <typename BUILDER>
426 arrow::Status flush(BUILDER&)
427 {
428 return arrow::Status::OK();
429 }
430};
431
432template <typename T>
434 static constexpr int CHUNK_SIZE = 256;
435
436 template <typename BUILDER>
437 arrow::Status append(BUILDER& builder, T value)
438 {
439 cache[pos % CHUNK_SIZE] = value;
440 ++pos;
441 if (pos % CHUNK_SIZE == 0) {
442 return builder->AppendValues(cache, CHUNK_SIZE, nullptr);
443 }
444 return arrow::Status::OK();
445 }
446
447 template <typename BUILDER>
448 arrow::Status flush(BUILDER& builder)
449 {
450 if (pos % CHUNK_SIZE != 0) {
451 return builder->AppendValues(cache, pos % CHUNK_SIZE, nullptr);
452 }
453 return arrow::Status::OK();
454 }
455 T cache[CHUNK_SIZE];
456 int pos = 0;
457};
458
459template <typename T>
461 static consteval DirectInsertion<T> policy();
462 using Policy = decltype(policy());
463};
464
465template <size_t I, typename T>
466struct BuilderHolder : InsertionTrait<T>::Policy {
467 static constexpr size_t index = I;
470 using BuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
471
472 BuilderHolder(arrow::MemoryPool* pool, size_t nRows = 0)
473 : builder{BuilderMaker<T>::make(pool)}
474 {
475 if (nRows > 0) {
476 auto s = builder->Reserve(nRows);
477 if (!s.ok()) {
478 throw runtime_error_f("Unable to reserve %ll rows", nRows);
479 }
480 }
481 }
482
483 std::unique_ptr<BuilderType> builder;
484};
485
487 template <typename... ARGS, size_t NCOLUMNS>
488 static std::array<arrow::DataType, NCOLUMNS> makeArrowColumnTypes()
489 {
491 }
492
494 template <typename... Ts, typename VALUES>
495 static bool append(std::tuple<Ts...>& holders, VALUES&& values)
496 {
497 return (BuilderUtils::append(std::get<Ts::index>(holders), std::get<Ts::index>(values)).ok() && ...);
498 }
499
503 template <typename... Ts, typename VALUES>
504 static void unsafeAppend(std::tuple<Ts...>& holders, VALUES&& values)
505 {
506 (BuilderUtils::unsafeAppend(std::get<Ts::index>(holders), std::get<Ts::index>(values)), ...);
507 }
508
510 template <typename... Ts>
511 static bool finalize(std::vector<std::shared_ptr<arrow::Array>>& arrays, std::tuple<Ts...>& holders)
512 {
513 return (finalize(arrays[Ts::index], std::get<Ts::index>(holders)) && ...);
514 }
515
516 template <typename HOLDER>
517 static bool finalize(std::shared_ptr<arrow::Array>& array, HOLDER& holder)
518 {
519 return BuilderUtils::flush(holder).ok() && holder.builder->Finish(&array).ok();
520 }
521};
522
523template <typename... ARGS>
524constexpr auto tuple_to_pack(std::tuple<ARGS...>&&)
525{
526 return framework::pack<ARGS...>{};
527}
528
531template <class T>
532auto constexpr to_tuple(T&& object) noexcept
533{
534 using type = std::decay_t<T>;
536 auto&& [p0, p1, p2, p3] = object;
537 return std::make_tuple(p0, p1, p2, p3);
539 auto&& [p0, p1, p2] = object;
540 return std::make_tuple(p0, p1, p2);
542 auto&& [p0, p1] = object;
543 return std::make_tuple(p0, p1);
544 } else if constexpr (is_braces_constructible<type, any_type>{}) {
545 auto&& [p0] = object;
546 return std::make_tuple(p0);
547 } else {
548 return std::make_tuple();
549 }
550}
551
552template <typename... ARGS>
553constexpr auto makeHolderTypes()
554{
555 return []<std::size_t... Is>(std::index_sequence<Is...>) {
556 return std::tuple(BuilderHolder<Is, ARGS>(arrow::default_memory_pool())...);
557 }(std::make_index_sequence<sizeof...(ARGS)>{});
558}
559
560template <typename... ARGS>
561auto makeHolders(arrow::MemoryPool* pool, size_t nRows)
562{
563 return [pool, nRows]<std::size_t... Is>(std::index_sequence<Is...>) {
564 return new std::tuple(BuilderHolder<Is, ARGS>(pool, nRows)...);
565 }(std::make_index_sequence<sizeof...(ARGS)>{});
566}
567
568template <typename... ARGS>
569using IndexedHoldersTuple = decltype(makeHolderTypes<ARGS...>());
570
571template <typename T>
572concept ShouldNotDeconstruct = std::is_bounded_array_v<T> || std::is_arithmetic_v<T> || framework::is_base_of_template_v<std::vector, T>;
573
578{
579 static void throwError(RuntimeErrorRef const& ref);
580
581 template <typename... ARGS>
582 using HoldersTuple = typename std::tuple<BuilderHolder<0, ARGS>...>;
583
584 template <typename... ARGS>
585 using HoldersTupleIndexed = decltype(makeHolderTypes<ARGS...>());
586
589 template <typename... ARGS>
590 auto getBuilders(o2::framework::pack<ARGS...>)
591 {
592 return (HoldersTupleIndexed<ARGS...>*)mHolders;
593 }
594
595 void validate() const;
596
597 template <typename... ARGS, size_t I = sizeof...(ARGS)>
598 auto makeBuilders(std::array<char const*, I> const& columnNames, size_t nRows)
599 {
600 char const* const* names_ptr = columnNames.data();
601 mSchema = std::make_shared<arrow::Schema>(
602 std::vector<std::shared_ptr<arrow::Field>>({std::make_shared<arrow::Field>(*names_ptr++, BuilderMaker<ARGS>::make_datatype(), true, nullptr)...}));
603
604 mHolders = makeHolders<ARGS...>(mMemoryPool, nRows);
605 mFinalizer = [](std::vector<std::shared_ptr<arrow::Array>>& arrays, void* holders) -> bool {
606 return TableBuilderHelpers::finalize(arrays, *(HoldersTupleIndexed<ARGS...>*)holders);
607 };
608 mDestructor = [](void* holders) mutable -> void {
609 delete (HoldersTupleIndexed<ARGS...>*)holders;
610 };
611 }
612
613 public:
614 template <typename ARG0, typename... ARGS>
615 requires(sizeof...(ARGS) == 0) && (!ShouldNotDeconstruct<ARG0>)
616 static constexpr int countColumns()
617 {
618 using argsPack_t = decltype(tuple_to_pack(framework::to_tuple(std::declval<ARG0>())));
619 return framework::pack_size(argsPack_t{});
620 }
621
622 template <typename ARG0, typename... ARGS>
623 requires(sizeof...(ARGS) > 0) || ShouldNotDeconstruct<ARG0>
624 static constexpr int countColumns()
625 {
626 return 1 + sizeof...(ARGS);
627 }
628
629 void setLabel(const char* label);
630
631 TableBuilder(arrow::MemoryPool* pool = arrow::default_memory_pool())
632 : mHolders{nullptr},
633 mMemoryPool{pool}
634 {
635 }
636
638 {
639 mDestructor(mHolders);
640 }
641
644 template <typename ARG0, typename... ARGS>
645 requires(sizeof...(ARGS) > 0) || ShouldNotDeconstruct<ARG0>
646 auto persist(std::array<char const*, sizeof...(ARGS) + 1> const& columnNames)
647 {
648 auto persister = persistTuple(framework::pack<ARG0, ARGS...>{}, columnNames);
649 // Callback used to fill the builders
650 return [persister = persister](unsigned int slot, typename BuilderMaker<ARG0>::FillType const& arg, typename BuilderMaker<ARGS>::FillType... args) -> void {
651 persister(slot, std::forward_as_tuple(arg, args...));
652 };
653 }
654
655 // Special case for a single parameter to handle the serialization of struct
656 // which can be decomposed
657 template <typename ARG0, typename... ARGS>
658 requires(sizeof...(ARGS) == 0) && (!ShouldNotDeconstruct<ARG0>)
659 auto persist(std::array<char const*, countColumns<ARG0, ARGS...>()> const& columnNames)
660 {
661 using argsPack_t = decltype(tuple_to_pack(framework::to_tuple(std::declval<ARG0>())));
662 auto persister = persistTuple(argsPack_t{}, columnNames);
663 return [persister = persister](unsigned int slot, ARG0 const& obj) -> void {
664 auto t = to_tuple(obj);
665 persister(slot, t);
666 };
667 }
668
670 template <typename... ARGS>
671 auto persistTuple(framework::pack<ARGS...>, std::array<char const*, sizeof...(ARGS)> const& columnNames)
672 {
673 constexpr int nColumns = sizeof...(ARGS);
674 validate();
675 mArrays.resize(nColumns);
676 makeBuilders<ARGS...>(columnNames, 10);
677
678 // Callback used to fill the builders
679 using FillTuple = std::tuple<typename BuilderMaker<ARGS>::FillType...>;
680 return [holders = mHolders](unsigned int /*slot*/, FillTuple const& t) -> void {
681 auto status = TableBuilderHelpers::append(*(HoldersTupleIndexed<ARGS...>*)holders, t);
682 if (status == false) {
683 throwError(runtime_error("Unable to append"));
684 }
685 };
686 }
687
688 // Same as above, but starting from a o2::soa::Table, which has all the
689 // information already available.
690 template <typename T>
691 auto cursor()
692 {
693 return [this]<typename... Cs>(pack<Cs...>) {
694 return this->template persist<typename Cs::type...>({Cs::columnLabel()...});
695 }(typename T::table_t::persistent_columns_t{});
696 }
697
698 template <typename... Cs>
700 {
701 return this->template persist<typename Cs::type...>({Cs::columnLabel()...});
702 }
703
704 template <typename T, typename E>
705 auto cursor()
706 {
707 return [this]<typename... Cs>(pack<Cs...>) {
708 return this->template persist<E>({Cs::columnLabel()...});
709 }(typename T::table_t::persistent_columns_t{});
710 }
711
713 template <typename... Ts>
714 auto reserveArrays(std::tuple<Ts...>& holders, int s)
715 {
716 return (std::get<Ts::index>(holders).builder->Reserve(s).ok() && ...);
717 }
718
719 template <typename... ARGS>
721 {
722 reserveArrays(*(HoldersTupleIndexed<ARGS...>*)mHolders, s);
723 }
724
726 void extracted(bool& status);
727 std::shared_ptr<arrow::Table> finalize();
728
729 private:
730 bool (*mFinalizer)(std::vector<std::shared_ptr<arrow::Array>>& arrays, void* holders);
731 void (*mDestructor)(void* holders);
732 void* mHolders;
733 arrow::MemoryPool* mMemoryPool;
734 std::shared_ptr<arrow::Schema> mSchema;
735 std::vector<std::shared_ptr<arrow::Array>> mArrays;
736};
737
738template <typename T>
739auto makeEmptyTable(const char* name)
740{
742 [[maybe_unused]] auto writer = b.cursor<T>();
743 b.setLabel(name);
744 return b.finalize();
745}
746
747template <soa::TableRef R>
749{
751 [[maybe_unused]] auto writer = b.cursor(typename aod::MetadataTrait<aod::Hash<R.desc_hash>>::metadata::persistent_columns_t{});
752 b.setLabel(aod::label<R>());
753 return b.finalize();
754}
755
756template <typename... Cs>
758{
760 [[maybe_unused]] auto writer = b.cursor(p);
761 b.setLabel(name);
762 return b.finalize();
763}
764
765std::shared_ptr<arrow::Table> spawnerHelper(std::shared_ptr<arrow::Table> const& fullTable, std::shared_ptr<arrow::Schema> newSchema, size_t nColumns,
766 expressions::Projector* projectors, const char* name, std::shared_ptr<gandiva::Projector>& projector);
767
769template <aod::is_aod_hash D>
771auto spawner(std::shared_ptr<arrow::Table> const& fullTable, const char* name, o2::framework::expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema> const& schema)
772{
773 using placeholders_pack_t = typename o2::aod::MetadataTrait<D>::metadata::placeholders_pack_t;
774 if (fullTable->num_rows() == 0) {
775 return makeEmptyTable(name, placeholders_pack_t{});
776 }
777 return spawnerHelper(fullTable, schema, framework::pack_size(placeholders_pack_t{}), projectors, name, projector);
778}
779
780template <aod::is_aod_hash D>
782auto spawner(std::vector<std::shared_ptr<arrow::Table>>&& tables, const char* name, o2::framework::expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema> const& schema)
783{
784 auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables), std::span{o2::aod::MetadataTrait<D>::metadata::base_table_t::originalLabels});
785 return spawner<D>(fullTable, name, projectors, projector, schema);
786}
787
788template <aod::is_aod_hash D>
790auto spawner(std::shared_ptr<arrow::Table> const& fullTable, const char* name, expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema> const& schema)
791{
792 using expression_pack_t = typename o2::aod::MetadataTrait<D>::metadata::expression_pack_t;
793 if (fullTable->num_rows() == 0) {
794 return makeEmptyTable(name, expression_pack_t{});
795 }
796 return spawnerHelper(fullTable, schema, framework::pack_size(expression_pack_t{}), projectors, name, projector);
797}
798
799template <aod::is_aod_hash D>
801auto spawner(std::vector<std::shared_ptr<arrow::Table>>&& tables, const char* name, expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema> const& schema)
802{
803 auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables), std::span{o2::aod::MetadataTrait<D>::metadata::base_table_t::originalLabels});
804 return spawner<D>(fullTable, name, projectors, projector, schema);
805}
806
807template <typename... C>
808auto spawner(framework::pack<C...>, std::vector<std::shared_ptr<arrow::Table>>&& tables, const char* name, expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema> const& schema)
809{
810 std::array<const char*, 1> labels{"original"};
811 auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables), std::span<const char* const>{labels});
812 if (fullTable->num_rows() == 0) {
814 }
815 return spawnerHelper(fullTable, schema, sizeof...(C), projectors, name, projector);
816}
817
818template <typename... T>
819using iterator_tuple_t = std::tuple<typename T::iterator...>;
820} // namespace o2::framework
821#endif // FRAMEWORK_TABLEBUILDER_H
constexpr int p2()
constexpr int p1()
constexpr to accelerate the coordinates changing
uint16_t pos
Definition RawData.h:3
#define O2_ARROW_STL_CONVERSION(c_type, ArrowType_)
auto cursor(framework::pack< Cs... >)
auto persist(std::array< char const *, countColumns< ARG0, ARGS... >()> const &columnNames)
auto persistTuple(framework::pack< ARGS... >, std::array< char const *, sizeof...(ARGS)> const &columnNames)
Same a the above, but use a tuple to persist stuff.
static constexpr int countColumns()
void extracted(bool &status)
Actually creates the arrow::Table from the builders.
static constexpr int countColumns()
auto reserve(o2::framework::pack< ARGS... > &&, int s)
TableBuilder(arrow::MemoryPool *pool=arrow::default_memory_pool())
auto reserveArrays(std::tuple< Ts... > &holders, int s)
Reserve method to expand the columns as needed.
auto persist(std::array< char const *, sizeof...(ARGS)+1 > const &columnNames)
GLsizeiptr size
Definition glcorearb.h:659
GLenum array
Definition glcorearb.h:4274
GLuint index
Definition glcorearb.h:781
GLuint const GLchar * name
Definition glcorearb.h:781
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei const GLfloat * value
Definition glcorearb.h:819
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
GLenum GLsizei GLsizei GLint * values
Definition glcorearb.h:1576
GLboolean * data
Definition glcorearb.h:298
GLuint GLsizei const GLchar * label
Definition glcorearb.h:2519
typedef void(APIENTRYP PFNGLCULLFACEPROC)(GLenum mode)
GLuint object
Definition glcorearb.h:4041
const GLuint * arrays
Definition glcorearb.h:1314
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
void addLabelToSchema(std::shared_ptr< arrow::Schema > &schema, const char *label)
RuntimeErrorRef runtime_error(const char *)
constexpr auto makeHolderTypes()
auto constexpr to_tuple(T &&object) noexcept
auto makeEmptyTable(const char *name)
auto spawner(std::shared_ptr< arrow::Table > const &fullTable, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector, std::shared_ptr< arrow::Schema > const &schema)
Expression-based column generator to materialize columns.
constexpr auto tuple_to_pack(std::tuple< ARGS... > &&)
auto makeHolders(arrow::MemoryPool *pool, size_t nRows)
std::shared_ptr< arrow::Table > spawnerHelper(std::shared_ptr< arrow::Table > const &fullTable, std::shared_ptr< arrow::Schema > newSchema, size_t nColumns, expressions::Projector *projectors, const char *name, std::shared_ptr< gandiva::Projector > &projector)
decltype(makeHolderTypes< ARGS... >()) IndexedHoldersTuple
auto make_builders()
RuntimeErrorRef runtime_error_f(const char *,...)
std::tuple< typename T::iterator... > iterator_tuple_t
Defining DataPointCompositeObject explicitly as copiable.
typename detail::ConversionTraits< T >::ArrowType ArrowType
std::unique_ptr< BuilderType > builder
typename arrow::TypeTraits< ArrowType >::BuilderType BuilderType
typename InsertionTrait< T >::Policy Policy
BuilderHolder(arrow::MemoryPool *pool, size_t nRows=0)
typename detail::ConversionTraits< T >::ArrowType ElementType
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
arrow::FixedSizeListBuilder BuilderType
arrow::FixedSizeListType ArrowType
static std::shared_ptr< arrow::DataType > make_datatype()
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
typename detail::ConversionTraits< T >::ArrowType ElementType
arrow::FixedSizeListType ArrowType
static std::shared_ptr< arrow::DataType > make_datatype()
arrow::FixedSizeListBuilder BuilderType
typename arrow::TypeTraits< ArrowType >::BuilderType BuilderType
static arrow::Status append(BuilderType &builder, bool value)
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
static std::shared_ptr< arrow::DataType > make_datatype()
typename detail::ConversionTraits< bool >::ArrowType ArrowType
typename detail::ConversionTraits< T >::ArrowType ElementType
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
static std::shared_ptr< arrow::DataType > make_datatype()
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
typename detail::ConversionTraits< typename ITERATOR::value_type >::ArrowType ValueType
typename arrow::TypeTraits< ValueType >::BuilderType ValueBuilder
static std::shared_ptr< arrow::DataType > make_datatype()
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
typename detail::ConversionTraits< T >::ArrowType ElementType
static std::shared_ptr< arrow::DataType > make_datatype()
typename arrow::TypeTraits< ArrowType >::BuilderType BuilderType
static arrow::Status append(BuilderType &builder, std::array< T, N > &value)
static arrow::Status append(BuilderType &builder, T value)
typename detail::ConversionTraits< T >::ArrowType ArrowType
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
static std::shared_ptr< arrow::DataType > make_datatype()
arrow::FixedSizeListBuilder BuilderType
arrow::FixedSizeListType ArrowType
typename arrow::TypeTraits< ArrowType >::BuilderType BuilderType
typename detail::ConversionTraits< T >::ArrowType ArrowType
static arrow::Status append(HolderType &holder, std::vector< T > const &data)
Appender for the vector case.
static arrow::Status appendToList(std::unique_ptr< arrow::FixedSizeListBuilder > &builder, T *data, int size=1)
static arrow::Status flush(HolderType &holder)
static arrow::Status append(HolderType &holder, T value)
static arrow::Status append(HolderType &holder, T(&data)[N])
Appender for the array case.
static arrow::Status append(HolderType &holder, T *data)
static void unsafeAppend(HolderType &holder, std::vector< T > const &value)
static arrow::Status append(HolderType &holder, std::pair< ITERATOR, ITERATOR > ip)
static void unsafeAppend(HolderType &holder, T *value)
static void unsafeAppend(HolderType &holder, std::pair< ITERATOR, ITERATOR > ip)
static arrow::Status append(HolderType &holder, std::array< T, N > const &data)
Appender for the array case.
static void unsafeAppend(HolderType &holder, T value)
arrow::Status append(BUILDER &builder, T value)
arrow::Status flush(BUILDER &builder)
arrow::Status append(BUILDER &builder, T value)
arrow::Status flush(BUILDER &)
static consteval DirectInsertion< T > policy()
static bool finalize(std::shared_ptr< arrow::Array > &array, HOLDER &holder)
static bool append(std::tuple< Ts... > &holders, VALUES &&values)
Invokes the append method for each entry in the tuple.
static std::array< arrow::DataType, NCOLUMNS > makeArrowColumnTypes()
static bool finalize(std::vector< std::shared_ptr< arrow::Array > > &arrays, std::tuple< Ts... > &holders)
Invokes the append method for each entry in the tuple.
static void unsafeAppend(std::tuple< Ts... > &holders, VALUES &&values)
FIXME: adapt type conversion to new arrow.
A struct, containing the root of the expression tree.