12#ifndef O2_FRAMEWORK_TABLEBUILDER_H_
13#define O2_FRAMEWORK_TABLEBUILDER_H_
18#include "arrow/type_traits.h"
23#include <arrow/chunked_array.h>
24#include <arrow/status.h>
25#include <arrow/memory_pool.h>
27#include <arrow/type_traits.h>
28#include <arrow/table.h>
29#include <arrow/builder.h>
45extern template class arrow::NumericBuilder<arrow::UInt8Type>;
46extern template class arrow::NumericBuilder<arrow::UInt32Type>;
47extern template class arrow::NumericBuilder<arrow::FloatType>;
48extern template class arrow::NumericBuilder<arrow::Int32Type>;
49extern template class arrow::NumericBuilder<arrow::Int8Type>;
60template <
typename T,
int N>
65template <
typename T,
int N>
70template <
typename T,
int N>
80#define O2_ARROW_STL_CONVERSION(c_type, ArrowType_) \
82 struct ConversionTraits<c_type> { \
83 using ArrowType = ::arrow::ArrowType_; \
106 template <
typename T>
107 static arrow::Status
appendToList(std::unique_ptr<arrow::FixedSizeListBuilder>& builder, T*
data,
int size = 1)
110 using BuilderType =
typename arrow::TypeTraits<ArrowType>::BuilderType;
111 size_t numElements =
static_cast<const arrow::FixedSizeListType*
>(builder->type().get())->list_size();
113 auto status = builder->AppendValues(
size);
114 auto ValueBuilder =
static_cast<BuilderType*
>(builder->value_builder());
115 status &= ValueBuilder->AppendValues(
data, numElements *
size,
nullptr);
120 template <
typename HolderType,
typename T>
123 return static_cast<typename HolderType::Policy&
>(holder).append(holder.builder,
value);
126 template <
typename HolderType>
127 static arrow::Status
flush(HolderType& holder)
129 return static_cast<typename HolderType::Policy&
>(holder).flush(holder.builder);
135 template <
typename HolderType,
typename T>
138 if constexpr (std::is_same_v<
decltype(holder.builder), std::unique_ptr<arrow::FixedSizeListBuilder>>) {
139 return appendToList<T>(holder.builder,
data);
141 return holder.builder->Append(
reinterpret_cast<const uint8_t*
>(
data));
145 template <
typename HolderType,
typename T,
int N>
146 static arrow::Status
append(HolderType& holder, T (&
data)[N])
148 return holder.builder->Append(
reinterpret_cast<const uint8_t*
>(
data));
152 template <
typename HolderType,
typename T,
int N>
153 static arrow::Status
append(HolderType& holder, std::array<T, N>
const&
data)
155 return holder.builder->Append(
reinterpret_cast<const uint8_t*
>(
data.data()));
159 template <
typename HolderType,
typename T>
160 static arrow::Status
append(HolderType& holder, std::vector<T>
const&
data)
163 using ValueBuilderType =
typename arrow::TypeTraits<ArrowType>::BuilderType;
164 auto status = holder.builder->Reserve(
data.size());
165 status &= holder.builder->Append();
166 auto vbuilder =
static_cast<ValueBuilderType*
>(holder.builder->value_builder());
167 status &= vbuilder->AppendValues(
data.begin(),
data.end());
172 template <
typename HolderType,
typename T>
175 auto status = append(holder,
value);
181 template <
typename HolderType,
typename T>
184 return holder.builder->UnsafeAppend(
value);
187 template <
typename HolderType,
typename T>
190 if constexpr (std::is_same_v<
decltype(holder.builder), std::unique_ptr<arrow::FixedSizeListBuilder>>) {
191 auto status = appendToList<T>(holder.builder,
value);
193 return holder.builder->UnsafeAppend(
reinterpret_cast<const uint8_t*
>(
value));
197 template <
typename HolderType,
typename ITERATOR>
198 static arrow::Status
append(HolderType& holder, std::pair<ITERATOR, ITERATOR> ip)
201 using ValueBuilderType =
typename arrow::TypeTraits<ArrowType>::BuilderType;
203 auto status = holder.builder->Append();
204 auto valueBuilder =
reinterpret_cast<ValueBuilderType*
>(holder.builder->value_builder());
205 return status & valueBuilder->AppendValues(&*ip.first, std::distance(ip.first, ip.second));
209 template <
typename HolderType,
typename ITERATOR>
210 static void unsafeAppend(HolderType& holder, std::pair<ITERATOR, ITERATOR> ip)
213 using ValueBuilderType =
typename arrow::TypeTraits<ArrowType>::BuilderType;
215 auto status = holder.builder->Append();
216 auto valueBuilder =
reinterpret_cast<ValueBuilderType*
>(holder.builder->value_builder());
217 status &= valueBuilder->AppendValues(&*ip.first, std::distance(ip.first, ip.second));
219 throw runtime_error(
"Unable to append values to valueBuilder!");
230 using BuilderType =
typename arrow::TypeTraits<ArrowType>::BuilderType;
232 static std::unique_ptr<BuilderType>
make(arrow::MemoryPool* pool)
234 return std::make_unique<BuilderType>(pool);
239 return arrow::TypeTraits<ArrowType>::type_singleton();
244 return builder.Append(
value);
250 return builder.Append(
value);
259 using BuilderType =
typename arrow::TypeTraits<ArrowType>::BuilderType;
261 static std::unique_ptr<BuilderType>
make(arrow::MemoryPool* pool)
263 return std::make_unique<BuilderType>(pool);
268 return arrow::TypeTraits<ArrowType>::type_singleton();
273 return builder.Append(
value);
277template <
typename ITERATOR>
284 using ValueBuilder =
typename arrow::TypeTraits<ValueType>::BuilderType;
286 static std::unique_ptr<BuilderType>
make(arrow::MemoryPool* pool)
288 auto valueBuilder = std::make_shared<ValueBuilder>(pool);
289 return std::make_unique<arrow::ListBuilder>(pool, valueBuilder);
294 return arrow::list(arrow::TypeTraits<ValueType>::type_singleton());
298template <
typename T,
int N>
306 static std::unique_ptr<BuilderType>
make(arrow::MemoryPool* pool)
308 std::unique_ptr<arrow::ArrayBuilder> valueBuilder;
310 arrow::MakeBuilder(pool, arrow::TypeTraits<ElementType>::type_singleton(), &valueBuilder);
311 return std::make_unique<BuilderType>(pool, std::move(valueBuilder), N);
316 return arrow::fixed_size_list(arrow::TypeTraits<ElementType>::type_singleton(), N);
320template <
typename T,
int N>
327 static std::unique_ptr<BuilderType>
make(arrow::MemoryPool* pool)
329 std::unique_ptr<arrow::ArrayBuilder> valueBuilder;
331 arrow::MakeBuilder(pool, arrow::TypeTraits<ElementType>::type_singleton(), &valueBuilder);
332 return std::make_unique<BuilderType>(pool, std::move(valueBuilder), N);
337 return arrow::fixed_size_list(arrow::TypeTraits<ElementType>::type_singleton(), N);
341template <
typename T,
int N>
348 static std::unique_ptr<BuilderType>
make(arrow::MemoryPool* pool)
350 std::unique_ptr<arrow::ArrayBuilder> valueBuilder;
352 arrow::MakeBuilder(pool, arrow::TypeTraits<ElementType>::type_singleton(), &valueBuilder);
353 return std::make_unique<BuilderType>(pool, std::move(valueBuilder), N);
358 return arrow::fixed_size_list(arrow::TypeTraits<ElementType>::type_singleton(), N);
369 static std::unique_ptr<BuilderType>
make(arrow::MemoryPool* pool)
371 std::unique_ptr<arrow::ArrayBuilder> valueBuilder;
373 arrow::MakeBuilder(pool, arrow::TypeTraits<ElementType>::type_singleton(), &valueBuilder);
374 return std::make_unique<BuilderType>(pool, std::move(valueBuilder));
379 return arrow::list(arrow::TypeTraits<ElementType>::type_singleton());
383template <
typename... ARGS>
386 return std::make_tuple(std::make_unique<ARGS>()...);
392 using BuilderType =
typename arrow::TypeTraits<ArrowType>::BuilderType;
397template <
typename ITERATOR>
405template <
typename T,
int N>
419 template <
typename BUILDER>
422 return builder->Append(
value);
425 template <
typename BUILDER>
428 return arrow::Status::OK();
434 static constexpr int CHUNK_SIZE = 256;
436 template <
typename BUILDER>
441 if (
pos % CHUNK_SIZE == 0) {
442 return builder->AppendValues(cache, CHUNK_SIZE,
nullptr);
444 return arrow::Status::OK();
447 template <
typename BUILDER>
448 arrow::Status
flush(BUILDER& builder)
450 if (
pos % CHUNK_SIZE != 0) {
451 return builder->AppendValues(cache,
pos % CHUNK_SIZE,
nullptr);
453 return arrow::Status::OK();
465template <
size_t I,
typename T>
470 using BuilderType =
typename arrow::TypeTraits<ArrowType>::BuilderType;
476 auto s = builder->Reserve(nRows);
487 template <
typename... ARGS,
size_t NCOLUMNS>
494 template <
typename...
Ts,
typename VALUES>
497 return (BuilderUtils::append(std::get<Ts::index>(holders), std::get<Ts::index>(
values)).ok() && ...);
503 template <
typename...
Ts,
typename VALUES>
506 (BuilderUtils::unsafeAppend(std::get<Ts::index>(holders), std::get<Ts::index>(
values)), ...);
510 template <
typename...
Ts>
511 static bool finalize(std::vector<std::shared_ptr<arrow::Array>>&
arrays, std::tuple<Ts...>& holders)
513 return (finalize(
arrays[Ts::index], std::get<Ts::index>(holders)) && ...);
516 template <
typename HOLDER>
519 return BuilderUtils::flush(holder).ok() && holder.builder->Finish(&
array).ok();
523template <
typename... ARGS>
534 using type = std::decay_t<T>;
537 return std::make_tuple(p0,
p1,
p2, p3);
540 return std::make_tuple(p0,
p1,
p2);
543 return std::make_tuple(p0,
p1);
546 return std::make_tuple(p0);
548 return std::make_tuple();
552template <
typename... ARGS>
555 return []<std::size_t... Is>(std::index_sequence<Is...>) {
557 }(std::make_index_sequence<
sizeof...(ARGS)>{});
560template <
typename... ARGS>
563 return [pool, nRows]<std::size_t... Is>(std::index_sequence<Is...>) {
565 }(std::make_index_sequence<
sizeof...(ARGS)>{});
568template <
typename... ARGS>
572concept ShouldNotDeconstruct = std::is_bounded_array_v<T> || std::is_arithmetic_v<T> || framework::is_base_of_template_v<std::vector, T>;
581 template <
typename... ARGS>
582 using HoldersTuple =
typename std::tuple<BuilderHolder<0, ARGS>...>;
584 template <
typename... ARGS>
589 template <
typename... ARGS>
592 return (HoldersTupleIndexed<ARGS...>*)mHolders;
595 void validate()
const;
597 template <
typename... ARGS,
size_t I =
sizeof...(ARGS)>
598 auto makeBuilders(std::array<char const*, I>
const& columnNames,
size_t nRows)
600 char const*
const* names_ptr = columnNames.data();
601 mSchema = std::make_shared<arrow::Schema>(
604 mHolders =
makeHolders<ARGS...>(mMemoryPool, nRows);
605 mFinalizer = [](std::vector<std::shared_ptr<arrow::Array>>&
arrays,
void* holders) ->
bool {
606 return TableBuilderHelpers::finalize(
arrays, *(HoldersTupleIndexed<ARGS...>*)holders);
608 mDestructor = [](
void* holders)
mutable ->
void {
609 delete (HoldersTupleIndexed<ARGS...>*)holders;
614 template <
typename ARG0,
typename... ARGS>
618 using argsPack_t =
decltype(
tuple_to_pack(framework::to_tuple(std::declval<ARG0>())));
619 return framework::pack_size(argsPack_t{});
622 template <
typename ARG0,
typename... ARGS>
626 return 1 +
sizeof...(ARGS);
629 void setLabel(
const char*
label);
639 mDestructor(mHolders);
644 template <
typename ARG0,
typename... ARGS>
646 auto persist(std::array<
char const*,
sizeof...(ARGS) + 1>
const& columnNames)
651 persister(slot, std::forward_as_tuple(arg, args...));
657 template <
typename ARG0,
typename... ARGS>
659 auto persist(std::array<
char const*, countColumns<ARG0, ARGS...>()>
const& columnNames)
661 using argsPack_t =
decltype(
tuple_to_pack(framework::to_tuple(std::declval<ARG0>())));
662 auto persister = persistTuple(argsPack_t{}, columnNames);
663 return [persister = persister](
unsigned int slot, ARG0
const& obj) ->
void {
670 template <
typename... ARGS>
673 constexpr int nColumns =
sizeof...(ARGS);
675 mArrays.resize(nColumns);
676 makeBuilders<ARGS...>(columnNames, 10);
679 using FillTuple = std::tuple<typename BuilderMaker<ARGS>::FillType...>;
680 return [holders = mHolders](
unsigned int , FillTuple
const& t) ->
void {
681 auto status = TableBuilderHelpers::append(*(HoldersTupleIndexed<ARGS...>*)holders, t);
682 if (status ==
false) {
690 template <
typename T>
693 return [
this]<
typename... Cs>(
pack<Cs...>) {
694 return this->
template persist<
typename Cs::type...>({Cs::columnLabel()...});
695 }(
typename T::table_t::persistent_columns_t{});
698 template <
typename... Cs>
701 return this->
template persist<
typename Cs::type...>({Cs::columnLabel()...});
704 template <
typename T,
typename E>
707 return [
this]<
typename... Cs>(
pack<Cs...>) {
708 return this->
template persist<E>({Cs::columnLabel()...});
709 }(
typename T::table_t::persistent_columns_t{});
713 template <
typename...
Ts>
716 return (std::get<Ts::index>(holders).builder->Reserve(s).ok() && ...);
719 template <
typename... ARGS>
722 reserveArrays(*(HoldersTupleIndexed<ARGS...>*)mHolders, s);
727 std::shared_ptr<arrow::Table> finalize();
730 bool (*mFinalizer)(std::vector<std::shared_ptr<arrow::Array>>&
arrays,
void* holders);
731 void (*mDestructor)(
void* holders);
733 arrow::MemoryPool* mMemoryPool;
734 std::shared_ptr<arrow::Schema> mSchema;
735 std::vector<std::shared_ptr<arrow::Array>> mArrays;
742 [[maybe_unused]]
auto writer =
b.
cursor<T>();
747template <soa::TableRef R>
752 b.setLabel(aod::label<R>());
756template <
typename... Cs>
760 [[maybe_unused]]
auto writer =
b.
cursor(p);
765std::shared_ptr<arrow::Table>
spawnerHelper(std::shared_ptr<arrow::Table>
const& fullTable, std::shared_ptr<arrow::Schema> newSchema,
size_t nColumns,
769template <aod::is_aod_hash D>
774 if (fullTable->num_rows() == 0) {
777 return spawnerHelper(fullTable, schema, framework::pack_size(placeholders_pack_t{}), projectors,
name, projector);
780template <aod::is_aod_hash D>
785 return spawner<D>(fullTable,
name, projectors, projector, schema);
788template <aod::is_aod_hash D>
790auto spawner(std::shared_ptr<arrow::Table>
const& fullTable,
const char*
name,
expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema>
const& schema)
793 if (fullTable->num_rows() == 0) {
796 return spawnerHelper(fullTable, schema, framework::pack_size(expression_pack_t{}), projectors,
name, projector);
799template <aod::is_aod_hash D>
801auto spawner(std::vector<std::shared_ptr<arrow::Table>>&& tables,
const char*
name,
expressions::Projector* projectors, std::shared_ptr<gandiva::Projector>& projector, std::shared_ptr<arrow::Schema>
const& schema)
804 return spawner<D>(fullTable,
name, projectors, projector, schema);
807template <
typename... C>
810 std::array<const char*, 1> labels{
"original"};
811 auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables), std::span<const char* const>{labels});
812 if (fullTable->num_rows() == 0) {
815 return spawnerHelper(fullTable, schema,
sizeof...(C), projectors,
name, projector);
818template <
typename... T>
constexpr int p1()
constexpr to accelerate the coordinates changing
#define O2_ARROW_STL_CONVERSION(c_type, ArrowType_)
auto cursor(framework::pack< Cs... >)
auto persist(std::array< char const *, countColumns< ARG0, ARGS... >()> const &columnNames)
auto persistTuple(framework::pack< ARGS... >, std::array< char const *, sizeof...(ARGS)> const &columnNames)
Same a the above, but use a tuple to persist stuff.
static constexpr int countColumns()
void extracted(bool &status)
Actually creates the arrow::Table from the builders.
static constexpr int countColumns()
auto reserve(o2::framework::pack< ARGS... > &&, int s)
TableBuilder(arrow::MemoryPool *pool=arrow::default_memory_pool())
auto reserveArrays(std::tuple< Ts... > &holders, int s)
Reserve method to expand the columns as needed.
auto persist(std::array< char const *, sizeof...(ARGS)+1 > const &columnNames)
GLuint const GLchar * name
GLboolean GLboolean GLboolean b
GLsizei const GLfloat * value
GLint GLint GLsizei GLint GLenum GLenum type
GLenum GLsizei GLsizei GLint * values
GLuint GLsizei const GLchar * label
typedef void(APIENTRYP PFNGLCULLFACEPROC)(GLenum mode)
Defining PrimaryVertex explicitly as messageable.
void addLabelToSchema(std::shared_ptr< arrow::Schema > &schema, const char *label)
RuntimeErrorRef runtime_error(const char *)
constexpr auto makeHolderTypes()
auto constexpr to_tuple(T &&object) noexcept
auto makeEmptyTable(const char *name)
auto spawner(std::shared_ptr< arrow::Table > const &fullTable, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector, std::shared_ptr< arrow::Schema > const &schema)
Expression-based column generator to materialize columns.
constexpr auto tuple_to_pack(std::tuple< ARGS... > &&)
auto makeHolders(arrow::MemoryPool *pool, size_t nRows)
std::shared_ptr< arrow::Table > spawnerHelper(std::shared_ptr< arrow::Table > const &fullTable, std::shared_ptr< arrow::Schema > newSchema, size_t nColumns, expressions::Projector *projectors, const char *name, std::shared_ptr< gandiva::Projector > &projector)
decltype(makeHolderTypes< ARGS... >()) IndexedHoldersTuple
RuntimeErrorRef runtime_error_f(const char *,...)
std::tuple< typename T::iterator... > iterator_tuple_t
Defining DataPointCompositeObject explicitly as copiable.
typename detail::ConversionTraits< T >::ArrowType ArrowType
std::unique_ptr< BuilderType > builder
typename arrow::TypeTraits< ArrowType >::BuilderType BuilderType
typename InsertionTrait< T >::Policy Policy
BuilderHolder(arrow::MemoryPool *pool, size_t nRows=0)
typename detail::ConversionTraits< T >::ArrowType ElementType
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
arrow::FixedSizeListBuilder BuilderType
arrow::FixedSizeListType ArrowType
static std::shared_ptr< arrow::DataType > make_datatype()
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
typename detail::ConversionTraits< T >::ArrowType ElementType
arrow::FixedSizeListType ArrowType
static std::shared_ptr< arrow::DataType > make_datatype()
arrow::FixedSizeListBuilder BuilderType
typename arrow::TypeTraits< ArrowType >::BuilderType BuilderType
static arrow::Status append(BuilderType &builder, bool value)
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
static std::shared_ptr< arrow::DataType > make_datatype()
typename detail::ConversionTraits< bool >::ArrowType ArrowType
typename detail::ConversionTraits< T >::ArrowType ElementType
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
arrow::FixedSizeListType ArrowType
arrow::FixedSizeListBuilder BuilderType
static std::shared_ptr< arrow::DataType > make_datatype()
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
std::pair< ITERATOR, ITERATOR > FillType
arrow::ListType ArrowType
arrow::ListBuilder BuilderType
typename detail::ConversionTraits< typename ITERATOR::value_type >::ArrowType ValueType
typename ITERATOR::value_type STLValueType
typename arrow::TypeTraits< ValueType >::BuilderType ValueBuilder
static std::shared_ptr< arrow::DataType > make_datatype()
arrow::ListType ArrowType
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
typename detail::ConversionTraits< T >::ArrowType ElementType
arrow::ListBuilder BuilderType
std::vector< T > FillType
static std::shared_ptr< arrow::DataType > make_datatype()
typename arrow::TypeTraits< ArrowType >::BuilderType BuilderType
static arrow::Status append(BuilderType &builder, std::array< T, N > &value)
static arrow::Status append(BuilderType &builder, T value)
typename detail::ConversionTraits< T >::ArrowType ArrowType
static std::unique_ptr< BuilderType > make(arrow::MemoryPool *pool)
static std::shared_ptr< arrow::DataType > make_datatype()
arrow::FixedSizeListBuilder BuilderType
arrow::FixedSizeListType ArrowType
arrow::ListType ArrowType
arrow::ListBuilder BuilderType
arrow::ListType ArrowType
arrow::ListBuilder BuilderType
typename arrow::TypeTraits< ArrowType >::BuilderType BuilderType
typename detail::ConversionTraits< T >::ArrowType ArrowType
static arrow::Status append(HolderType &holder, std::vector< T > const &data)
Appender for the vector case.
static arrow::Status appendToList(std::unique_ptr< arrow::FixedSizeListBuilder > &builder, T *data, int size=1)
static arrow::Status flush(HolderType &holder)
static arrow::Status append(HolderType &holder, T value)
static arrow::Status append(HolderType &holder, T(&data)[N])
Appender for the array case.
static arrow::Status append(HolderType &holder, T *data)
static void unsafeAppend(HolderType &holder, std::vector< T > const &value)
static arrow::Status append(HolderType &holder, std::pair< ITERATOR, ITERATOR > ip)
static void unsafeAppend(HolderType &holder, T *value)
static void unsafeAppend(HolderType &holder, std::pair< ITERATOR, ITERATOR > ip)
static arrow::Status append(HolderType &holder, std::array< T, N > const &data)
Appender for the array case.
static void unsafeAppend(HolderType &holder, T value)
arrow::Status append(BUILDER &builder, T value)
arrow::Status flush(BUILDER &builder)
arrow::Status append(BUILDER &builder, T value)
arrow::Status flush(BUILDER &)
static consteval DirectInsertion< T > policy()
decltype(policy()) Policy
static bool finalize(std::shared_ptr< arrow::Array > &array, HOLDER &holder)
static bool append(std::tuple< Ts... > &holders, VALUES &&values)
Invokes the append method for each entry in the tuple.
static std::array< arrow::DataType, NCOLUMNS > makeArrowColumnTypes()
static bool finalize(std::vector< std::shared_ptr< arrow::Array > > &arrays, std::tuple< Ts... > &holders)
Invokes the append method for each entry in the tuple.
static void unsafeAppend(std::tuple< Ts... > &holders, VALUES &&values)
::arrow::FixedSizeListType ArrowType
::arrow::FixedSizeListType ArrowType
::arrow::FixedSizeListType ArrowType
::arrow::ListType ArrowType
FIXME: adapt type conversion to new arrow.
A struct, containing the root of the expression tree.