11#ifndef o2_framework_AnalysisHelpers_H_DEFINED
12#define o2_framework_AnalysisHelpers_H_DEFINED
34 std::string{
"input:"} + o2::aod::label<R>(),
42template <soa::with_sources T>
43inline constexpr auto getSources()
45 return []<
size_t N, std::array<soa::TableRef, N> refs>() {
46 return []<
size_t... Is>(std::index_sequence<Is...>) {
47 return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
48 }(std::make_index_sequence<N>());
49 }.template operator()<T::sources.size(), T::sources>();
52template <soa::with_sources T>
53constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
55 std::vector<framework::ConfigParamSpec> inputMetadata;
56 auto inputSources = getSources<T>();
57 std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec
const&
a, framework::ConfigParamSpec
const&
b) { return a.name < b.name; });
58 auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec
const&
a, framework::ConfigParamSpec
const&
b) { return a.name == b.name; });
59 inputSources.erase(last, inputSources.end());
60 inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());
65 requires(!soa::with_sources<T>)
66constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
78 o2::aod::description(o2::aod::signature<R>()),
80 framework::Lifetime::Timeframe,
90 o2::aod::description(o2::aod::signature<R>()),
99 o2::aod::description(o2::aod::signature<R>()),
122template <is_producable T>
128 template <
typename...
Ts>
131 static_assert(
sizeof...(Ts) ==
framework::pack_size(
typename persistent_table_t::persistent_columns_t{}),
"Argument number mismatch");
133 cursor(0, extract(args)...);
144 mBuilder = std::move(builder);
152 mBuilder->setLabel(
label);
159 mBuilder->reserve(
typename persistent_table_t::column_types{},
size);
170 template <
typename A>
171 requires requires { &A::globalIndex; }
172 static decltype(
auto) extract(
A const& arg)
174 return arg.globalIndex();
177 template <
typename A>
178 static decltype(
auto) extract(
A const& arg)
186 LifetimeHolder<TableBuilder> mBuilder =
nullptr;
191template <soa::is_table T>
196template <soa::is_iterator T>
197consteval auto typeWithRef() ->
typename T::parent_t
202 requires soa::is_table<T> || soa::is_iterator<T>
209 return OutputSpec{
OutputLabel{aod::label<table_t::ref>()}, o2::aod::origin<table_t::ref>(), o2::aod::description(o2::aod::signature<table_t::ref>()), table_t::ref.version};
214 return OutputRef{aod::label<table_t::ref>(), table_t::ref.version};
222template <is_producable T>
227concept is_produces =
requires(T t) {
typename T::cursor_t;
typename T::persistent_table_t; &T::cursor; };
244template <soa::is_metadata M, soa::TableRef Ref>
249 template <soa::TableRef R>
252 return soa::tableRef2InputSpec<R>();
257 return []<
size_t... Is>(std::index_sequence<Is...>) -> std::vector<InputSpec> {
258 return {base_spec<sources[Is]>()...};
259 }(std::make_index_sequence<
sources.size()>{});
264 return soa::tableRef2OutputSpec<Ref>();
269 return soa::tableRef2Output<Ref>();
272 constexpr auto ref()
const
274 return soa::tableRef2OutputRef<Ref>();
286template <is_spawnable T>
293template <is_spawnable T>
294struct Spawns : decltype(transformBase<T>()) {
319 std::shared_ptr<typename T::table_t>
table =
nullptr;
321 std::shared_ptr<gandiva::Projector>
projector =
nullptr;
326 typename T::metadata;
327 requires std::same_as<
decltype(t.pack()),
typename T::expression_pack_t>;
328 requires std::same_as<
decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
336template <is_dynamically_spawnable T>
337struct Defines : decltype(transformBase<T>()) {
363 std::shared_ptr<typename T::table_t>
table =
nullptr;
366 std::array<o2::framework::expressions::Projector, N>
projectors;
367 std::shared_ptr<gandiva::Projector>
projector =
nullptr;
372 typename T::metadata;
373 requires std::same_as<
decltype(t.pack()),
typename T::placeholders_pack_t>;
374 requires std::same_as<
decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
388template <
typename T,
typename Key>
389inline std::shared_ptr<arrow::ChunkedArray> getIndexToKey(arrow::Table* table)
392 return table->column(framework::has_type_at_v<IC>(
typename T::persistent_columns_t{}));
395template <soa::is_column C>
399 static consteval auto listSize()
401 if constexpr (std::same_as<typename C::type, std::vector<int>>) {
403 }
else if constexpr (std::same_as<int[2], typename C::type>) {
410 template <
typename T,
typename Key>
411 static std::shared_ptr<SelfIndexColumnBuilder> makeColumnBuilder(arrow::Table* table, arrow::MemoryPool* pool)
413 if constexpr (!std::same_as<T, Key>) {
414 return std::make_shared<IndexColumnBuilder>(getIndexToKey<T, Key>(table), C::columnLabel(), listSize(), pool);
416 return std::make_shared<SelfIndexColumnBuilder>(C::columnLabel(), pool);
421template <
typename Key,
typename C>
423 using type =
typename std::conditional<soa::is_binding_compatible_v<Key, typename C::binding_t>(), SelfIndexColumnBuilder, IndexColumnBuilder>
::type;
426template <
typename Key,
typename C>
427using reduced_t = Reduction<Key, C>::type;
430template <
typename Kind>
432 template <
typename Key,
size_t N, std::array<soa::TableRef, N> refs,
typename C1,
typename... Cs>
435 auto pool = arrow::default_memory_pool();
437 std::unique_ptr<ChunkedArrayIterator> keyIndex =
nullptr;
438 if constexpr (!Key::template hasOriginal<refs[0]>()) {
439 keyIndex = std::make_unique<ChunkedArrayIterator>(tables[0]->column(
o2::aod::MetadataTrait<
o2::aod::Hash<refs[0].desc_hash>>::metadata::template getIndexPosToKey<Key>()));
442 auto sq = std::make_index_sequence<
sizeof...(Cs)>();
444 auto columnBuilders = [&tables, &pool ]<
size_t... Is>(std::index_sequence<Is...>) -> std::array<std::shared_ptr<framework::SelfIndexColumnBuilder>,
sizeof...(Cs)>
446 return {[](arrow::Table* table, arrow::MemoryPool* pool) {
448 if constexpr (!Key::template hasOriginal<refs[Is + 1]>()) {
450 return std::make_shared<IndexColumnBuilder>(table->column(
pos), T::columnLabel(), ColumnTrait<T>::listSize(), pool);
452 return std::make_shared<SelfIndexColumnBuilder>(T::columnLabel(), pool);
454 }(tables[Is + 1].get(), pool)...};
458 std::array<bool,
sizeof...(Cs)> finds;
462 if constexpr (Key::template hasOriginal<refs[0]>()) {
465 idx = keyIndex->valueAt(
counter);
467 finds = [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
469 [&idx, &columnBuilders]() {
471 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template find<T>(idx);
474 if constexpr (std::same_as<Kind, Sparse>) {
475 [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
476 ([&idx, &columnBuilders]() {
478 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template fill<T>(idx); }(), ...);
481 }
else if constexpr (std::same_as<Kind, Exclusive>) {
482 if (std::none_of(finds.begin(), finds.end(), [](
bool const x) {
return x ==
false; })) {
483 [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
484 ([&idx, &columnBuilders]() {
486 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template fill<T>(idx);
495 return [&
label, &columnBuilders, &self]<
size_t... Is>(std::index_sequence<Is...>) {
497 {self.template
result<C1>(), [&columnBuilders]() {
499 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template result<T>();
501 {self.field(), [&columnBuilders]() {
503 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->field();
511template <soa::is_index_table T>
518template <soa::is_index_table T>
519struct Builds : decltype(transformBase<T>()) {
523 using Key = metadata::Key;
524 using H =
typename T::first_t;
525 using Ts =
typename T::rest_t;
539 return table->asArrowTable();
548 template <
typename Key,
typename... Cs>
551 this->table = std::make_shared<T>(IP::template indexBuilder<
Key, metadata::sources.
size(), metadata::sources>(o2::aod::label<T::ref>(), std::forward<std::vector<std::shared_ptr<arrow::Table>>>(tables),
framework::pack<Cs...>{}));
552 return (this->table !=
nullptr);
558 typename T::metadata;
560 requires std::same_as<
decltype(t.pack()),
typename T::index_pack_t>;
594 object = std::make_shared<T>(t);
595 object->SetName(
label.c_str());
600 object = std::make_shared<T>(t);
601 object->SetName(
label.c_str());
607 object->SetName(
label.c_str());
613 object->SetName(
label.c_str());
626 std::memset(desc.str,
'_', 16);
628 s << std::hex << lhash;
630 s << std::hex << reinterpret_cast<uint64_t>(
this);
631 std::memcpy(desc.str, s.str().c_str(), 12);
642 return *
object.get();
663 requires std::same_as<
decltype(t.operator->()),
typename T::obj_t*>;
664 requires std::same_as<
decltype(t.object), std::shared_ptr<typename T::obj_t>>;
675 decltype(
auto) operator->()
const
687 requires std::same_as<
decltype(t.service),
typename T::service_t*>;
693 return std::make_unique<
o2::soa::Filtered<std::decay_t<
decltype(table)>>>(std::vector{table}, std::forward<soa::SelectionVector>(selection));
698 return std::make_unique<
o2::soa::Filtered<std::decay_t<
decltype(table)>>>(std::vector{table.asArrowTable()}, std::forward<soa::SelectionVector>(selection));
716 void intializeCaches(std::set<uint32_t>
const& hashes, std::shared_ptr<arrow::Schema>
const& schema)
730 template <
typename...
Ts>
734 mFiltered->bindExternalIndices(tables...);
738 template <
typename E>
761 template <
typename T1>
777 template <
typename T1,
typename Policy,
bool OPT>
784 std::unique_ptr<o2::soa::Filtered<T>>
mFiltered =
nullptr;
785 gandiva::NodePtr
tree =
nullptr;
818 &T::updatePlaceholders;
819 requires std::same_as<
decltype(t.filter), expressions::Filter>;
831 static std::shared_ptr<gandiva::Projector> projector =
nullptr;
841 return output_t{{table.asArrowTable()}, table.offset()};
constexpr uint32_t runtime_hash(char const *str)
T::template iterator_template_o< FilteredIndexPolicy, self_t > iterator
Helper to check if a type T is an iterator.
GLboolean GLboolean GLboolean b
GLsizei const GLfloat * value
GLint GLint GLsizei GLint GLenum GLenum type
GLuint GLsizei const GLchar * label
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
GLboolean GLboolean GLboolean GLboolean a
std::shared_ptr< gandiva::Filter > FilterPtr
std::variant< OriginValueMatcher, DescriptionValueMatcher, SubSpecificationTypeValueMatcher, std::unique_ptr< DataDescriptorMatcher >, ConstantValueMatcher, StartTimeValueMatcher > Node
gandiva::Selection createSelection(std::shared_ptr< arrow::Table > const &table, Filter const &expression)
Function for creating gandiva selection from our internal filter tree.
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
Defining PrimaryVertex explicitly as messageable.
auto spawner(std::vector< std::shared_ptr< arrow::Table > > &&tables, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector)
Expression-based column generator to materialize columns.
typename pack_element< I, T >::type pack_element_t
constexpr std::size_t pack_size(pack< Ts... > const &)
template function to determine number of types in a pack
constexpr auto transformBase()
This helper struct allows you to declare index tables to be created in a task.
memfun_type< decltype(&F::operator())>::type FFL(F const &func)
auto getTableFromFilter(soa::is_filtered_table auto const &table, soa::SelectionVector &&selection)
consteval auto typeWithRef() -> T
Helper to define output for a Table.
OutputObjHandlingPolicy
Policy enum to determine OutputObj handling when writing.
void initializePartitionCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema, expressions::Filter const &filter, gandiva::NodePtr &tree, gandiva::FilterPtr &gfilter)
std::shared_ptr< arrow::Table > makeArrowTable(const char *label, std::vector< std::shared_ptr< arrow::ChunkedArray > > &&columns, std::vector< std::shared_ptr< arrow::Field > > &&fields)
SelectionVector selectionToVector(gandiva::Selection const &sel)
constexpr auto tableRef2InputSpec()
constexpr auto tableRef2Output()
std::vector< int64_t > SelectionVector
auto Attach(T const &table)
constexpr auto tableRef2ConfigParamSpec()
auto Extend(T const &table)
On-the-fly adding of expression columns.
constexpr auto tableRef2OutputSpec()
constexpr auto tableRef2OutputRef()
Defining DataPointCompositeObject explicitly as copiable.
std::shared_ptr< T > table
T const & operator*() const
auto build(framework::pack< Cs... >, std::vector< std::shared_ptr< arrow::Table > > &&tables)
metadata::index_pack_t index_pack_t
decltype(transformBase< T >())::metadata metadata
std::array< o2::framework::expressions::Projector, N > projectors
std::shared_ptr< extension_t > extension
typename metadata::extension_table_t extension_t
T::table_t const & operator*() const
typename metadata::placeholders_pack_t placeholders_pack_t
static constexpr size_t N
std::shared_ptr< gandiva::Projector > projector
typename metadata::base_table_t base_table_t
T::table_t * operator->()
std::shared_ptr< typename T::table_t > table
decltype(transformBase< T >())::metadata metadata
static auto indexBuilder(const char *label, std::vector< std::shared_ptr< arrow::Table > > &&tables, framework::pack< C1, Cs... >)
static OutputSpec const spec()
aod::MetadataTrait< o2::aod::Hash< table_t::ref.desc_hash > >::metadata metadata
decltype(typeWithRef< T >()) table_t
OutputObj(T &&t, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
void setObject(std::shared_ptr< T > t)
OutputObjSourceType sourceType
OutputRef ref(uint16_t index, uint16_t max)
std::shared_ptr< T > object
void setObject(T const &t)
OutputObj(std::string const &label_, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
OutputObjHandlingPolicy policy
void setHash(uint32_t hash)
Partition(expressions::Node &&filter_, T const &table)
typename o2::soa::Filtered< T >::const_iterator filtered_const_iterator
void bindTable(T const &table)
filtered_const_iterator begin() const
auto rawSliceBy(o2::framework::Preslice< T1 > const &container, int value) const
auto sliceByCached(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
void intializeCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema)
Partition(expressions::Node &&filter_)
typename o2::soa::Filtered< T >::iterator filtered_iterator
o2::soa::Filtered< T > * operator->()
typename o2::soa::Filtered< T >::iterator iterator
auto sliceBy(o2::framework::PresliceBase< T1, Policy, OPT > const &container, int value) const
std::shared_ptr< arrow::Table > asArrowTable() const
o2::soa::RowViewSentinel end() const
void updatePlaceholders(InitContext &context)
std::unique_ptr< o2::soa::Filtered< T > > mFiltered
void bindInternalIndicesTo(E *ptr)
expressions::Filter filter
o2::soa::RowViewSentinel end()
auto sliceByCachedUnsorted(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
typename o2::soa::Filtered< T >::const_iterator const_iterator
gandiva::FilterPtr gfilter
void bindExternalIndices(Ts *... tables)
filtered_iterator begin()
T::table_t * operator->()
std::shared_ptr< gandiva::Projector > projector
std::shared_ptr< extension_t > extension
T::table_t const & operator*() const
typename metadata::base_table_t base_table_t
typename metadata::expression_pack_t expression_pack_t
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
typename metadata::extension_table_t extension_t
int64_t lastIndex()
Last index inserted in the table.
decltype(FFL(std::declval< cursor_t >())) cursor
bool resetCursor(LifetimeHolder< TableBuilder > builder)
void operator()(Ts... args)
void setLabel(const char *label)
void reserve(int64_t size)
decltype([]() { if constexpr(soa::is_iterator< T >) { return typename T::parent_t{nullptr} persistent_table_t
decltype(std::declval< TableBuilder >().cursor< persistent_table_t >()) cursor_t
An expression tree node corresponding to a column binding.
A struct, containing the root of the expression tree.
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))