11#ifndef o2_framework_AnalysisHelpers_H_DEFINED
12#define o2_framework_AnalysisHelpers_H_DEFINED
34 std::string{
"input:"} + o2::aod::label<R>(),
42template <soa::with_sources T>
43inline constexpr auto getSources()
45 return []<
size_t N, std::array<soa::TableRef, N> refs>() {
46 return []<
size_t... Is>(std::index_sequence<Is...>) {
47 return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
48 }(std::make_index_sequence<N>());
49 }.template operator()<T::sources.size(), T::sources>();
52template <soa::with_sources T>
53constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
55 std::vector<framework::ConfigParamSpec> inputMetadata;
56 auto inputSources = getSources<T>();
57 std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec
const&
a, framework::ConfigParamSpec
const&
b) { return a.name < b.name; });
58 auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec
const&
a, framework::ConfigParamSpec
const&
b) { return a.name == b.name; });
59 inputSources.erase(last, inputSources.end());
60 inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());
65 requires(!soa::with_sources<T>)
66constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
78 o2::aod::description(o2::aod::signature<R>()),
80 framework::Lifetime::Timeframe,
90 o2::aod::description(o2::aod::signature<R>()),
99 o2::aod::description(o2::aod::signature<R>()),
125template <is_producable T>
131 template <
typename...
Ts>
133 requires(
sizeof...(Ts) ==
framework::pack_size(
typename persistent_table_t::persistent_columns_t{}))
136 cursor(0, extract(args)...);
147 mBuilder = std::move(builder);
155 mBuilder->setLabel(
label);
162 mBuilder->reserve(
typename persistent_table_t::column_types{},
size);
175 return arg.globalIndex();
178 template <
typename A>
180 static decltype(
auto) extract(
A&& arg)
193template <soa::is_table T>
198template <soa::is_iterator T>
199consteval auto typeWithRef() ->
typename T::parent_t
204 requires soa::is_table<T> || soa::is_iterator<T>
211 return OutputSpec{
OutputLabel{aod::label<table_t::ref>()}, o2::aod::origin<table_t::ref>(), o2::aod::description(o2::aod::signature<table_t::ref>()), table_t::ref.version};
216 return OutputRef{aod::label<table_t::ref>(), table_t::ref.version};
224template <is_producable T>
229concept is_produces =
requires(T t) {
typename T::cursor_t;
typename T::persistent_table_t; &T::cursor; };
246template <soa::is_metadata M, soa::TableRef Ref>
251 template <soa::TableRef R>
254 return soa::tableRef2InputSpec<R>();
259 return []<
size_t... Is>(std::index_sequence<Is...>) -> std::vector<InputSpec> {
260 return {base_spec<sources[Is]>()...};
261 }(std::make_index_sequence<
sources.size()>{});
266 return soa::tableRef2OutputSpec<Ref>();
271 return soa::tableRef2Output<Ref>();
274 constexpr auto ref()
const
276 return soa::tableRef2OutputRef<Ref>();
288template <is_spawnable T>
295template <is_spawnable T>
296struct Spawns : decltype(transformBase<T>()) {
322 std::shared_ptr<typename T::table_t>
table =
nullptr;
326 return {{std::move(C::Projector())...}};
329 std::shared_ptr<gandiva::Projector>
projector =
nullptr;
335 typename T::metadata;
336 requires std::same_as<
decltype(t.pack()),
typename T::expression_pack_t>;
337 requires std::same_as<
decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
345template <is_dynamically_spawnable T,
bool DELAYED = false>
346struct Defines : decltype(transformBase<T>()) {
373 std::shared_ptr<typename T::table_t>
table =
nullptr;
376 std::array<o2::framework::expressions::Projector, N>
projectors;
377 std::shared_ptr<gandiva::Projector>
projector =
nullptr;
389template <is_dynamically_spawnable T>
394 typename T::metadata;
395 requires std::same_as<
decltype(t.pack()),
typename T::placeholders_pack_t>;
396 requires std::same_as<
decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
397 requires std::same_as<
decltype(t.needRecompilation),
bool>;
412template <
typename T,
typename Key>
413inline std::shared_ptr<arrow::ChunkedArray> getIndexToKey(arrow::Table* table)
416 return table->column(framework::has_type_at_v<IC>(
typename T::persistent_columns_t{}));
419template <soa::is_column C>
423 static consteval auto listSize()
425 if constexpr (std::same_as<typename C::type, std::vector<int>>) {
427 }
else if constexpr (std::same_as<int[2], typename C::type>) {
434 template <
typename T,
typename Key>
435 static std::shared_ptr<SelfIndexColumnBuilder> makeColumnBuilder(arrow::Table* table, arrow::MemoryPool* pool)
437 if constexpr (!std::same_as<T, Key>) {
438 return std::make_shared<IndexColumnBuilder>(getIndexToKey<T, Key>(table), C::columnLabel(), listSize(), pool);
440 return std::make_shared<SelfIndexColumnBuilder>(C::columnLabel(), pool);
445template <
typename Key,
typename C>
447 using type =
typename std::conditional<soa::is_binding_compatible_v<Key, typename C::binding_t>(), SelfIndexColumnBuilder, IndexColumnBuilder>
::type;
450template <
typename Key,
typename C>
451using reduced_t = Reduction<Key, C>::type;
454template <
typename Kind>
456 template <
typename Key,
size_t N, std::array<soa::TableRef, N> refs,
typename C1,
typename... Cs>
459 auto pool = arrow::default_memory_pool();
461 std::unique_ptr<ChunkedArrayIterator> keyIndex =
nullptr;
462 if constexpr (!Key::template hasOriginal<refs[0]>()) {
463 keyIndex = std::make_unique<ChunkedArrayIterator>(tables[0]->column(
o2::aod::MetadataTrait<
o2::aod::Hash<refs[0].desc_hash>>::metadata::template getIndexPosToKey<Key>()));
466 auto sq = std::make_index_sequence<
sizeof...(Cs)>();
468 auto columnBuilders = [&tables, &pool ]<
size_t... Is>(std::index_sequence<Is...>) -> std::array<std::shared_ptr<framework::SelfIndexColumnBuilder>,
sizeof...(Cs)>
470 return {[](arrow::Table* table, arrow::MemoryPool* pool) {
472 if constexpr (!Key::template hasOriginal<refs[Is + 1]>()) {
474 return std::make_shared<IndexColumnBuilder>(table->column(
pos), T::columnLabel(), ColumnTrait<T>::listSize(), pool);
476 return std::make_shared<SelfIndexColumnBuilder>(T::columnLabel(), pool);
478 }(tables[Is + 1].get(), pool)...};
482 std::array<bool,
sizeof...(Cs)> finds;
486 if constexpr (Key::template hasOriginal<refs[0]>()) {
489 idx = keyIndex->valueAt(
counter);
491 finds = [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
493 [&idx, &columnBuilders]() {
495 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template find<T>(idx);
498 if constexpr (std::same_as<Kind, Sparse>) {
499 [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
500 ([&idx, &columnBuilders]() {
502 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template fill<T>(idx); }(), ...);
505 }
else if constexpr (std::same_as<Kind, Exclusive>) {
506 if (std::none_of(finds.begin(), finds.end(), [](
bool const x) {
return x ==
false; })) {
507 [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
508 ([&idx, &columnBuilders]() {
510 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template fill<T>(idx);
519 return [&
label, &columnBuilders, &self]<
size_t... Is>(std::index_sequence<Is...>) {
521 {self.template
result<C1>(), [&columnBuilders]() {
523 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template result<T>();
525 {self.field(), [&columnBuilders]() {
527 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->field();
535template <soa::is_index_table T>
542template <soa::is_index_table T>
543struct Builds : decltype(transformBase<T>()) {
547 using Key = metadata::Key;
548 using H =
typename T::first_t;
549 using Ts =
typename T::rest_t;
563 return table->asArrowTable();
572 template <
typename Key,
typename... Cs>
575 this->table = std::make_shared<T>(IP::template indexBuilder<
Key, metadata::sources.
size(), metadata::sources>(o2::aod::label<T::ref>(), std::forward<std::vector<std::shared_ptr<arrow::Table>>>(tables),
framework::pack<Cs...>{}));
576 return (this->table !=
nullptr);
582 typename T::metadata;
584 requires std::same_as<
decltype(t.pack()),
typename T::index_pack_t>;
618 object = std::make_shared<T>(t);
619 object->SetName(
label.c_str());
624 object = std::make_shared<T>(t);
625 object->SetName(
label.c_str());
631 object->SetName(
label.c_str());
637 object->SetName(
label.c_str());
650 std::memset(desc.str,
'_', 16);
652 s << std::hex << lhash;
654 s << std::hex << reinterpret_cast<uint64_t>(
this);
655 std::memcpy(desc.str, s.str().c_str(), 12);
666 return *
object.get();
687 requires std::same_as<
decltype(t.operator->()),
typename T::obj_t*>;
688 requires std::same_as<
decltype(t.object), std::shared_ptr<typename T::obj_t>>;
699 decltype(
auto) operator->()
const
711 requires std::same_as<
decltype(t.service),
typename T::service_t*>;
717 return std::make_unique<
o2::soa::Filtered<std::decay_t<
decltype(table)>>>(std::vector{table}, std::forward<soa::SelectionVector>(selection));
722 return std::make_unique<
o2::soa::Filtered<std::decay_t<
decltype(table)>>>(std::vector{table.asArrowTable()}, std::forward<soa::SelectionVector>(selection));
740 void intializeCaches(std::set<uint32_t>
const& hashes, std::shared_ptr<arrow::Schema>
const& schema)
754 template <
typename...
Ts>
758 mFiltered->bindExternalIndices(tables...);
762 template <
typename E>
785 template <
typename T1>
801 template <
typename T1,
typename Policy,
bool OPT>
808 std::unique_ptr<o2::soa::Filtered<T>>
mFiltered =
nullptr;
809 gandiva::NodePtr
tree =
nullptr;
842 &T::updatePlaceholders;
843 requires std::same_as<
decltype(t.filter), expressions::Filter>;
856 static std::shared_ptr<gandiva::Projector> projector =
nullptr;
867 return output_t{{table.asArrowTable()}, table.offset()};
constexpr uint32_t runtime_hash(char const *str)
T::template iterator_template_o< FilteredIndexPolicy, self_t > iterator
Helper to check if a type T is an iterator.
GLboolean GLboolean GLboolean b
GLsizei const GLfloat * value
GLint GLint GLsizei GLint GLenum GLenum type
GLuint GLsizei const GLchar * label
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
GLboolean GLboolean GLboolean GLboolean a
std::shared_ptr< gandiva::Filter > FilterPtr
std::variant< OriginValueMatcher, DescriptionValueMatcher, SubSpecificationTypeValueMatcher, std::unique_ptr< DataDescriptorMatcher >, ConstantValueMatcher, StartTimeValueMatcher > Node
std::shared_ptr< gandiva::Projector > createProjectorHelper(size_t nColumns, expressions::Projector *projectors, std::shared_ptr< arrow::Schema > schema, std::vector< std::shared_ptr< arrow::Field > > const &fields)
gandiva::Selection createSelection(std::shared_ptr< arrow::Table > const &table, Filter const &expression)
Function for creating gandiva selection from our internal filter tree.
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
Defining PrimaryVertex explicitly as messageable.
auto spawner(std::shared_ptr< arrow::Table > const &fullTable, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector, std::shared_ptr< arrow::Schema > const &schema)
Expression-based column generator to materialize columns.
typename pack_element< I, T >::type pack_element_t
constexpr std::size_t pack_size(pack< Ts... > const &)
template function to determine number of types in a pack
constexpr auto transformBase()
This helper struct allows you to declare index tables to be created in a task.
memfun_type< decltype(&F::operator())>::type FFL(F const &func)
auto getTableFromFilter(soa::is_filtered_table auto const &table, soa::SelectionVector &&selection)
consteval auto typeWithRef() -> T
Helper to define output for a Table.
OutputObjHandlingPolicy
Policy enum to determine OutputObj handling when writing.
void initializePartitionCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema, expressions::Filter const &filter, gandiva::NodePtr &tree, gandiva::FilterPtr &gfilter)
std::shared_ptr< arrow::Table > makeArrowTable(const char *label, std::vector< std::shared_ptr< arrow::ChunkedArray > > &&columns, std::vector< std::shared_ptr< arrow::Field > > &&fields)
auto createFieldsFromColumns(framework::pack< C... >)
SelectionVector selectionToVector(gandiva::Selection const &sel)
constexpr auto tableRef2InputSpec()
constexpr auto tableRef2Output()
std::vector< int64_t > SelectionVector
auto Attach(T const &table)
constexpr auto tableRef2ConfigParamSpec()
auto Extend(T const &table)
On-the-fly adding of expression columns.
constexpr auto tableRef2OutputSpec()
constexpr auto tableRef2OutputRef()
Defining DataPointCompositeObject explicitly as copiable.
std::shared_ptr< T > table
T const & operator*() const
auto build(framework::pack< Cs... >, std::vector< std::shared_ptr< arrow::Table > > &&tables)
metadata::index_pack_t index_pack_t
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
T::table_t const & operator*() const
decltype(transformBase< T >())::metadata metadata
typename metadata::extension_table_t extension_t
std::array< o2::framework::expressions::Projector, N > projectors
std::shared_ptr< extension_t > extension
std::shared_ptr< arrow::Schema > inputSchema
std::shared_ptr< arrow::Schema > schema
typename metadata::base_table_t base_table_t
static constexpr bool delayed
static constexpr size_t N
typename metadata::placeholders_pack_t placeholders_pack_t
T::table_t * operator->()
std::shared_ptr< gandiva::Projector > projector
static auto indexBuilder(const char *label, std::vector< std::shared_ptr< arrow::Table > > &&tables, framework::pack< C1, Cs... >)
static OutputSpec const spec()
aod::MetadataTrait< o2::aod::Hash< table_t::ref.desc_hash > >::metadata metadata
decltype(typeWithRef< T >()) table_t
OutputObj(T &&t, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
void setObject(std::shared_ptr< T > t)
OutputObjSourceType sourceType
OutputRef ref(uint16_t index, uint16_t max)
std::shared_ptr< T > object
void setObject(T const &t)
OutputObj(std::string const &label_, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
OutputObjHandlingPolicy policy
void setHash(uint32_t hash)
Partition(expressions::Node &&filter_, T const &table)
typename o2::soa::Filtered< T >::const_iterator filtered_const_iterator
void bindTable(T const &table)
filtered_const_iterator begin() const
auto rawSliceBy(o2::framework::Preslice< T1 > const &container, int value) const
auto sliceByCached(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
void intializeCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema)
Partition(expressions::Node &&filter_)
typename o2::soa::Filtered< T >::iterator filtered_iterator
o2::soa::Filtered< T > * operator->()
typename o2::soa::Filtered< T >::iterator iterator
auto sliceBy(o2::framework::PresliceBase< T1, Policy, OPT > const &container, int value) const
std::shared_ptr< arrow::Table > asArrowTable() const
o2::soa::RowViewSentinel end() const
void updatePlaceholders(InitContext &context)
std::unique_ptr< o2::soa::Filtered< T > > mFiltered
void bindInternalIndicesTo(E *ptr)
expressions::Filter filter
o2::soa::RowViewSentinel end()
auto sliceByCachedUnsorted(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
typename o2::soa::Filtered< T >::const_iterator const_iterator
gandiva::FilterPtr gfilter
void bindExternalIndices(Ts *... tables)
filtered_iterator begin()
T::table_t * operator->()
std::shared_ptr< gandiva::Projector > projector
std::shared_ptr< extension_t > extension
T::table_t const & operator*() const
static constexpr size_t N
typename metadata::base_table_t base_table_t
std::shared_ptr< arrow::Schema > schema
typename metadata::expression_pack_t expression_pack_t
std::array< o2::framework::expressions::Projector, N > projectors
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
typename metadata::extension_table_t extension_t
int64_t lastIndex()
Last index inserted in the table.
decltype(FFL(std::declval< cursor_t >())) cursor
bool resetCursor(LifetimeHolder< TableBuilder > builder)
void setLabel(const char *label)
void operator()(Ts &&... args)
void reserve(int64_t size)
decltype([]() { if constexpr(soa::is_iterator< T >) { return typename T::parent_t{nullptr} persistent_table_t
decltype(std::declval< TableBuilder >().cursor< persistent_table_t >()) cursor_t
An expression tree node corresponding to a column binding.
A struct, containing the root of the expression tree.
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))