11#ifndef o2_framework_AnalysisHelpers_H_DEFINED
12#define o2_framework_AnalysisHelpers_H_DEFINED
35 std::string{
"input:"} + o2::aod::label<R>(),
43template <soa::with_sources T>
44inline constexpr auto getSources()
46 return []<
size_t N, std::array<soa::TableRef, N> refs>() {
47 return []<
size_t... Is>(std::index_sequence<Is...>) {
48 return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
49 }(std::make_index_sequence<N>());
50 }.template operator()<T::sources.size(), T::sources>();
53template <soa::with_ccdb_urls T>
54inline constexpr auto getCCDBUrls()
56 std::vector<framework::ConfigParamSpec>
result;
57 for (
size_t i = 0;
i < T::ccdb_urls.size(); ++
i) {
58 result.push_back({std::string{
"ccdb:"} + std::string{T::ccdb_bindings[
i]},
66template <soa::with_sources T>
67constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
69 std::vector<framework::ConfigParamSpec> inputMetadata;
70 auto inputSources = getSources<T>();
71 std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec
const&
a, framework::ConfigParamSpec
const&
b) { return a.name < b.name; });
72 auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec
const&
a, framework::ConfigParamSpec
const&
b) { return a.name == b.name; });
73 inputSources.erase(last, inputSources.end());
74 inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());
79 requires(!soa::with_sources<T>)
80constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
85template <soa::with_ccdb_urls T>
86constexpr auto getCCDBMetadata() -> std::vector<framework::ConfigParamSpec>
88 std::vector<framework::ConfigParamSpec> results = getCCDBUrls<T>();
89 std::sort(results.begin(), results.end(), [](framework::ConfigParamSpec
const&
a, framework::ConfigParamSpec
const&
b) { return a.name < b.name; });
90 auto last = std::unique(results.begin(), results.end(), [](framework::ConfigParamSpec
const&
a, framework::ConfigParamSpec
const&
b) { return a.name == b.name; });
91 results.erase(last, results.end());
96constexpr auto getCCDBMetadata() -> std::vector<framework::ConfigParamSpec>
105 std::vector<framework::ConfigParamSpec> metadata;
107 metadata.insert(metadata.end(),
m.begin(),
m.end());
109 metadata.insert(metadata.end(), ccdbMetadata.begin(), ccdbMetadata.end());
113 o2::aod::origin<R>(),
114 o2::aod::description(o2::aod::signature<R>()),
116 framework::Lifetime::Timeframe,
125 o2::aod::origin<R>(),
126 o2::aod::description(o2::aod::signature<R>()),
134 o2::aod::origin<R>(),
135 o2::aod::description(o2::aod::signature<R>()),
161template <is_producable T>
167 template <
typename...
Ts>
169 requires(
sizeof...(Ts) ==
framework::pack_size(
typename persistent_table_t::persistent_columns_t{}))
172 cursor(0, extract(args)...);
183 mBuilder = std::move(builder);
191 mBuilder->setLabel(
label);
198 mBuilder->reserve(
typename persistent_table_t::column_types{},
size);
211 return arg.globalIndex();
214 template <
typename A>
216 static decltype(
auto) extract(
A&& arg)
229template <soa::is_table T>
234template <soa::is_iterator T>
235consteval auto typeWithRef() ->
typename T::parent_t
240 requires soa::is_table<T> || soa::is_iterator<T>
247 return OutputSpec{
OutputLabel{aod::label<table_t::ref>()}, o2::aod::origin<table_t::ref>(), o2::aod::description(o2::aod::signature<table_t::ref>()), table_t::ref.version};
252 return OutputRef{aod::label<table_t::ref>(), table_t::ref.version};
260template <is_producable T>
265concept is_produces =
requires(T t) {
typename T::cursor_t;
typename T::persistent_table_t; &T::cursor; };
282template <soa::is_metadata M, soa::TableRef Ref>
287 template <soa::TableRef R>
290 return soa::tableRef2InputSpec<R>();
295 return []<
size_t... Is>(std::index_sequence<Is...>) -> std::vector<InputSpec> {
296 return {base_spec<sources[Is]>()...};
297 }(std::make_index_sequence<
sources.size()>{});
302 return soa::tableRef2OutputSpec<Ref>();
307 return soa::tableRef2Output<Ref>();
310 constexpr auto ref()
const
312 return soa::tableRef2OutputRef<Ref>();
324template <is_spawnable T>
331template <is_spawnable T>
332struct Spawns : decltype(transformBase<T>()) {
358 std::shared_ptr<typename T::table_t>
table =
nullptr;
362 return {{std::move(C::Projector())...}};
365 std::shared_ptr<gandiva::Projector>
projector =
nullptr;
371 typename T::metadata;
372 requires std::same_as<
decltype(t.pack()),
typename T::expression_pack_t>;
373 requires std::same_as<
decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
381template <is_dynamically_spawnable T,
bool DELAYED = false>
382struct Defines : decltype(transformBase<T>()) {
409 std::shared_ptr<typename T::table_t>
table =
nullptr;
412 std::array<o2::framework::expressions::Projector, N>
projectors;
413 std::shared_ptr<gandiva::Projector>
projector =
nullptr;
425template <is_dynamically_spawnable T>
430 typename T::metadata;
431 requires std::same_as<
decltype(t.pack()),
typename T::placeholders_pack_t>;
432 requires std::same_as<
decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
433 requires std::same_as<
decltype(t.needRecompilation),
bool>;
448template <
typename T,
typename Key>
449inline std::shared_ptr<arrow::ChunkedArray> getIndexToKey(arrow::Table* table)
452 return table->column(framework::has_type_at_v<IC>(
typename T::persistent_columns_t{}));
455template <soa::is_column C>
459 static consteval auto listSize()
461 if constexpr (std::same_as<typename C::type, std::vector<int>>) {
463 }
else if constexpr (std::same_as<int[2], typename C::type>) {
470 template <
typename T,
typename Key>
471 static std::shared_ptr<SelfIndexColumnBuilder> makeColumnBuilder(arrow::Table* table, arrow::MemoryPool* pool)
473 if constexpr (!std::same_as<T, Key>) {
474 return std::make_shared<IndexColumnBuilder>(getIndexToKey<T, Key>(table), C::columnLabel(), listSize(), pool);
476 return std::make_shared<SelfIndexColumnBuilder>(C::columnLabel(), pool);
481template <
typename Key,
typename C>
483 using type =
typename std::conditional<soa::is_binding_compatible_v<Key, typename C::binding_t>(), SelfIndexColumnBuilder, IndexColumnBuilder>
::type;
486template <
typename Key,
typename C>
487using reduced_t = Reduction<Key, C>::type;
490template <
typename Kind>
492 template <
typename Key,
size_t N, std::array<soa::TableRef, N> refs,
typename C1,
typename... Cs>
495 auto pool = arrow::default_memory_pool();
497 std::unique_ptr<ChunkedArrayIterator> keyIndex =
nullptr;
498 if constexpr (!Key::template hasOriginal<refs[0]>()) {
499 keyIndex = std::make_unique<ChunkedArrayIterator>(tables[0]->column(
o2::aod::MetadataTrait<
o2::aod::Hash<refs[0].desc_hash>>::metadata::template getIndexPosToKey<Key>()));
502 auto sq = std::make_index_sequence<
sizeof...(Cs)>();
504 auto columnBuilders = [&tables, &pool ]<
size_t... Is>(std::index_sequence<Is...>) -> std::array<std::shared_ptr<framework::SelfIndexColumnBuilder>,
sizeof...(Cs)>
506 return {[](arrow::Table* table, arrow::MemoryPool* pool) {
508 if constexpr (!Key::template hasOriginal<refs[Is + 1]>()) {
510 return std::make_shared<IndexColumnBuilder>(table->column(
pos), T::columnLabel(), ColumnTrait<T>::listSize(), pool);
512 return std::make_shared<SelfIndexColumnBuilder>(T::columnLabel(), pool);
514 }(tables[Is + 1].get(), pool)...};
518 std::array<bool,
sizeof...(Cs)> finds;
522 if constexpr (Key::template hasOriginal<refs[0]>()) {
525 idx = keyIndex->valueAt(
counter);
527 finds = [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
529 [&idx, &columnBuilders]() {
531 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template find<T>(idx);
534 if constexpr (std::same_as<Kind, Sparse>) {
535 [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
536 ([&idx, &columnBuilders]() {
538 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template fill<T>(idx); }(), ...);
541 }
else if constexpr (std::same_as<Kind, Exclusive>) {
542 if (std::none_of(finds.begin(), finds.end(), [](
bool const x) {
return x ==
false; })) {
543 [&idx, &columnBuilders]<
size_t... Is>(std::index_sequence<Is...>) {
544 ([&idx, &columnBuilders]() {
546 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template fill<T>(idx);
555 return [&
label, &columnBuilders, &self]<
size_t... Is>(std::index_sequence<Is...>) {
557 {self.template
result<C1>(), [&columnBuilders]() {
559 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->
template result<T>();
561 {self.field(), [&columnBuilders]() {
563 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->field();
571template <soa::is_index_table T>
578template <soa::is_index_table T>
579struct Builds : decltype(transformBase<T>()) {
583 using Key = metadata::Key;
584 using H =
typename T::first_t;
585 using Ts =
typename T::rest_t;
599 return table->asArrowTable();
608 template <
typename Key,
typename... Cs>
611 this->table = std::make_shared<T>(IP::template indexBuilder<
Key, metadata::sources.
size(), metadata::sources>(o2::aod::label<T::ref>(), std::forward<std::vector<std::shared_ptr<arrow::Table>>>(tables),
framework::pack<Cs...>{}));
612 return (this->table !=
nullptr);
618 typename T::metadata;
620 requires std::same_as<
decltype(t.pack()),
typename T::index_pack_t>;
654 object = std::make_shared<T>(t);
655 object->SetName(
label.c_str());
660 object = std::make_shared<T>(t);
661 object->SetName(
label.c_str());
667 object->SetName(
label.c_str());
673 object->SetName(
label.c_str());
686 std::memset(desc.str,
'_', 16);
688 s << std::hex << lhash;
690 s << std::hex << reinterpret_cast<uint64_t>(
this);
691 std::memcpy(desc.str, s.str().c_str(), 12);
702 return *
object.get();
723 requires std::same_as<
decltype(t.operator->()),
typename T::obj_t*>;
724 requires std::same_as<
decltype(t.object), std::shared_ptr<typename T::obj_t>>;
735 decltype(
auto) operator->()
const
747 requires std::same_as<
decltype(t.service),
typename T::service_t*>;
753 return std::make_unique<
o2::soa::Filtered<std::decay_t<
decltype(table)>>>(std::vector{table}, std::forward<soa::SelectionVector>(selection));
758 return std::make_unique<
o2::soa::Filtered<std::decay_t<
decltype(table)>>>(std::vector{table.asArrowTable()}, std::forward<soa::SelectionVector>(selection));
776 void intializeCaches(std::set<uint32_t>
const& hashes, std::shared_ptr<arrow::Schema>
const& schema)
790 template <
typename...
Ts>
794 mFiltered->bindExternalIndices(tables...);
798 template <
typename E>
821 template <
typename T1>
837 template <
typename T1,
typename Policy,
bool OPT>
844 std::unique_ptr<o2::soa::Filtered<T>>
mFiltered =
nullptr;
845 gandiva::NodePtr
tree =
nullptr;
878 &T::updatePlaceholders;
879 requires std::same_as<
decltype(t.filter), expressions::Filter>;
892 static std::shared_ptr<gandiva::Projector> projector =
nullptr;
903 return output_t{{table.asArrowTable()}, table.offset()};
constexpr uint32_t runtime_hash(char const *str)
T::template iterator_template_o< FilteredIndexPolicy, self_t > iterator
Helper to check if a type T is an iterator.
GLboolean GLboolean GLboolean b
GLsizei const GLfloat * value
GLint GLint GLsizei GLint GLenum GLenum type
GLuint GLsizei const GLchar * label
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
GLboolean GLboolean GLboolean GLboolean a
std::shared_ptr< gandiva::Filter > FilterPtr
std::variant< OriginValueMatcher, DescriptionValueMatcher, SubSpecificationTypeValueMatcher, std::unique_ptr< DataDescriptorMatcher >, ConstantValueMatcher, StartTimeValueMatcher > Node
std::shared_ptr< gandiva::Projector > createProjectorHelper(size_t nColumns, expressions::Projector *projectors, std::shared_ptr< arrow::Schema > schema, std::vector< std::shared_ptr< arrow::Field > > const &fields)
gandiva::Selection createSelection(std::shared_ptr< arrow::Table > const &table, Filter const &expression)
Function for creating gandiva selection from our internal filter tree.
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
Defining PrimaryVertex explicitly as messageable.
auto spawner(std::shared_ptr< arrow::Table > const &fullTable, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector, std::shared_ptr< arrow::Schema > const &schema)
Expression-based column generator to materialize columns.
typename pack_element< I, T >::type pack_element_t
constexpr std::size_t pack_size(pack< Ts... > const &)
template function to determine number of types in a pack
constexpr auto transformBase()
This helper struct allows you to declare index tables to be created in a task.
memfun_type< decltype(&F::operator())>::type FFL(F const &func)
auto getTableFromFilter(soa::is_filtered_table auto const &table, soa::SelectionVector &&selection)
consteval auto typeWithRef() -> T
Helper to define output for a Table.
OutputObjHandlingPolicy
Policy enum to determine OutputObj handling when writing.
void initializePartitionCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema, expressions::Filter const &filter, gandiva::NodePtr &tree, gandiva::FilterPtr &gfilter)
std::shared_ptr< arrow::Table > makeArrowTable(const char *label, std::vector< std::shared_ptr< arrow::ChunkedArray > > &&columns, std::vector< std::shared_ptr< arrow::Field > > &&fields)
auto createFieldsFromColumns(framework::pack< C... >)
SelectionVector selectionToVector(gandiva::Selection const &sel)
constexpr auto tableRef2InputSpec()
constexpr auto tableRef2Output()
std::vector< int64_t > SelectionVector
auto Attach(T const &table)
constexpr auto tableRef2ConfigParamSpec()
auto Extend(T const &table)
On-the-fly adding of expression columns.
constexpr auto tableRef2OutputSpec()
constexpr auto tableRef2OutputRef()
Defining DataPointCompositeObject explicitly as copiable.
std::shared_ptr< T > table
T const & operator*() const
auto build(framework::pack< Cs... >, std::vector< std::shared_ptr< arrow::Table > > &&tables)
metadata::index_pack_t index_pack_t
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
T::table_t const & operator*() const
decltype(transformBase< T >())::metadata metadata
typename metadata::extension_table_t extension_t
std::array< o2::framework::expressions::Projector, N > projectors
std::shared_ptr< extension_t > extension
std::shared_ptr< arrow::Schema > inputSchema
std::shared_ptr< arrow::Schema > schema
typename metadata::base_table_t base_table_t
static constexpr bool delayed
static constexpr size_t N
typename metadata::placeholders_pack_t placeholders_pack_t
T::table_t * operator->()
std::shared_ptr< gandiva::Projector > projector
static auto indexBuilder(const char *label, std::vector< std::shared_ptr< arrow::Table > > &&tables, framework::pack< C1, Cs... >)
static OutputSpec const spec()
aod::MetadataTrait< o2::aod::Hash< table_t::ref.desc_hash > >::metadata metadata
decltype(typeWithRef< T >()) table_t
OutputObj(T &&t, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
void setObject(std::shared_ptr< T > t)
OutputObjSourceType sourceType
OutputRef ref(uint16_t index, uint16_t max)
std::shared_ptr< T > object
void setObject(T const &t)
OutputObj(std::string const &label_, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
OutputObjHandlingPolicy policy
void setHash(uint32_t hash)
Partition(expressions::Node &&filter_, T const &table)
typename o2::soa::Filtered< T >::const_iterator filtered_const_iterator
void bindTable(T const &table)
filtered_const_iterator begin() const
auto rawSliceBy(o2::framework::Preslice< T1 > const &container, int value) const
auto sliceByCached(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
void intializeCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema)
Partition(expressions::Node &&filter_)
typename o2::soa::Filtered< T >::iterator filtered_iterator
o2::soa::Filtered< T > * operator->()
typename o2::soa::Filtered< T >::iterator iterator
auto sliceBy(o2::framework::PresliceBase< T1, Policy, OPT > const &container, int value) const
std::shared_ptr< arrow::Table > asArrowTable() const
o2::soa::RowViewSentinel end() const
void updatePlaceholders(InitContext &context)
std::unique_ptr< o2::soa::Filtered< T > > mFiltered
void bindInternalIndicesTo(E *ptr)
expressions::Filter filter
o2::soa::RowViewSentinel end()
auto sliceByCachedUnsorted(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
typename o2::soa::Filtered< T >::const_iterator const_iterator
gandiva::FilterPtr gfilter
void bindExternalIndices(Ts *... tables)
filtered_iterator begin()
T::table_t * operator->()
std::shared_ptr< gandiva::Projector > projector
std::shared_ptr< extension_t > extension
T::table_t const & operator*() const
static constexpr size_t N
typename metadata::base_table_t base_table_t
std::shared_ptr< arrow::Schema > schema
typename metadata::expression_pack_t expression_pack_t
std::array< o2::framework::expressions::Projector, N > projectors
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
typename metadata::extension_table_t extension_t
int64_t lastIndex()
Last index inserted in the table.
decltype(FFL(std::declval< cursor_t >())) cursor
bool resetCursor(LifetimeHolder< TableBuilder > builder)
void setLabel(const char *label)
void operator()(Ts &&... args)
void reserve(int64_t size)
decltype([]() { if constexpr(soa::is_iterator< T >) { return typename T::parent_t{nullptr} persistent_table_t
decltype(std::declval< TableBuilder >().cursor< persistent_table_t >()) cursor_t
An expression tree node corresponding to a column binding.
A struct, containing the root of the expression tree.
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))