Project
Loading...
Searching...
No Matches
AnalysisHelpers.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11#ifndef o2_framework_AnalysisHelpers_H_DEFINED
12#define o2_framework_AnalysisHelpers_H_DEFINED
13
14#include "Framework/ASoA.h"
17#include "Framework/InputSpec.h"
18#include "Framework/Output.h"
20#include "Framework/OutputRef.h"
22#include "Framework/Plugins.h"
25#include "Framework/Traits.h"
26
27#include <string>
28namespace o2::soa
29{
30template <TableRef R>
32{
34 std::string{"input:"} + o2::aod::label<R>(),
36 aod::sourceSpec<R>(),
37 {"\"\""}};
38}
39
40namespace
41{
42template <soa::with_sources T>
43inline constexpr auto getSources()
44{
45 return []<size_t N, std::array<soa::TableRef, N> refs>() {
46 return []<size_t... Is>(std::index_sequence<Is...>) {
47 return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
48 }(std::make_index_sequence<N>());
49 }.template operator()<T::sources.size(), T::sources>();
50}
51
52template <soa::with_sources T>
53constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
54{
55 std::vector<framework::ConfigParamSpec> inputMetadata;
56 auto inputSources = getSources<T>();
57 std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
58 auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
59 inputSources.erase(last, inputSources.end());
60 inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());
61 return inputMetadata;
62}
63
64template <typename T>
65 requires(!soa::with_sources<T>)
66constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
67{
68 return {};
69}
70} // namespace
71
72template <TableRef R>
73constexpr auto tableRef2InputSpec()
74{
76 o2::aod::label<R>(),
77 o2::aod::origin<R>(),
78 o2::aod::description(o2::aod::signature<R>()),
79 R.version,
80 framework::Lifetime::Timeframe,
81 getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>()};
82}
83
84template <TableRef R>
85constexpr auto tableRef2OutputSpec()
86{
88 framework::OutputLabel{o2::aod::label<R>()},
89 o2::aod::origin<R>(),
90 o2::aod::description(o2::aod::signature<R>()),
91 R.version};
92}
93
94template <TableRef R>
95constexpr auto tableRef2Output()
96{
97 return framework::Output{
98 o2::aod::origin<R>(),
99 o2::aod::description(o2::aod::signature<R>()),
100 R.version};
101}
102
103template <TableRef R>
104constexpr auto tableRef2OutputRef()
105{
107 o2::aod::label<R>(),
108 R.version};
109}
110} // namespace o2::soa
111
112namespace o2::framework
113{
114class TableConsumer;
115
119template <typename T>
121
122template <is_producable T>
124 public:
125 using persistent_table_t = decltype([]() { if constexpr (soa::is_iterator<T>) { return typename T::parent_t{nullptr}; } else { return T{nullptr}; } }());
126 using cursor_t = decltype(std::declval<TableBuilder>().cursor<persistent_table_t>());
127
128 template <typename... Ts>
129 void operator()(Ts... args)
130 {
131 static_assert(sizeof...(Ts) == framework::pack_size(typename persistent_table_t::persistent_columns_t{}), "Argument number mismatch");
132 ++mCount;
133 cursor(0, extract(args)...);
134 }
135
137 int64_t lastIndex()
138 {
139 return mCount;
140 }
141
143 {
144 mBuilder = std::move(builder);
145 cursor = std::move(FFL(mBuilder->cursor<persistent_table_t>()));
146 mCount = -1;
147 return true;
148 }
149
150 void setLabel(const char* label)
151 {
152 mBuilder->setLabel(label);
153 }
154
157 void reserve(int64_t size)
158 {
159 mBuilder->reserve(typename persistent_table_t::column_types{}, size);
160 }
161
162 void release()
163 {
164 mBuilder.release();
165 }
166
167 decltype(FFL(std::declval<cursor_t>())) cursor;
168
169 private:
170 template <typename A>
171 requires requires { &A::globalIndex; }
172 static decltype(auto) extract(A const& arg)
173 {
174 return arg.globalIndex();
175 }
176
177 template <typename A>
178 static decltype(auto) extract(A const& arg)
179 {
180 return arg;
181 }
182
186 LifetimeHolder<TableBuilder> mBuilder = nullptr;
187 int64_t mCount = -1;
188};
189
191template <soa::is_table T>
192consteval auto typeWithRef() -> T
193{
194}
195
196template <soa::is_iterator T>
197consteval auto typeWithRef() -> typename T::parent_t
198{
199}
200
201template <typename T>
202 requires soa::is_table<T> || soa::is_iterator<T>
204 using table_t = decltype(typeWithRef<T>());
205 using metadata = aod::MetadataTrait<o2::aod::Hash<table_t::ref.desc_hash>>::metadata;
206
207 static OutputSpec const spec()
208 {
209 return OutputSpec{OutputLabel{aod::label<table_t::ref>()}, o2::aod::origin<table_t::ref>(), o2::aod::description(o2::aod::signature<table_t::ref>()), table_t::ref.version};
210 }
211
212 static OutputRef ref()
213 {
214 return OutputRef{aod::label<table_t::ref>(), table_t::ref.version};
215 }
216};
217
222template <is_producable T>
224};
225
226template <typename T>
227concept is_produces = requires(T t) { typename T::cursor_t; typename T::persistent_table_t; &T::cursor; };
228
238};
239
240template <typename T>
241concept is_produces_group = std::derived_from<T, ProducesGroup>;
242
244template <soa::is_metadata M, soa::TableRef Ref>
246 using metadata = M;
247 constexpr static auto sources = M::sources;
248
249 template <soa::TableRef R>
250 static constexpr auto base_spec()
251 {
252 return soa::tableRef2InputSpec<R>();
253 }
254
255 static auto base_specs()
256 {
257 return []<size_t... Is>(std::index_sequence<Is...>) -> std::vector<InputSpec> {
258 return {base_spec<sources[Is]>()...};
259 }(std::make_index_sequence<sources.size()>{});
260 }
261
262 constexpr auto spec() const
263 {
264 return soa::tableRef2OutputSpec<Ref>();
265 }
266
267 constexpr auto output() const
268 {
269 return soa::tableRef2Output<Ref>();
270 }
271
272 constexpr auto ref() const
273 {
274 return soa::tableRef2OutputRef<Ref>();
275 }
276};
277
280template <typename T>
281concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
282
283template <typename T>
284concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
285
286template <is_spawnable T>
287constexpr auto transformBase()
288{
289 using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
291}
292
293template <is_spawnable T>
294struct Spawns : decltype(transformBase<T>()) {
295 using spawnable_t = T;
296 using metadata = decltype(transformBase<T>())::metadata;
297 using extension_t = typename metadata::extension_table_t;
298 using base_table_t = typename metadata::base_table_t;
299 using expression_pack_t = typename metadata::expression_pack_t;
300 static constexpr size_t N = framework::pack_size(expression_pack_t{});
301
302 constexpr auto pack()
303 {
304 return expression_pack_t{};
305 }
306
307 typename T::table_t* operator->()
308 {
309 return table.get();
310 }
311 typename T::table_t const& operator*() const
312 {
313 return *table;
314 }
315
317 {
318 return extension->asArrowTable();
319 }
320 std::shared_ptr<typename T::table_t> table = nullptr;
321 std::shared_ptr<extension_t> extension = nullptr;
322 std::array<o2::framework::expressions::Projector, N> projectors = []<typename... C>(framework::pack<C...>) -> std::array<expressions::Projector, sizeof...(C)>
323 {
324 return {{std::move(C::Projector())...}};
325 }
327 std::shared_ptr<gandiva::Projector> projector = nullptr;
328 std::shared_ptr<arrow::Schema> schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(expression_pack_t{}));
329};
330
331template <typename T>
332concept is_spawns = requires(T t) {
333 typename T::metadata;
334 requires std::same_as<decltype(t.pack()), typename T::expression_pack_t>;
335 requires std::same_as<decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
336};
337
342
343template <is_dynamically_spawnable T, bool DELAYED = false>
344struct Defines : decltype(transformBase<T>()) {
345 static constexpr bool delayed = DELAYED;
346 using spawnable_t = T;
347 using metadata = decltype(transformBase<T>())::metadata;
348 using extension_t = typename metadata::extension_table_t;
349 using base_table_t = typename metadata::base_table_t;
350 using placeholders_pack_t = typename metadata::placeholders_pack_t;
351 static constexpr size_t N = framework::pack_size(placeholders_pack_t{});
352
353 constexpr auto pack()
354 {
355 return placeholders_pack_t{};
356 }
357
358 typename T::table_t* operator->()
359 {
360 return table.get();
361 }
362 typename T::table_t const& operator*() const
363 {
364 return *table;
365 }
366
368 {
369 return extension->asArrowTable();
370 }
371 std::shared_ptr<typename T::table_t> table = nullptr;
372 std::shared_ptr<extension_t> extension = nullptr;
373
374 std::array<o2::framework::expressions::Projector, N> projectors;
375 std::shared_ptr<gandiva::Projector> projector = nullptr;
376 std::shared_ptr<arrow::Schema> schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(placeholders_pack_t{}));
377 std::shared_ptr<arrow::Schema> inputSchema = nullptr;
378
379 bool needRecompilation = false;
380
385};
386
387template <is_dynamically_spawnable T>
389
390template <typename T>
391concept is_defines = requires(T t) {
392 typename T::metadata;
393 requires std::same_as<decltype(t.pack()), typename T::placeholders_pack_t>;
394 requires std::same_as<decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
395 requires std::same_as<decltype(t.needRecompilation), bool>;
396 &T::recompile;
397};
398
403struct Exclusive {
404};
405struct Sparse {
406};
407
408namespace
409{
410template <typename T, typename Key>
411inline std::shared_ptr<arrow::ChunkedArray> getIndexToKey(arrow::Table* table)
412{
413 using IC = framework::pack_element_t<framework::has_type_at_conditional_v<soa::is_binding_compatible, Key>(typename T::external_index_columns_t{}), typename T::external_index_columns_t>;
414 return table->column(framework::has_type_at_v<IC>(typename T::persistent_columns_t{}));
415}
416
417template <soa::is_column C>
418struct ColumnTrait {
419 using column_t = C;
420
421 static consteval auto listSize()
422 {
423 if constexpr (std::same_as<typename C::type, std::vector<int>>) {
424 return -1;
425 } else if constexpr (std::same_as<int[2], typename C::type>) {
426 return 2;
427 } else {
428 return 1;
429 }
430 }
431
432 template <typename T, typename Key>
433 static std::shared_ptr<SelfIndexColumnBuilder> makeColumnBuilder(arrow::Table* table, arrow::MemoryPool* pool)
434 {
435 if constexpr (!std::same_as<T, Key>) {
436 return std::make_shared<IndexColumnBuilder>(getIndexToKey<T, Key>(table), C::columnLabel(), listSize(), pool);
437 } else {
438 return std::make_shared<SelfIndexColumnBuilder>(C::columnLabel(), pool);
439 }
440 }
441};
442
443template <typename Key, typename C>
444struct Reduction {
445 using type = typename std::conditional<soa::is_binding_compatible_v<Key, typename C::binding_t>(), SelfIndexColumnBuilder, IndexColumnBuilder>::type;
446};
447
448template <typename Key, typename C>
449using reduced_t = Reduction<Key, C>::type;
450} // namespace
451
452template <typename Kind>
454 template <typename Key, size_t N, std::array<soa::TableRef, N> refs, typename C1, typename... Cs>
455 static auto indexBuilder(const char* label, std::vector<std::shared_ptr<arrow::Table>>&& tables, framework::pack<C1, Cs...>)
456 {
457 auto pool = arrow::default_memory_pool();
458 SelfIndexColumnBuilder self{C1::columnLabel(), pool};
459 std::unique_ptr<ChunkedArrayIterator> keyIndex = nullptr;
460 if constexpr (!Key::template hasOriginal<refs[0]>()) {
461 keyIndex = std::make_unique<ChunkedArrayIterator>(tables[0]->column(o2::aod::MetadataTrait<o2::aod::Hash<refs[0].desc_hash>>::metadata::template getIndexPosToKey<Key>()));
462 }
463
464 auto sq = std::make_index_sequence<sizeof...(Cs)>();
465
466 auto columnBuilders = [&tables, &pool ]<size_t... Is>(std::index_sequence<Is...>) -> std::array<std::shared_ptr<framework::SelfIndexColumnBuilder>, sizeof...(Cs)>
467 {
468 return {[](arrow::Table* table, arrow::MemoryPool* pool) {
469 using T = framework::pack_element_t<Is, framework::pack<Cs...>>;
470 if constexpr (!Key::template hasOriginal<refs[Is + 1]>()) {
471 constexpr auto pos = o2::aod::MetadataTrait<o2::aod::Hash<refs[Is + 1].desc_hash>>::metadata::template getIndexPosToKey<Key>();
472 return std::make_shared<IndexColumnBuilder>(table->column(pos), T::columnLabel(), ColumnTrait<T>::listSize(), pool);
473 } else {
474 return std::make_shared<SelfIndexColumnBuilder>(T::columnLabel(), pool);
475 }
476 }(tables[Is + 1].get(), pool)...};
477 }
478 (sq);
479
480 std::array<bool, sizeof...(Cs)> finds;
481
482 for (int64_t counter = 0; counter < tables[0]->num_rows(); ++counter) {
483 int64_t idx = -1;
484 if constexpr (Key::template hasOriginal<refs[0]>()) {
485 idx = counter;
486 } else {
487 idx = keyIndex->valueAt(counter);
488 }
489 finds = [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
490 return std::array{
491 [&idx, &columnBuilders]() {
492 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
493 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template find<T>(idx);
494 }()...};
495 }(sq);
496 if constexpr (std::same_as<Kind, Sparse>) {
497 [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
498 ([&idx, &columnBuilders]() {
499 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
500 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template fill<T>(idx); }(), ...);
501 }(sq);
502 self.fill<C1>(counter);
503 } else if constexpr (std::same_as<Kind, Exclusive>) {
504 if (std::none_of(finds.begin(), finds.end(), [](bool const x) { return x == false; })) {
505 [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
506 ([&idx, &columnBuilders]() {
507 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
508 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template fill<T>(idx);
509 }(),
510 ...);
511 }(sq);
512 self.fill<C1>(counter);
513 }
514 }
515 }
516
517 return [&label, &columnBuilders, &self]<size_t... Is>(std::index_sequence<Is...>) {
518 return makeArrowTable(label,
519 {self.template result<C1>(), [&columnBuilders]() {
520 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
521 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template result<T>();
522 }()...},
523 {self.field(), [&columnBuilders]() {
524 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
525 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->field();
526 }()...});
527 }(sq);
528 }
529};
530
532
533template <soa::is_index_table T>
534constexpr auto transformBase()
535{
536 using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
538}
539
540template <soa::is_index_table T>
541struct Builds : decltype(transformBase<T>()) {
542 using buildable_t = T;
543 using metadata = decltype(transformBase<T>())::metadata;
544 using IP = std::conditional_t<metadata::exclusive, IndexBuilder<Exclusive>, IndexBuilder<Sparse>>;
545 using Key = metadata::Key;
546 using H = typename T::first_t;
547 using Ts = typename T::rest_t;
548 using index_pack_t = metadata::index_pack_t;
549
551 {
552 return table.get();
553 }
554 T const& operator*() const
555 {
556 return *table;
557 }
558
560 {
561 return table->asArrowTable();
562 }
563 std::shared_ptr<T> table = nullptr;
564
565 constexpr auto pack()
566 {
567 return index_pack_t{};
568 }
569
570 template <typename Key, typename... Cs>
571 auto build(framework::pack<Cs...>, std::vector<std::shared_ptr<arrow::Table>>&& tables)
572 {
573 this->table = std::make_shared<T>(IP::template indexBuilder<Key, metadata::sources.size(), metadata::sources>(o2::aod::label<T::ref>(), std::forward<std::vector<std::shared_ptr<arrow::Table>>>(tables), framework::pack<Cs...>{}));
574 return (this->table != nullptr);
575 }
576};
577
578template <typename T>
579concept is_builds = requires(T t) {
580 typename T::metadata;
581 typename T::Key;
582 requires std::same_as<decltype(t.pack()), typename T::index_pack_t>;
583};
584
592template <typename T>
593struct OutputObj {
594 using obj_t = T;
595
597 : object(std::make_shared<T>(t)),
598 label(t.GetName()),
599 policy{policy_},
600 sourceType{sourceType_},
601 mTaskHash{0}
602 {
603 }
604
606 : object(nullptr),
607 label(label_),
608 policy{policy_},
609 sourceType{sourceType_},
610 mTaskHash{0}
611 {
612 }
613
614 void setObject(T const& t)
615 {
616 object = std::make_shared<T>(t);
617 object->SetName(label.c_str());
618 }
619
620 void setObject(T&& t)
621 {
622 object = std::make_shared<T>(t);
623 object->SetName(label.c_str());
624 }
625
626 void setObject(T* t)
627 {
628 object.reset(t);
629 object->SetName(label.c_str());
630 }
631
632 void setObject(std::shared_ptr<T> t)
633 {
634 object = t;
635 object->SetName(label.c_str());
636 }
637
638 void setHash(uint32_t hash)
639 {
640 mTaskHash = hash;
641 }
642
645 {
647 auto lhash = runtime_hash(label.c_str());
648 std::memset(desc.str, '_', 16);
649 std::stringstream s;
650 s << std::hex << lhash;
651 s << std::hex << mTaskHash;
652 s << std::hex << reinterpret_cast<uint64_t>(this);
653 std::memcpy(desc.str, s.str().c_str(), 12);
654 return OutputSpec{OutputLabel{label}, "ATSK", desc, 0, Lifetime::QA};
655 }
656
658 {
659 return object.get();
660 }
661
663 {
664 return *object.get();
665 }
666
667 OutputRef ref(uint16_t index, uint16_t max)
668 {
669 return OutputRef{std::string{label}, 0,
671 }
672
673 std::shared_ptr<T> object;
674 std::string label;
677 uint32_t mTaskHash;
678};
679
680template <typename T>
681concept is_outputobj = requires(T t) {
682 &T::setHash;
683 &T::spec;
684 &T::ref;
685 requires std::same_as<decltype(t.operator->()), typename T::obj_t*>;
686 requires std::same_as<decltype(t.object), std::shared_ptr<typename T::obj_t>>;
687};
688
692template <typename T>
693struct Service {
694 using service_t = T;
696
697 decltype(auto) operator->() const
698 {
700 return service->get();
701 } else {
702 return service;
703 }
704 }
705};
706
707template <typename T>
708concept is_service = requires(T t) {
709 requires std::same_as<decltype(t.service), typename T::service_t*>;
710 &T::operator->;
711};
712
714{
715 return std::make_unique<o2::soa::Filtered<std::decay_t<decltype(table)>>>(std::vector{table}, std::forward<soa::SelectionVector>(selection));
716}
717
719{
720 return std::make_unique<o2::soa::Filtered<std::decay_t<decltype(table)>>>(std::vector{table.asArrowTable()}, std::forward<soa::SelectionVector>(selection));
721}
722
723void initializePartitionCaches(std::set<uint32_t> const& hashes, std::shared_ptr<arrow::Schema> const& schema, expressions::Filter const& filter, gandiva::NodePtr& tree, gandiva::FilterPtr& gfilter);
724
725template <typename T>
726struct Partition {
727 using content_t = T;
728 Partition(expressions::Node&& filter_) : filter{std::forward<expressions::Node>(filter_)}
729 {
730 }
731
732 Partition(expressions::Node&& filter_, T const& table)
733 : filter{std::forward<expressions::Node>(filter_)}
734 {
735 setTable(table);
736 }
737
738 void intializeCaches(std::set<uint32_t> const& hashes, std::shared_ptr<arrow::Schema> const& schema)
739 {
740 initializePartitionCaches(hashes, schema, filter, tree, gfilter);
741 }
742
743 void bindTable(T const& table)
744 {
745 intializeCaches(T::table_t::hashes(), table.asArrowTable()->schema());
746 if (dataframeChanged) {
748 dataframeChanged = false;
749 }
750 }
751
752 template <typename... Ts>
753 void bindExternalIndices(Ts*... tables)
754 {
755 if (mFiltered != nullptr) {
756 mFiltered->bindExternalIndices(tables...);
757 }
758 }
759
760 template <typename E>
762 {
763 if (mFiltered != nullptr) {
764 mFiltered->bindInternalIndicesTo(ptr);
765 }
766 }
767
769 {
771 }
772
773 [[nodiscard]] std::shared_ptr<arrow::Table> asArrowTable() const
774 {
775 return mFiltered->asArrowTable();
776 }
777
779 {
780 return mFiltered.get();
781 }
782
783 template <typename T1>
784 [[nodiscard]] auto rawSliceBy(o2::framework::Preslice<T1> const& container, int value) const
785 {
786 return mFiltered->rawSliceBy(container, value);
787 }
788
789 [[nodiscard]] auto sliceByCached(framework::expressions::BindingNode const& node, int value, o2::framework::SliceCache& cache) const
790 {
791 return mFiltered->sliceByCached(node, value, cache);
792 }
793
795 {
796 return mFiltered->sliceByCachedUnsorted(node, value, cache);
797 }
798
799 template <typename T1, typename Policy, bool OPT>
800 [[nodiscard]] auto sliceBy(o2::framework::PresliceBase<T1, Policy, OPT> const& container, int value) const
801 {
802 return mFiltered->sliceBy(container, value);
803 }
804
806 std::unique_ptr<o2::soa::Filtered<T>> mFiltered = nullptr;
807 gandiva::NodePtr tree = nullptr;
809 bool dataframeChanged = true;
810
816 {
817 return mFiltered->begin();
818 }
820 {
821 return mFiltered->end();
822 }
824 {
825 return mFiltered->begin();
826 }
828 {
829 return mFiltered->end();
830 }
831
832 int64_t size() const
833 {
834 return mFiltered->size();
835 }
836};
837
838template <typename T>
839concept is_partition = requires(T t) {
840 &T::updatePlaceholders;
841 requires std::same_as<decltype(t.filter), expressions::Filter>;
842 requires std::same_as<decltype(t.mFiltered), std::unique_ptr<o2::soa::Filtered<typename T::content_t>>>;
843};
844} // namespace o2::framework
845
846namespace o2::soa
847{
849template <soa::is_table T, soa::is_spawnable_column... Cs>
850auto Extend(T const& table)
851{
852 using output_t = Join<T, soa::Table<o2::aod::Hash<"JOIN"_h>, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Cs...>>;
853 static std::array<framework::expressions::Projector, sizeof...(Cs)> projectors{{std::move(Cs::Projector())...}};
854 static std::shared_ptr<gandiva::Projector> projector = nullptr;
855 static auto schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(framework::pack<Cs...>{}));
856 return output_t{{o2::framework::spawner(framework::pack<Cs...>{}, {table.asArrowTable()}, "dynamicExtension", projectors.data(), projector, schema), table.asArrowTable()}, 0};
857}
858
861template <soa::is_table T, soa::is_dynamic_column... Cs>
862auto Attach(T const& table)
863{
864 using output_t = Join<T, o2::soa::Table<o2::aod::Hash<"JOIN"_h>, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Cs...>>;
865 return output_t{{table.asArrowTable()}, table.offset()};
866}
867} // namespace o2::soa
868
869#endif // o2_framework_AnalysisHelpers_H_DEFINED
uint16_t pos
Definition RawData.h:3
uint32_t gfilter
Definition RawData.h:6
constexpr uint32_t runtime_hash(char const *str)
TBranch * ptr
Definition A.h:16
iterator const_iterator
Definition ASoA.h:3643
T::template iterator_template_o< FilteredIndexPolicy, self_t > iterator
Definition ASoA.h:3641
Helper to check if a type T is an iterator.
Definition ASoA.h:1257
GLint GLenum GLint x
Definition glcorearb.h:403
GLuint64EXT * result
Definition glcorearb.h:5662
GLsizeiptr size
Definition glcorearb.h:659
GLuint index
Definition glcorearb.h:781
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei const GLfloat * value
Definition glcorearb.h:819
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
GLuint GLsizei const GLchar * label
Definition glcorearb.h:2519
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition glcorearb.h:1308
GLuint object
Definition glcorearb.h:4041
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
GLsizei GLenum * sources
Definition glcorearb.h:2516
GLuint counter
Definition glcorearb.h:3987
std::shared_ptr< gandiva::Filter > FilterPtr
Definition Expressions.h:47
std::variant< OriginValueMatcher, DescriptionValueMatcher, SubSpecificationTypeValueMatcher, std::unique_ptr< DataDescriptorMatcher >, ConstantValueMatcher, StartTimeValueMatcher > Node
std::shared_ptr< gandiva::Projector > createProjectorHelper(size_t nColumns, expressions::Projector *projectors, std::shared_ptr< arrow::Schema > schema, std::vector< std::shared_ptr< arrow::Field > > const &fields)
gandiva::Selection createSelection(std::shared_ptr< arrow::Table > const &table, Filter const &expression)
Function for creating gandiva selection from our internal filter tree.
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
auto spawner(std::shared_ptr< arrow::Table > const &fullTable, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector, std::shared_ptr< arrow::Schema > const &schema)
Expression-based column generator to materialize columns.
typename pack_element< I, T >::type pack_element_t
Definition Pack.h:56
constexpr std::size_t pack_size(pack< Ts... > const &)
template function to determine number of types in a pack
Definition Pack.h:28
constexpr auto transformBase()
This helper struct allows you to declare index tables to be created in a task.
memfun_type< decltype(&F::operator())>::type FFL(F const &func)
auto getTableFromFilter(soa::is_filtered_table auto const &table, soa::SelectionVector &&selection)
consteval auto typeWithRef() -> T
Helper to define output for a Table.
OutputObjHandlingPolicy
Policy enum to determine OutputObj handling when writing.
void initializePartitionCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema, expressions::Filter const &filter, gandiva::NodePtr &tree, gandiva::FilterPtr &gfilter)
std::shared_ptr< arrow::Table > makeArrowTable(const char *label, std::vector< std::shared_ptr< arrow::ChunkedArray > > &&columns, std::vector< std::shared_ptr< arrow::Field > > &&fields)
auto createFieldsFromColumns(framework::pack< C... >)
Definition ASoA.h:407
SelectionVector selectionToVector(gandiva::Selection const &sel)
Definition ASoA.cxx:43
constexpr auto tableRef2InputSpec()
constexpr auto tableRef2Output()
std::vector< int64_t > SelectionVector
Definition ASoA.h:412
auto Attach(T const &table)
constexpr auto tableRef2ConfigParamSpec()
auto Extend(T const &table)
On-the-fly adding of expression columns.
constexpr auto tableRef2OutputSpec()
constexpr auto tableRef2OutputRef()
@ C
Definition Defs.h:36
Defining DataPointCompositeObject explicitly as copiable.
std::shared_ptr< T > table
constexpr auto pack()
typename T::first_t H
T const & operator*() const
auto build(framework::pack< Cs... >, std::vector< std::shared_ptr< arrow::Table > > &&tables)
metadata::index_pack_t index_pack_t
typename T::rest_t Ts
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
T::table_t const & operator*() const
decltype(transformBase< T >())::metadata metadata
typename metadata::extension_table_t extension_t
std::array< o2::framework::expressions::Projector, N > projectors
std::shared_ptr< extension_t > extension
std::shared_ptr< arrow::Schema > inputSchema
std::shared_ptr< arrow::Schema > schema
typename metadata::base_table_t base_table_t
static constexpr bool delayed
static constexpr size_t N
typename metadata::placeholders_pack_t placeholders_pack_t
std::shared_ptr< gandiva::Projector > projector
static auto indexBuilder(const char *label, std::vector< std::shared_ptr< arrow::Table > > &&tables, framework::pack< C1, Cs... >)
static OutputSpec const spec()
aod::MetadataTrait< o2::aod::Hash< table_t::ref.desc_hash > >::metadata metadata
decltype(typeWithRef< T >()) table_t
O2 header for OutputObj metadata.
OutputObj(T &&t, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
void setObject(std::shared_ptr< T > t)
OutputObjSourceType sourceType
OutputRef ref(uint16_t index, uint16_t max)
std::shared_ptr< T > object
OutputObj(std::string const &label_, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
OutputSpec const spec()
OutputObjHandlingPolicy policy
void setHash(uint32_t hash)
Partition(expressions::Node &&filter_, T const &table)
typename o2::soa::Filtered< T >::const_iterator filtered_const_iterator
void bindTable(T const &table)
filtered_const_iterator begin() const
auto rawSliceBy(o2::framework::Preslice< T1 > const &container, int value) const
auto sliceByCached(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
void intializeCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema)
Partition(expressions::Node &&filter_)
typename o2::soa::Filtered< T >::iterator filtered_iterator
o2::soa::Filtered< T > * operator->()
typename o2::soa::Filtered< T >::iterator iterator
auto sliceBy(o2::framework::PresliceBase< T1, Policy, OPT > const &container, int value) const
std::shared_ptr< arrow::Table > asArrowTable() const
o2::soa::RowViewSentinel end() const
void updatePlaceholders(InitContext &context)
std::unique_ptr< o2::soa::Filtered< T > > mFiltered
expressions::Filter filter
o2::soa::RowViewSentinel end()
auto sliceByCachedUnsorted(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
typename o2::soa::Filtered< T >::const_iterator const_iterator
gandiva::FilterPtr gfilter
void bindExternalIndices(Ts *... tables)
filtered_iterator begin()
T::table_t * operator->()
std::shared_ptr< gandiva::Projector > projector
std::shared_ptr< extension_t > extension
T::table_t const & operator*() const
static constexpr size_t N
typename metadata::base_table_t base_table_t
std::shared_ptr< arrow::Schema > schema
typename metadata::expression_pack_t expression_pack_t
constexpr auto pack()
std::array< o2::framework::expressions::Projector, N > projectors
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
typename metadata::extension_table_t extension_t
Helper template for table transformations.
constexpr auto spec() const
constexpr auto output() const
static constexpr auto base_spec()
constexpr auto ref() const
int64_t lastIndex()
Last index inserted in the table.
decltype(FFL(std::declval< cursor_t >())) cursor
bool resetCursor(LifetimeHolder< TableBuilder > builder)
void setLabel(const char *label)
decltype([]() { if constexpr(soa::is_iterator< T >) { return typename T::parent_t{nullptr} persistent_table_t
decltype(std::declval< TableBuilder >().cursor< persistent_table_t >()) cursor_t
An expression tree node corresponding to a column binding.
A struct, containing the root of the expression tree.
a move-only header stack with serialized headers This is the flat buffer where all the headers in a m...
Definition Stack.h:33
constexpr size_t max
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))