Project
Loading...
Searching...
No Matches
AnalysisHelpers.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11#ifndef o2_framework_AnalysisHelpers_H_DEFINED
12#define o2_framework_AnalysisHelpers_H_DEFINED
13
14#include "Framework/ASoA.h"
17#include "Framework/InputSpec.h"
18#include "Framework/Output.h"
20#include "Framework/OutputRef.h"
22#include "Framework/Plugins.h"
25#include "Framework/Traits.h"
26
27#include <string>
28namespace o2::soa
29{
30template <TableRef R>
32{
34 std::string{"input:"} + o2::aod::label<R>(),
36 aod::sourceSpec<R>(),
37 {"\"\""}};
38}
39
40namespace
41{
42template <soa::with_sources T>
43inline constexpr auto getSources()
44{
45 return []<size_t N, std::array<soa::TableRef, N> refs>() {
46 return []<size_t... Is>(std::index_sequence<Is...>) {
47 return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
48 }(std::make_index_sequence<N>());
49 }.template operator()<T::sources.size(), T::sources>();
50}
51
52template <soa::with_sources T>
53constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
54{
55 std::vector<framework::ConfigParamSpec> inputMetadata;
56 auto inputSources = getSources<T>();
57 std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
58 auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
59 inputSources.erase(last, inputSources.end());
60 inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());
61 return inputMetadata;
62}
63
64template <typename T>
65 requires(!soa::with_sources<T>)
66constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
67{
68 return {};
69}
70} // namespace
71
72template <TableRef R>
73constexpr auto tableRef2InputSpec()
74{
76 o2::aod::label<R>(),
77 o2::aod::origin<R>(),
78 o2::aod::description(o2::aod::signature<R>()),
79 R.version,
80 framework::Lifetime::Timeframe,
81 getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>()};
82}
83
84template <TableRef R>
85constexpr auto tableRef2OutputSpec()
86{
88 framework::OutputLabel{o2::aod::label<R>()},
89 o2::aod::origin<R>(),
90 o2::aod::description(o2::aod::signature<R>()),
91 R.version};
92}
93
94template <TableRef R>
95constexpr auto tableRef2Output()
96{
97 return framework::Output{
98 o2::aod::origin<R>(),
99 o2::aod::description(o2::aod::signature<R>()),
100 R.version};
101}
102
103template <TableRef R>
104constexpr auto tableRef2OutputRef()
105{
107 o2::aod::label<R>(),
108 R.version};
109}
110} // namespace o2::soa
111
112namespace o2::framework
113{
114class TableConsumer;
115
119template <typename T>
121
122template <typename T>
123concept is_enumerated_iterator = requires(T t) { t.globalIndex(); };
124
125template <is_producable T>
127 public:
128 using persistent_table_t = decltype([]() { if constexpr (soa::is_iterator<T>) { return typename T::parent_t{nullptr}; } else { return T{nullptr}; } }());
129 using cursor_t = decltype(std::declval<TableBuilder>().cursor<persistent_table_t>());
130
131 template <typename... Ts>
132 void operator()(Ts&&... args)
133 requires(sizeof...(Ts) == framework::pack_size(typename persistent_table_t::persistent_columns_t{}))
134 {
135 ++mCount;
136 cursor(0, extract(args)...);
137 }
138
140 int64_t lastIndex()
141 {
142 return mCount;
143 }
144
146 {
147 mBuilder = std::move(builder);
148 cursor = std::move(FFL(mBuilder->cursor<persistent_table_t>()));
149 mCount = -1;
150 return true;
151 }
152
153 void setLabel(const char* label)
154 {
155 mBuilder->setLabel(label);
156 }
157
160 void reserve(int64_t size)
161 {
162 mBuilder->reserve(typename persistent_table_t::column_types{}, size);
163 }
164
165 void release()
166 {
167 mBuilder.release();
168 }
169
170 decltype(FFL(std::declval<cursor_t>())) cursor;
171
172 private:
173 static decltype(auto) extract(is_enumerated_iterator auto const& arg)
174 {
175 return arg.globalIndex();
176 }
177
178 template <typename A>
180 static decltype(auto) extract(A&& arg)
181 {
182 return arg;
183 }
184
188 LifetimeHolder<TableBuilder> mBuilder = nullptr;
189 int64_t mCount = -1;
190};
191
193template <soa::is_table T>
194consteval auto typeWithRef() -> T
195{
196}
197
198template <soa::is_iterator T>
199consteval auto typeWithRef() -> typename T::parent_t
200{
201}
202
203template <typename T>
204 requires soa::is_table<T> || soa::is_iterator<T>
206 using table_t = decltype(typeWithRef<T>());
207 using metadata = aod::MetadataTrait<o2::aod::Hash<table_t::ref.desc_hash>>::metadata;
208
209 static OutputSpec const spec()
210 {
211 return OutputSpec{OutputLabel{aod::label<table_t::ref>()}, o2::aod::origin<table_t::ref>(), o2::aod::description(o2::aod::signature<table_t::ref>()), table_t::ref.version};
212 }
213
214 static OutputRef ref()
215 {
216 return OutputRef{aod::label<table_t::ref>(), table_t::ref.version};
217 }
218};
219
224template <is_producable T>
226};
227
228template <typename T>
229concept is_produces = requires(T t) { typename T::cursor_t; typename T::persistent_table_t; &T::cursor; };
230
240};
241
242template <typename T>
243concept is_produces_group = std::derived_from<T, ProducesGroup>;
244
246template <soa::is_metadata M, soa::TableRef Ref>
248 using metadata = M;
249 constexpr static auto sources = M::sources;
250
251 template <soa::TableRef R>
252 static constexpr auto base_spec()
253 {
254 return soa::tableRef2InputSpec<R>();
255 }
256
257 static auto base_specs()
258 {
259 return []<size_t... Is>(std::index_sequence<Is...>) -> std::vector<InputSpec> {
260 return {base_spec<sources[Is]>()...};
261 }(std::make_index_sequence<sources.size()>{});
262 }
263
264 constexpr auto spec() const
265 {
266 return soa::tableRef2OutputSpec<Ref>();
267 }
268
269 constexpr auto output() const
270 {
271 return soa::tableRef2Output<Ref>();
272 }
273
274 constexpr auto ref() const
275 {
276 return soa::tableRef2OutputRef<Ref>();
277 }
278};
279
282template <typename T>
283concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
284
285template <typename T>
286concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
287
288template <is_spawnable T>
289constexpr auto transformBase()
290{
291 using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
293}
294
295template <is_spawnable T>
296struct Spawns : decltype(transformBase<T>()) {
297 using spawnable_t = T;
298 using metadata = decltype(transformBase<T>())::metadata;
299 using extension_t = typename metadata::extension_table_t;
300 using base_table_t = typename metadata::base_table_t;
301 using expression_pack_t = typename metadata::expression_pack_t;
302 static constexpr size_t N = framework::pack_size(expression_pack_t{});
303
304 constexpr auto pack()
305 {
306 return expression_pack_t{};
307 }
308
309 typename T::table_t* operator->()
310 {
311 return table.get();
312 }
313 typename T::table_t const& operator*() const
314 {
315 return *table;
316 }
317
319 {
320 return extension->asArrowTable();
321 }
322 std::shared_ptr<typename T::table_t> table = nullptr;
323 std::shared_ptr<extension_t> extension = nullptr;
324 std::array<o2::framework::expressions::Projector, N> projectors = []<typename... C>(framework::pack<C...>) -> std::array<expressions::Projector, sizeof...(C)>
325 {
326 return {{std::move(C::Projector())...}};
327 }
329 std::shared_ptr<gandiva::Projector> projector = nullptr;
330 std::shared_ptr<arrow::Schema> schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(expression_pack_t{}));
331};
332
333template <typename T>
334concept is_spawns = requires(T t) {
335 typename T::metadata;
336 requires std::same_as<decltype(t.pack()), typename T::expression_pack_t>;
337 requires std::same_as<decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
338};
339
344
345template <is_dynamically_spawnable T, bool DELAYED = false>
346struct Defines : decltype(transformBase<T>()) {
347 static constexpr bool delayed = DELAYED;
348 using spawnable_t = T;
349 using metadata = decltype(transformBase<T>())::metadata;
350 using extension_t = typename metadata::extension_table_t;
351 using base_table_t = typename metadata::base_table_t;
352 using placeholders_pack_t = typename metadata::placeholders_pack_t;
353 static constexpr size_t N = framework::pack_size(placeholders_pack_t{});
354
355 constexpr auto pack()
356 {
357 return placeholders_pack_t{};
358 }
359
360 typename T::table_t* operator->()
361 {
362 return table.get();
363 }
364 typename T::table_t const& operator*() const
365 {
366 return *table;
367 }
368
370 {
371 return extension->asArrowTable();
372 }
373 std::shared_ptr<typename T::table_t> table = nullptr;
374 std::shared_ptr<extension_t> extension = nullptr;
375
376 std::array<o2::framework::expressions::Projector, N> projectors;
377 std::shared_ptr<gandiva::Projector> projector = nullptr;
378 std::shared_ptr<arrow::Schema> schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(placeholders_pack_t{}));
379 std::shared_ptr<arrow::Schema> inputSchema = nullptr;
380
381 bool needRecompilation = false;
382
387};
388
389template <is_dynamically_spawnable T>
391
392template <typename T>
393concept is_defines = requires(T t) {
394 typename T::metadata;
395 requires std::same_as<decltype(t.pack()), typename T::placeholders_pack_t>;
396 requires std::same_as<decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
397 requires std::same_as<decltype(t.needRecompilation), bool>;
398 &T::recompile;
399};
400
405struct Exclusive {
406};
407struct Sparse {
408};
409
410namespace
411{
412template <typename T, typename Key>
413inline std::shared_ptr<arrow::ChunkedArray> getIndexToKey(arrow::Table* table)
414{
415 using IC = framework::pack_element_t<framework::has_type_at_conditional_v<soa::is_binding_compatible, Key>(typename T::external_index_columns_t{}), typename T::external_index_columns_t>;
416 return table->column(framework::has_type_at_v<IC>(typename T::persistent_columns_t{}));
417}
418
419template <soa::is_column C>
420struct ColumnTrait {
421 using column_t = C;
422
423 static consteval auto listSize()
424 {
425 if constexpr (std::same_as<typename C::type, std::vector<int>>) {
426 return -1;
427 } else if constexpr (std::same_as<int[2], typename C::type>) {
428 return 2;
429 } else {
430 return 1;
431 }
432 }
433
434 template <typename T, typename Key>
435 static std::shared_ptr<SelfIndexColumnBuilder> makeColumnBuilder(arrow::Table* table, arrow::MemoryPool* pool)
436 {
437 if constexpr (!std::same_as<T, Key>) {
438 return std::make_shared<IndexColumnBuilder>(getIndexToKey<T, Key>(table), C::columnLabel(), listSize(), pool);
439 } else {
440 return std::make_shared<SelfIndexColumnBuilder>(C::columnLabel(), pool);
441 }
442 }
443};
444
445template <typename Key, typename C>
446struct Reduction {
447 using type = typename std::conditional<soa::is_binding_compatible_v<Key, typename C::binding_t>(), SelfIndexColumnBuilder, IndexColumnBuilder>::type;
448};
449
450template <typename Key, typename C>
451using reduced_t = Reduction<Key, C>::type;
452} // namespace
453
454template <typename Kind>
456 template <typename Key, size_t N, std::array<soa::TableRef, N> refs, typename C1, typename... Cs>
457 static auto indexBuilder(const char* label, std::vector<std::shared_ptr<arrow::Table>>&& tables, framework::pack<C1, Cs...>)
458 {
459 auto pool = arrow::default_memory_pool();
460 SelfIndexColumnBuilder self{C1::columnLabel(), pool};
461 std::unique_ptr<ChunkedArrayIterator> keyIndex = nullptr;
462 if constexpr (!Key::template hasOriginal<refs[0]>()) {
463 keyIndex = std::make_unique<ChunkedArrayIterator>(tables[0]->column(o2::aod::MetadataTrait<o2::aod::Hash<refs[0].desc_hash>>::metadata::template getIndexPosToKey<Key>()));
464 }
465
466 auto sq = std::make_index_sequence<sizeof...(Cs)>();
467
468 auto columnBuilders = [&tables, &pool ]<size_t... Is>(std::index_sequence<Is...>) -> std::array<std::shared_ptr<framework::SelfIndexColumnBuilder>, sizeof...(Cs)>
469 {
470 return {[](arrow::Table* table, arrow::MemoryPool* pool) {
471 using T = framework::pack_element_t<Is, framework::pack<Cs...>>;
472 if constexpr (!Key::template hasOriginal<refs[Is + 1]>()) {
473 constexpr auto pos = o2::aod::MetadataTrait<o2::aod::Hash<refs[Is + 1].desc_hash>>::metadata::template getIndexPosToKey<Key>();
474 return std::make_shared<IndexColumnBuilder>(table->column(pos), T::columnLabel(), ColumnTrait<T>::listSize(), pool);
475 } else {
476 return std::make_shared<SelfIndexColumnBuilder>(T::columnLabel(), pool);
477 }
478 }(tables[Is + 1].get(), pool)...};
479 }
480 (sq);
481
482 std::array<bool, sizeof...(Cs)> finds;
483
484 for (int64_t counter = 0; counter < tables[0]->num_rows(); ++counter) {
485 int64_t idx = -1;
486 if constexpr (Key::template hasOriginal<refs[0]>()) {
487 idx = counter;
488 } else {
489 idx = keyIndex->valueAt(counter);
490 }
491 finds = [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
492 return std::array{
493 [&idx, &columnBuilders]() {
494 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
495 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template find<T>(idx);
496 }()...};
497 }(sq);
498 if constexpr (std::same_as<Kind, Sparse>) {
499 [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
500 ([&idx, &columnBuilders]() {
501 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
502 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template fill<T>(idx); }(), ...);
503 }(sq);
504 self.fill<C1>(counter);
505 } else if constexpr (std::same_as<Kind, Exclusive>) {
506 if (std::none_of(finds.begin(), finds.end(), [](bool const x) { return x == false; })) {
507 [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
508 ([&idx, &columnBuilders]() {
509 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
510 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template fill<T>(idx);
511 }(),
512 ...);
513 }(sq);
514 self.fill<C1>(counter);
515 }
516 }
517 }
518
519 return [&label, &columnBuilders, &self]<size_t... Is>(std::index_sequence<Is...>) {
520 return makeArrowTable(label,
521 {self.template result<C1>(), [&columnBuilders]() {
522 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
523 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template result<T>();
524 }()...},
525 {self.field(), [&columnBuilders]() {
526 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
527 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->field();
528 }()...});
529 }(sq);
530 }
531};
532
534
535template <soa::is_index_table T>
536constexpr auto transformBase()
537{
538 using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
540}
541
542template <soa::is_index_table T>
543struct Builds : decltype(transformBase<T>()) {
544 using buildable_t = T;
545 using metadata = decltype(transformBase<T>())::metadata;
546 using IP = std::conditional_t<metadata::exclusive, IndexBuilder<Exclusive>, IndexBuilder<Sparse>>;
547 using Key = metadata::Key;
548 using H = typename T::first_t;
549 using Ts = typename T::rest_t;
550 using index_pack_t = metadata::index_pack_t;
551
553 {
554 return table.get();
555 }
556 T const& operator*() const
557 {
558 return *table;
559 }
560
562 {
563 return table->asArrowTable();
564 }
565 std::shared_ptr<T> table = nullptr;
566
567 constexpr auto pack()
568 {
569 return index_pack_t{};
570 }
571
572 template <typename Key, typename... Cs>
573 auto build(framework::pack<Cs...>, std::vector<std::shared_ptr<arrow::Table>>&& tables)
574 {
575 this->table = std::make_shared<T>(IP::template indexBuilder<Key, metadata::sources.size(), metadata::sources>(o2::aod::label<T::ref>(), std::forward<std::vector<std::shared_ptr<arrow::Table>>>(tables), framework::pack<Cs...>{}));
576 return (this->table != nullptr);
577 }
578};
579
580template <typename T>
581concept is_builds = requires(T t) {
582 typename T::metadata;
583 typename T::Key;
584 requires std::same_as<decltype(t.pack()), typename T::index_pack_t>;
585};
586
594template <typename T>
595struct OutputObj {
596 using obj_t = T;
597
599 : object(std::make_shared<T>(t)),
600 label(t.GetName()),
601 policy{policy_},
602 sourceType{sourceType_},
603 mTaskHash{0}
604 {
605 }
606
608 : object(nullptr),
609 label(label_),
610 policy{policy_},
611 sourceType{sourceType_},
612 mTaskHash{0}
613 {
614 }
615
616 void setObject(T const& t)
617 {
618 object = std::make_shared<T>(t);
619 object->SetName(label.c_str());
620 }
621
622 void setObject(T&& t)
623 {
624 object = std::make_shared<T>(t);
625 object->SetName(label.c_str());
626 }
627
628 void setObject(T* t)
629 {
630 object.reset(t);
631 object->SetName(label.c_str());
632 }
633
634 void setObject(std::shared_ptr<T> t)
635 {
636 object = t;
637 object->SetName(label.c_str());
638 }
639
640 void setHash(uint32_t hash)
641 {
642 mTaskHash = hash;
643 }
644
647 {
649 auto lhash = runtime_hash(label.c_str());
650 std::memset(desc.str, '_', 16);
651 std::stringstream s;
652 s << std::hex << lhash;
653 s << std::hex << mTaskHash;
654 s << std::hex << reinterpret_cast<uint64_t>(this);
655 std::memcpy(desc.str, s.str().c_str(), 12);
656 return OutputSpec{OutputLabel{label}, "ATSK", desc, 0, Lifetime::QA};
657 }
658
660 {
661 return object.get();
662 }
663
665 {
666 return *object.get();
667 }
668
669 OutputRef ref(uint16_t index, uint16_t max)
670 {
671 return OutputRef{std::string{label}, 0,
673 }
674
675 std::shared_ptr<T> object;
676 std::string label;
679 uint32_t mTaskHash;
680};
681
682template <typename T>
683concept is_outputobj = requires(T t) {
684 &T::setHash;
685 &T::spec;
686 &T::ref;
687 requires std::same_as<decltype(t.operator->()), typename T::obj_t*>;
688 requires std::same_as<decltype(t.object), std::shared_ptr<typename T::obj_t>>;
689};
690
694template <typename T>
695struct Service {
696 using service_t = T;
698
699 decltype(auto) operator->() const
700 {
702 return service->get();
703 } else {
704 return service;
705 }
706 }
707};
708
709template <typename T>
710concept is_service = requires(T t) {
711 requires std::same_as<decltype(t.service), typename T::service_t*>;
712 &T::operator->;
713};
714
716{
717 return std::make_unique<o2::soa::Filtered<std::decay_t<decltype(table)>>>(std::vector{table}, std::forward<soa::SelectionVector>(selection));
718}
719
721{
722 return std::make_unique<o2::soa::Filtered<std::decay_t<decltype(table)>>>(std::vector{table.asArrowTable()}, std::forward<soa::SelectionVector>(selection));
723}
724
725void initializePartitionCaches(std::set<uint32_t> const& hashes, std::shared_ptr<arrow::Schema> const& schema, expressions::Filter const& filter, gandiva::NodePtr& tree, gandiva::FilterPtr& gfilter);
726
727template <typename T>
728struct Partition {
729 using content_t = T;
730 Partition(expressions::Node&& filter_) : filter{std::forward<expressions::Node>(filter_)}
731 {
732 }
733
734 Partition(expressions::Node&& filter_, T const& table)
735 : filter{std::forward<expressions::Node>(filter_)}
736 {
737 setTable(table);
738 }
739
740 void intializeCaches(std::set<uint32_t> const& hashes, std::shared_ptr<arrow::Schema> const& schema)
741 {
742 initializePartitionCaches(hashes, schema, filter, tree, gfilter);
743 }
744
745 void bindTable(T const& table)
746 {
747 intializeCaches(T::table_t::hashes(), table.asArrowTable()->schema());
748 if (dataframeChanged) {
750 dataframeChanged = false;
751 }
752 }
753
754 template <typename... Ts>
755 void bindExternalIndices(Ts*... tables)
756 {
757 if (mFiltered != nullptr) {
758 mFiltered->bindExternalIndices(tables...);
759 }
760 }
761
762 template <typename E>
764 {
765 if (mFiltered != nullptr) {
766 mFiltered->bindInternalIndicesTo(ptr);
767 }
768 }
769
771 {
773 }
774
775 [[nodiscard]] std::shared_ptr<arrow::Table> asArrowTable() const
776 {
777 return mFiltered->asArrowTable();
778 }
779
781 {
782 return mFiltered.get();
783 }
784
785 template <typename T1>
786 [[nodiscard]] auto rawSliceBy(o2::framework::Preslice<T1> const& container, int value) const
787 {
788 return mFiltered->rawSliceBy(container, value);
789 }
790
791 [[nodiscard]] auto sliceByCached(framework::expressions::BindingNode const& node, int value, o2::framework::SliceCache& cache) const
792 {
793 return mFiltered->sliceByCached(node, value, cache);
794 }
795
797 {
798 return mFiltered->sliceByCachedUnsorted(node, value, cache);
799 }
800
801 template <typename T1, typename Policy, bool OPT>
802 [[nodiscard]] auto sliceBy(o2::framework::PresliceBase<T1, Policy, OPT> const& container, int value) const
803 {
804 return mFiltered->sliceBy(container, value);
805 }
806
808 std::unique_ptr<o2::soa::Filtered<T>> mFiltered = nullptr;
809 gandiva::NodePtr tree = nullptr;
811 bool dataframeChanged = true;
812
818 {
819 return mFiltered->begin();
820 }
822 {
823 return mFiltered->end();
824 }
826 {
827 return mFiltered->begin();
828 }
830 {
831 return mFiltered->end();
832 }
833
834 int64_t size() const
835 {
836 return mFiltered->size();
837 }
838};
839
840template <typename T>
841concept is_partition = requires(T t) {
842 &T::updatePlaceholders;
843 requires std::same_as<decltype(t.filter), expressions::Filter>;
844 requires std::same_as<decltype(t.mFiltered), std::unique_ptr<o2::soa::Filtered<typename T::content_t>>>;
845};
846} // namespace o2::framework
847
848namespace o2::soa
849{
851template <soa::is_table T, soa::is_spawnable_column... Cs>
852auto Extend(T const& table)
853{
854 using output_t = Join<T, soa::Table<o2::aod::Hash<"JOIN"_h>, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Cs...>>;
855 static std::array<framework::expressions::Projector, sizeof...(Cs)> projectors{{std::move(Cs::Projector())...}};
856 static std::shared_ptr<gandiva::Projector> projector = nullptr;
857 static auto schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(framework::pack<Cs...>{}));
858 return output_t{{o2::framework::spawner(framework::pack<Cs...>{}, {table.asArrowTable()}, "dynamicExtension", projectors.data(), projector, schema), table.asArrowTable()}, 0};
859}
860
863template <soa::is_table T, soa::is_dynamic_column... Cs>
864auto Attach(T const& table)
865{
866 using output_t = Join<T, o2::soa::Table<o2::aod::Hash<"JOIN"_h>, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Cs...>>;
867 return output_t{{table.asArrowTable()}, table.offset()};
868}
869} // namespace o2::soa
870
871#endif // o2_framework_AnalysisHelpers_H_DEFINED
uint16_t pos
Definition RawData.h:3
uint32_t gfilter
Definition RawData.h:6
constexpr uint32_t runtime_hash(char const *str)
TBranch * ptr
Definition A.h:16
iterator const_iterator
Definition ASoA.h:3622
T::template iterator_template_o< FilteredIndexPolicy, self_t > iterator
Definition ASoA.h:3620
Helper to check if a type T is an iterator.
Definition ASoA.h:1265
GLint GLenum GLint x
Definition glcorearb.h:403
GLuint64EXT * result
Definition glcorearb.h:5662
GLsizeiptr size
Definition glcorearb.h:659
GLuint index
Definition glcorearb.h:781
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei const GLfloat * value
Definition glcorearb.h:819
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
GLuint GLsizei const GLchar * label
Definition glcorearb.h:2519
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition glcorearb.h:1308
GLuint object
Definition glcorearb.h:4041
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
GLsizei GLenum * sources
Definition glcorearb.h:2516
GLuint counter
Definition glcorearb.h:3987
std::shared_ptr< gandiva::Filter > FilterPtr
Definition Expressions.h:47
std::variant< OriginValueMatcher, DescriptionValueMatcher, SubSpecificationTypeValueMatcher, std::unique_ptr< DataDescriptorMatcher >, ConstantValueMatcher, StartTimeValueMatcher > Node
std::shared_ptr< gandiva::Projector > createProjectorHelper(size_t nColumns, expressions::Projector *projectors, std::shared_ptr< arrow::Schema > schema, std::vector< std::shared_ptr< arrow::Field > > const &fields)
gandiva::Selection createSelection(std::shared_ptr< arrow::Table > const &table, Filter const &expression)
Function for creating gandiva selection from our internal filter tree.
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
auto spawner(std::shared_ptr< arrow::Table > const &fullTable, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector, std::shared_ptr< arrow::Schema > const &schema)
Expression-based column generator to materialize columns.
typename pack_element< I, T >::type pack_element_t
Definition Pack.h:56
constexpr std::size_t pack_size(pack< Ts... > const &)
template function to determine number of types in a pack
Definition Pack.h:28
constexpr auto transformBase()
This helper struct allows you to declare index tables to be created in a task.
memfun_type< decltype(&F::operator())>::type FFL(F const &func)
auto getTableFromFilter(soa::is_filtered_table auto const &table, soa::SelectionVector &&selection)
consteval auto typeWithRef() -> T
Helper to define output for a Table.
OutputObjHandlingPolicy
Policy enum to determine OutputObj handling when writing.
void initializePartitionCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema, expressions::Filter const &filter, gandiva::NodePtr &tree, gandiva::FilterPtr &gfilter)
std::shared_ptr< arrow::Table > makeArrowTable(const char *label, std::vector< std::shared_ptr< arrow::ChunkedArray > > &&columns, std::vector< std::shared_ptr< arrow::Field > > &&fields)
auto createFieldsFromColumns(framework::pack< C... >)
Definition ASoA.h:407
SelectionVector selectionToVector(gandiva::Selection const &sel)
Definition ASoA.cxx:43
constexpr auto tableRef2InputSpec()
constexpr auto tableRef2Output()
std::vector< int64_t > SelectionVector
Definition ASoA.h:412
auto Attach(T const &table)
constexpr auto tableRef2ConfigParamSpec()
auto Extend(T const &table)
On-the-fly adding of expression columns.
constexpr auto tableRef2OutputSpec()
constexpr auto tableRef2OutputRef()
@ C
Definition Defs.h:36
Defining DataPointCompositeObject explicitly as copiable.
std::shared_ptr< T > table
constexpr auto pack()
typename T::first_t H
T const & operator*() const
auto build(framework::pack< Cs... >, std::vector< std::shared_ptr< arrow::Table > > &&tables)
metadata::index_pack_t index_pack_t
typename T::rest_t Ts
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
T::table_t const & operator*() const
decltype(transformBase< T >())::metadata metadata
typename metadata::extension_table_t extension_t
std::array< o2::framework::expressions::Projector, N > projectors
std::shared_ptr< extension_t > extension
std::shared_ptr< arrow::Schema > inputSchema
std::shared_ptr< arrow::Schema > schema
typename metadata::base_table_t base_table_t
static constexpr bool delayed
static constexpr size_t N
typename metadata::placeholders_pack_t placeholders_pack_t
std::shared_ptr< gandiva::Projector > projector
static auto indexBuilder(const char *label, std::vector< std::shared_ptr< arrow::Table > > &&tables, framework::pack< C1, Cs... >)
static OutputSpec const spec()
aod::MetadataTrait< o2::aod::Hash< table_t::ref.desc_hash > >::metadata metadata
decltype(typeWithRef< T >()) table_t
O2 header for OutputObj metadata.
OutputObj(T &&t, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
void setObject(std::shared_ptr< T > t)
OutputObjSourceType sourceType
OutputRef ref(uint16_t index, uint16_t max)
std::shared_ptr< T > object
OutputObj(std::string const &label_, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
OutputSpec const spec()
OutputObjHandlingPolicy policy
void setHash(uint32_t hash)
Partition(expressions::Node &&filter_, T const &table)
typename o2::soa::Filtered< T >::const_iterator filtered_const_iterator
void bindTable(T const &table)
filtered_const_iterator begin() const
auto rawSliceBy(o2::framework::Preslice< T1 > const &container, int value) const
auto sliceByCached(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
void intializeCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema)
Partition(expressions::Node &&filter_)
typename o2::soa::Filtered< T >::iterator filtered_iterator
o2::soa::Filtered< T > * operator->()
typename o2::soa::Filtered< T >::iterator iterator
auto sliceBy(o2::framework::PresliceBase< T1, Policy, OPT > const &container, int value) const
std::shared_ptr< arrow::Table > asArrowTable() const
o2::soa::RowViewSentinel end() const
void updatePlaceholders(InitContext &context)
std::unique_ptr< o2::soa::Filtered< T > > mFiltered
expressions::Filter filter
o2::soa::RowViewSentinel end()
auto sliceByCachedUnsorted(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
typename o2::soa::Filtered< T >::const_iterator const_iterator
gandiva::FilterPtr gfilter
void bindExternalIndices(Ts *... tables)
filtered_iterator begin()
T::table_t * operator->()
std::shared_ptr< gandiva::Projector > projector
std::shared_ptr< extension_t > extension
T::table_t const & operator*() const
static constexpr size_t N
typename metadata::base_table_t base_table_t
std::shared_ptr< arrow::Schema > schema
typename metadata::expression_pack_t expression_pack_t
constexpr auto pack()
std::array< o2::framework::expressions::Projector, N > projectors
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
typename metadata::extension_table_t extension_t
Helper template for table transformations.
constexpr auto spec() const
constexpr auto output() const
static constexpr auto base_spec()
constexpr auto ref() const
int64_t lastIndex()
Last index inserted in the table.
decltype(FFL(std::declval< cursor_t >())) cursor
bool resetCursor(LifetimeHolder< TableBuilder > builder)
void setLabel(const char *label)
void operator()(Ts &&... args)
decltype([]() { if constexpr(soa::is_iterator< T >) { return typename T::parent_t{nullptr} persistent_table_t
decltype(std::declval< TableBuilder >().cursor< persistent_table_t >()) cursor_t
An expression tree node corresponding to a column binding.
A struct, containing the root of the expression tree.
a move-only header stack with serialized headers This is the flat buffer where all the headers in a m...
Definition Stack.h:33
constexpr size_t max
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))