Project
Loading...
Searching...
No Matches
AnalysisHelpers.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11#ifndef o2_framework_AnalysisHelpers_H_DEFINED
12#define o2_framework_AnalysisHelpers_H_DEFINED
13
14#include "ConfigParamSpec.h"
15#include "Framework/ASoA.h"
18#include "Framework/InputSpec.h"
19#include "Framework/Output.h"
21#include "Framework/OutputRef.h"
23#include "Framework/Plugins.h"
26#include "Framework/Traits.h"
27
28#include <string>
29namespace o2::soa
30{
31template <TableRef R>
33{
35 std::string{"input:"} + o2::aod::label<R>(),
37 aod::sourceSpec<R>(),
38 {"\"\""}};
39}
40
41namespace
42{
43template <soa::with_sources T>
44inline constexpr auto getSources()
45{
46 return []<size_t N, std::array<soa::TableRef, N> refs>() {
47 return []<size_t... Is>(std::index_sequence<Is...>) {
48 return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
49 }(std::make_index_sequence<N>());
50 }.template operator()<T::sources.size(), T::sources>();
51}
52
53template <soa::with_ccdb_urls T>
54inline constexpr auto getCCDBUrls()
55{
56 std::vector<framework::ConfigParamSpec> result;
57 for (size_t i = 0; i < T::ccdb_urls.size(); ++i) {
58 result.push_back({std::string{"ccdb:"} + std::string{T::ccdb_bindings[i]},
60 T::ccdb_urls[i],
61 {"\"\""}});
62 }
63 return result;
64}
65
66template <soa::with_sources T>
67constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
68{
69 std::vector<framework::ConfigParamSpec> inputMetadata;
70 auto inputSources = getSources<T>();
71 std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
72 auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
73 inputSources.erase(last, inputSources.end());
74 inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());
75 return inputMetadata;
76}
77
78template <typename T>
79 requires(!soa::with_sources<T>)
80constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
81{
82 return {};
83}
84
85template <soa::with_ccdb_urls T>
86constexpr auto getCCDBMetadata() -> std::vector<framework::ConfigParamSpec>
87{
88 std::vector<framework::ConfigParamSpec> results = getCCDBUrls<T>();
89 std::sort(results.begin(), results.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
90 auto last = std::unique(results.begin(), results.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
91 results.erase(last, results.end());
92 return results;
93}
94
95template <typename T>
96constexpr auto getCCDBMetadata() -> std::vector<framework::ConfigParamSpec>
97{
98 return {};
99}
100} // namespace
101
102template <TableRef R>
103constexpr auto tableRef2InputSpec()
104{
105 std::vector<framework::ConfigParamSpec> metadata;
106 auto m = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
107 metadata.insert(metadata.end(), m.begin(), m.end());
108 auto ccdbMetadata = getCCDBMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
109 metadata.insert(metadata.end(), ccdbMetadata.begin(), ccdbMetadata.end());
110
112 o2::aod::label<R>(),
113 o2::aod::origin<R>(),
114 o2::aod::description(o2::aod::signature<R>()),
115 R.version,
116 framework::Lifetime::Timeframe,
117 metadata};
118}
119
120template <TableRef R>
121constexpr auto tableRef2OutputSpec()
122{
124 framework::OutputLabel{o2::aod::label<R>()},
125 o2::aod::origin<R>(),
126 o2::aod::description(o2::aod::signature<R>()),
127 R.version};
128}
129
130template <TableRef R>
131constexpr auto tableRef2Output()
132{
133 return framework::Output{
134 o2::aod::origin<R>(),
135 o2::aod::description(o2::aod::signature<R>()),
136 R.version};
137}
138
139template <TableRef R>
140constexpr auto tableRef2OutputRef()
141{
143 o2::aod::label<R>(),
144 R.version};
145}
146} // namespace o2::soa
147
148namespace o2::framework
149{
150class TableConsumer;
151
155template <typename T>
157
158template <typename T>
159concept is_enumerated_iterator = requires(T t) { t.globalIndex(); };
160
161template <is_producable T>
163 public:
164 using persistent_table_t = decltype([]() { if constexpr (soa::is_iterator<T>) { return typename T::parent_t{nullptr}; } else { return T{nullptr}; } }());
165 using cursor_t = decltype(std::declval<TableBuilder>().cursor<persistent_table_t>());
166
167 template <typename... Ts>
168 void operator()(Ts&&... args)
169 requires(sizeof...(Ts) == framework::pack_size(typename persistent_table_t::persistent_columns_t{}))
170 {
171 ++mCount;
172 cursor(0, extract(args)...);
173 }
174
176 int64_t lastIndex()
177 {
178 return mCount;
179 }
180
182 {
183 mBuilder = std::move(builder);
184 cursor = std::move(FFL(mBuilder->cursor<persistent_table_t>()));
185 mCount = -1;
186 return true;
187 }
188
189 void setLabel(const char* label)
190 {
191 mBuilder->setLabel(label);
192 }
193
196 void reserve(int64_t size)
197 {
198 mBuilder->reserve(typename persistent_table_t::column_types{}, size);
199 }
200
201 void release()
202 {
203 mBuilder.release();
204 }
205
206 decltype(FFL(std::declval<cursor_t>())) cursor;
207
208 private:
209 static decltype(auto) extract(is_enumerated_iterator auto const& arg)
210 {
211 return arg.globalIndex();
212 }
213
214 template <typename A>
216 static decltype(auto) extract(A&& arg)
217 {
218 return arg;
219 }
220
224 LifetimeHolder<TableBuilder> mBuilder = nullptr;
225 int64_t mCount = -1;
226};
227
229template <soa::is_table T>
230consteval auto typeWithRef() -> T
231{
232}
233
234template <soa::is_iterator T>
235consteval auto typeWithRef() -> typename T::parent_t
236{
237}
238
239template <typename T>
240 requires soa::is_table<T> || soa::is_iterator<T>
242 using table_t = decltype(typeWithRef<T>());
243 using metadata = aod::MetadataTrait<o2::aod::Hash<table_t::ref.desc_hash>>::metadata;
244
245 static OutputSpec const spec()
246 {
247 return OutputSpec{OutputLabel{aod::label<table_t::ref>()}, o2::aod::origin<table_t::ref>(), o2::aod::description(o2::aod::signature<table_t::ref>()), table_t::ref.version};
248 }
249
250 static OutputRef ref()
251 {
252 return OutputRef{aod::label<table_t::ref>(), table_t::ref.version};
253 }
254};
255
260template <is_producable T>
262};
263
264template <typename T>
265concept is_produces = requires(T t) { typename T::cursor_t; typename T::persistent_table_t; &T::cursor; };
266
276};
277
278template <typename T>
279concept is_produces_group = std::derived_from<T, ProducesGroup>;
280
282template <soa::is_metadata M, soa::TableRef Ref>
284 using metadata = M;
285 constexpr static auto sources = M::sources;
286
287 template <soa::TableRef R>
288 static constexpr auto base_spec()
289 {
290 return soa::tableRef2InputSpec<R>();
291 }
292
293 static auto base_specs()
294 {
295 return []<size_t... Is>(std::index_sequence<Is...>) -> std::vector<InputSpec> {
296 return {base_spec<sources[Is]>()...};
297 }(std::make_index_sequence<sources.size()>{});
298 }
299
300 constexpr auto spec() const
301 {
302 return soa::tableRef2OutputSpec<Ref>();
303 }
304
305 constexpr auto output() const
306 {
307 return soa::tableRef2Output<Ref>();
308 }
309
310 constexpr auto ref() const
311 {
312 return soa::tableRef2OutputRef<Ref>();
313 }
314};
315
318template <typename T>
319concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
320
321template <typename T>
322concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
323
324template <is_spawnable T>
325constexpr auto transformBase()
326{
327 using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
329}
330
331template <is_spawnable T>
332struct Spawns : decltype(transformBase<T>()) {
333 using spawnable_t = T;
334 using metadata = decltype(transformBase<T>())::metadata;
335 using extension_t = typename metadata::extension_table_t;
336 using base_table_t = typename metadata::base_table_t;
337 using expression_pack_t = typename metadata::expression_pack_t;
338 static constexpr size_t N = framework::pack_size(expression_pack_t{});
339
340 constexpr auto pack()
341 {
342 return expression_pack_t{};
343 }
344
345 typename T::table_t* operator->()
346 {
347 return table.get();
348 }
349 typename T::table_t const& operator*() const
350 {
351 return *table;
352 }
353
355 {
356 return extension->asArrowTable();
357 }
358 std::shared_ptr<typename T::table_t> table = nullptr;
359 std::shared_ptr<extension_t> extension = nullptr;
360 std::array<o2::framework::expressions::Projector, N> projectors = []<typename... C>(framework::pack<C...>) -> std::array<expressions::Projector, sizeof...(C)>
361 {
362 return {{std::move(C::Projector())...}};
363 }
365 std::shared_ptr<gandiva::Projector> projector = nullptr;
366 std::shared_ptr<arrow::Schema> schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(expression_pack_t{}));
367};
368
369template <typename T>
370concept is_spawns = requires(T t) {
371 typename T::metadata;
372 requires std::same_as<decltype(t.pack()), typename T::expression_pack_t>;
373 requires std::same_as<decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
374};
375
380
381template <is_dynamically_spawnable T, bool DELAYED = false>
382struct Defines : decltype(transformBase<T>()) {
383 static constexpr bool delayed = DELAYED;
384 using spawnable_t = T;
385 using metadata = decltype(transformBase<T>())::metadata;
386 using extension_t = typename metadata::extension_table_t;
387 using base_table_t = typename metadata::base_table_t;
388 using placeholders_pack_t = typename metadata::placeholders_pack_t;
389 static constexpr size_t N = framework::pack_size(placeholders_pack_t{});
390
391 constexpr auto pack()
392 {
393 return placeholders_pack_t{};
394 }
395
396 typename T::table_t* operator->()
397 {
398 return table.get();
399 }
400 typename T::table_t const& operator*() const
401 {
402 return *table;
403 }
404
406 {
407 return extension->asArrowTable();
408 }
409 std::shared_ptr<typename T::table_t> table = nullptr;
410 std::shared_ptr<extension_t> extension = nullptr;
411
412 std::array<o2::framework::expressions::Projector, N> projectors;
413 std::shared_ptr<gandiva::Projector> projector = nullptr;
414 std::shared_ptr<arrow::Schema> schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(placeholders_pack_t{}));
415 std::shared_ptr<arrow::Schema> inputSchema = nullptr;
416
417 bool needRecompilation = false;
418
423};
424
425template <is_dynamically_spawnable T>
427
428template <typename T>
429concept is_defines = requires(T t) {
430 typename T::metadata;
431 requires std::same_as<decltype(t.pack()), typename T::placeholders_pack_t>;
432 requires std::same_as<decltype(t.projector), std::shared_ptr<gandiva::Projector>>;
433 requires std::same_as<decltype(t.needRecompilation), bool>;
434 &T::recompile;
435};
436
441struct Exclusive {
442};
443struct Sparse {
444};
445
446namespace
447{
448template <typename T, typename Key>
449inline std::shared_ptr<arrow::ChunkedArray> getIndexToKey(arrow::Table* table)
450{
451 using IC = framework::pack_element_t<framework::has_type_at_conditional_v<soa::is_binding_compatible, Key>(typename T::external_index_columns_t{}), typename T::external_index_columns_t>;
452 return table->column(framework::has_type_at_v<IC>(typename T::persistent_columns_t{}));
453}
454
455template <soa::is_column C>
456struct ColumnTrait {
457 using column_t = C;
458
459 static consteval auto listSize()
460 {
461 if constexpr (std::same_as<typename C::type, std::vector<int>>) {
462 return -1;
463 } else if constexpr (std::same_as<int[2], typename C::type>) {
464 return 2;
465 } else {
466 return 1;
467 }
468 }
469
470 template <typename T, typename Key>
471 static std::shared_ptr<SelfIndexColumnBuilder> makeColumnBuilder(arrow::Table* table, arrow::MemoryPool* pool)
472 {
473 if constexpr (!std::same_as<T, Key>) {
474 return std::make_shared<IndexColumnBuilder>(getIndexToKey<T, Key>(table), C::columnLabel(), listSize(), pool);
475 } else {
476 return std::make_shared<SelfIndexColumnBuilder>(C::columnLabel(), pool);
477 }
478 }
479};
480
481template <typename Key, typename C>
482struct Reduction {
483 using type = typename std::conditional<soa::is_binding_compatible_v<Key, typename C::binding_t>(), SelfIndexColumnBuilder, IndexColumnBuilder>::type;
484};
485
486template <typename Key, typename C>
487using reduced_t = Reduction<Key, C>::type;
488} // namespace
489
490template <typename Kind>
492 template <typename Key, size_t N, std::array<soa::TableRef, N> refs, typename C1, typename... Cs>
493 static auto indexBuilder(const char* label, std::vector<std::shared_ptr<arrow::Table>>&& tables, framework::pack<C1, Cs...>)
494 {
495 auto pool = arrow::default_memory_pool();
496 SelfIndexColumnBuilder self{C1::columnLabel(), pool};
497 std::unique_ptr<ChunkedArrayIterator> keyIndex = nullptr;
498 if constexpr (!Key::template hasOriginal<refs[0]>()) {
499 keyIndex = std::make_unique<ChunkedArrayIterator>(tables[0]->column(o2::aod::MetadataTrait<o2::aod::Hash<refs[0].desc_hash>>::metadata::template getIndexPosToKey<Key>()));
500 }
501
502 auto sq = std::make_index_sequence<sizeof...(Cs)>();
503
504 auto columnBuilders = [&tables, &pool ]<size_t... Is>(std::index_sequence<Is...>) -> std::array<std::shared_ptr<framework::SelfIndexColumnBuilder>, sizeof...(Cs)>
505 {
506 return {[](arrow::Table* table, arrow::MemoryPool* pool) {
507 using T = framework::pack_element_t<Is, framework::pack<Cs...>>;
508 if constexpr (!Key::template hasOriginal<refs[Is + 1]>()) {
509 constexpr auto pos = o2::aod::MetadataTrait<o2::aod::Hash<refs[Is + 1].desc_hash>>::metadata::template getIndexPosToKey<Key>();
510 return std::make_shared<IndexColumnBuilder>(table->column(pos), T::columnLabel(), ColumnTrait<T>::listSize(), pool);
511 } else {
512 return std::make_shared<SelfIndexColumnBuilder>(T::columnLabel(), pool);
513 }
514 }(tables[Is + 1].get(), pool)...};
515 }
516 (sq);
517
518 std::array<bool, sizeof...(Cs)> finds;
519
520 for (int64_t counter = 0; counter < tables[0]->num_rows(); ++counter) {
521 int64_t idx = -1;
522 if constexpr (Key::template hasOriginal<refs[0]>()) {
523 idx = counter;
524 } else {
525 idx = keyIndex->valueAt(counter);
526 }
527 finds = [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
528 return std::array{
529 [&idx, &columnBuilders]() {
530 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
531 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template find<T>(idx);
532 }()...};
533 }(sq);
534 if constexpr (std::same_as<Kind, Sparse>) {
535 [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
536 ([&idx, &columnBuilders]() {
537 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
538 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template fill<T>(idx); }(), ...);
539 }(sq);
540 self.fill<C1>(counter);
541 } else if constexpr (std::same_as<Kind, Exclusive>) {
542 if (std::none_of(finds.begin(), finds.end(), [](bool const x) { return x == false; })) {
543 [&idx, &columnBuilders]<size_t... Is>(std::index_sequence<Is...>) {
544 ([&idx, &columnBuilders]() {
545 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
546 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template fill<T>(idx);
547 }(),
548 ...);
549 }(sq);
550 self.fill<C1>(counter);
551 }
552 }
553 }
554
555 return [&label, &columnBuilders, &self]<size_t... Is>(std::index_sequence<Is...>) {
556 return makeArrowTable(label,
557 {self.template result<C1>(), [&columnBuilders]() {
558 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
559 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->template result<T>();
560 }()...},
561 {self.field(), [&columnBuilders]() {
562 using T = typename framework::pack_element_t<Is, framework::pack<Cs...>>;
563 return std::static_pointer_cast<reduced_t<Key, T>>(columnBuilders[Is])->field();
564 }()...});
565 }(sq);
566 }
567};
568
570
571template <soa::is_index_table T>
572constexpr auto transformBase()
573{
574 using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
576}
577
578template <soa::is_index_table T>
579struct Builds : decltype(transformBase<T>()) {
580 using buildable_t = T;
581 using metadata = decltype(transformBase<T>())::metadata;
582 using IP = std::conditional_t<metadata::exclusive, IndexBuilder<Exclusive>, IndexBuilder<Sparse>>;
583 using Key = metadata::Key;
584 using H = typename T::first_t;
585 using Ts = typename T::rest_t;
586 using index_pack_t = metadata::index_pack_t;
587
589 {
590 return table.get();
591 }
592 T const& operator*() const
593 {
594 return *table;
595 }
596
598 {
599 return table->asArrowTable();
600 }
601 std::shared_ptr<T> table = nullptr;
602
603 constexpr auto pack()
604 {
605 return index_pack_t{};
606 }
607
608 template <typename Key, typename... Cs>
609 auto build(framework::pack<Cs...>, std::vector<std::shared_ptr<arrow::Table>>&& tables)
610 {
611 this->table = std::make_shared<T>(IP::template indexBuilder<Key, metadata::sources.size(), metadata::sources>(o2::aod::label<T::ref>(), std::forward<std::vector<std::shared_ptr<arrow::Table>>>(tables), framework::pack<Cs...>{}));
612 return (this->table != nullptr);
613 }
614};
615
616template <typename T>
617concept is_builds = requires(T t) {
618 typename T::metadata;
619 typename T::Key;
620 requires std::same_as<decltype(t.pack()), typename T::index_pack_t>;
621};
622
630template <typename T>
631struct OutputObj {
632 using obj_t = T;
633
635 : object(std::make_shared<T>(t)),
636 label(t.GetName()),
637 policy{policy_},
638 sourceType{sourceType_},
639 mTaskHash{0}
640 {
641 }
642
644 : object(nullptr),
645 label(label_),
646 policy{policy_},
647 sourceType{sourceType_},
648 mTaskHash{0}
649 {
650 }
651
652 void setObject(T const& t)
653 {
654 object = std::make_shared<T>(t);
655 object->SetName(label.c_str());
656 }
657
658 void setObject(T&& t)
659 {
660 object = std::make_shared<T>(t);
661 object->SetName(label.c_str());
662 }
663
664 void setObject(T* t)
665 {
666 object.reset(t);
667 object->SetName(label.c_str());
668 }
669
670 void setObject(std::shared_ptr<T> t)
671 {
672 object = t;
673 object->SetName(label.c_str());
674 }
675
676 void setHash(uint32_t hash)
677 {
678 mTaskHash = hash;
679 }
680
683 {
685 auto lhash = runtime_hash(label.c_str());
686 std::memset(desc.str, '_', 16);
687 std::stringstream s;
688 s << std::hex << lhash;
689 s << std::hex << mTaskHash;
690 s << std::hex << reinterpret_cast<uint64_t>(this);
691 std::memcpy(desc.str, s.str().c_str(), 12);
692 return OutputSpec{OutputLabel{label}, "ATSK", desc, 0, Lifetime::QA};
693 }
694
696 {
697 return object.get();
698 }
699
701 {
702 return *object.get();
703 }
704
705 OutputRef ref(uint16_t index, uint16_t max)
706 {
707 return OutputRef{std::string{label}, 0,
709 }
710
711 std::shared_ptr<T> object;
712 std::string label;
715 uint32_t mTaskHash;
716};
717
718template <typename T>
719concept is_outputobj = requires(T t) {
720 &T::setHash;
721 &T::spec;
722 &T::ref;
723 requires std::same_as<decltype(t.operator->()), typename T::obj_t*>;
724 requires std::same_as<decltype(t.object), std::shared_ptr<typename T::obj_t>>;
725};
726
730template <typename T>
731struct Service {
732 using service_t = T;
734
735 decltype(auto) operator->() const
736 {
738 return service->get();
739 } else {
740 return service;
741 }
742 }
743};
744
745template <typename T>
746concept is_service = requires(T t) {
747 requires std::same_as<decltype(t.service), typename T::service_t*>;
748 &T::operator->;
749};
750
752{
753 return std::make_unique<o2::soa::Filtered<std::decay_t<decltype(table)>>>(std::vector{table}, std::forward<soa::SelectionVector>(selection));
754}
755
757{
758 return std::make_unique<o2::soa::Filtered<std::decay_t<decltype(table)>>>(std::vector{table.asArrowTable()}, std::forward<soa::SelectionVector>(selection));
759}
760
761void initializePartitionCaches(std::set<uint32_t> const& hashes, std::shared_ptr<arrow::Schema> const& schema, expressions::Filter const& filter, gandiva::NodePtr& tree, gandiva::FilterPtr& gfilter);
762
763template <typename T>
764struct Partition {
765 using content_t = T;
766 Partition(expressions::Node&& filter_) : filter{std::forward<expressions::Node>(filter_)}
767 {
768 }
769
770 Partition(expressions::Node&& filter_, T const& table)
771 : filter{std::forward<expressions::Node>(filter_)}
772 {
773 setTable(table);
774 }
775
776 void intializeCaches(std::set<uint32_t> const& hashes, std::shared_ptr<arrow::Schema> const& schema)
777 {
778 initializePartitionCaches(hashes, schema, filter, tree, gfilter);
779 }
780
781 void bindTable(T const& table)
782 {
783 intializeCaches(T::table_t::hashes(), table.asArrowTable()->schema());
784 if (dataframeChanged) {
786 dataframeChanged = false;
787 }
788 }
789
790 template <typename... Ts>
791 void bindExternalIndices(Ts*... tables)
792 {
793 if (mFiltered != nullptr) {
794 mFiltered->bindExternalIndices(tables...);
795 }
796 }
797
798 template <typename E>
800 {
801 if (mFiltered != nullptr) {
802 mFiltered->bindInternalIndicesTo(ptr);
803 }
804 }
805
807 {
809 }
810
811 [[nodiscard]] std::shared_ptr<arrow::Table> asArrowTable() const
812 {
813 return mFiltered->asArrowTable();
814 }
815
817 {
818 return mFiltered.get();
819 }
820
821 template <typename T1>
822 [[nodiscard]] auto rawSliceBy(o2::framework::Preslice<T1> const& container, int value) const
823 {
824 return mFiltered->rawSliceBy(container, value);
825 }
826
827 [[nodiscard]] auto sliceByCached(framework::expressions::BindingNode const& node, int value, o2::framework::SliceCache& cache) const
828 {
829 return mFiltered->sliceByCached(node, value, cache);
830 }
831
833 {
834 return mFiltered->sliceByCachedUnsorted(node, value, cache);
835 }
836
837 template <typename T1, typename Policy, bool OPT>
838 [[nodiscard]] auto sliceBy(o2::framework::PresliceBase<T1, Policy, OPT> const& container, int value) const
839 {
840 return mFiltered->sliceBy(container, value);
841 }
842
844 std::unique_ptr<o2::soa::Filtered<T>> mFiltered = nullptr;
845 gandiva::NodePtr tree = nullptr;
847 bool dataframeChanged = true;
848
854 {
855 return mFiltered->begin();
856 }
858 {
859 return mFiltered->end();
860 }
862 {
863 return mFiltered->begin();
864 }
866 {
867 return mFiltered->end();
868 }
869
870 int64_t size() const
871 {
872 return mFiltered->size();
873 }
874};
875
876template <typename T>
877concept is_partition = requires(T t) {
878 &T::updatePlaceholders;
879 requires std::same_as<decltype(t.filter), expressions::Filter>;
880 requires std::same_as<decltype(t.mFiltered), std::unique_ptr<o2::soa::Filtered<typename T::content_t>>>;
881};
882} // namespace o2::framework
883
884namespace o2::soa
885{
887template <soa::is_table T, soa::is_spawnable_column... Cs>
888auto Extend(T const& table)
889{
890 using output_t = Join<T, soa::Table<o2::aod::Hash<"JOIN"_h>, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Cs...>>;
891 static std::array<framework::expressions::Projector, sizeof...(Cs)> projectors{{std::move(Cs::Projector())...}};
892 static std::shared_ptr<gandiva::Projector> projector = nullptr;
893 static auto schema = std::make_shared<arrow::Schema>(o2::soa::createFieldsFromColumns(framework::pack<Cs...>{}));
894 return output_t{{o2::framework::spawner(framework::pack<Cs...>{}, {table.asArrowTable()}, "dynamicExtension", projectors.data(), projector, schema), table.asArrowTable()}, 0};
895}
896
899template <soa::is_table T, soa::is_dynamic_column... Cs>
900auto Attach(T const& table)
901{
902 using output_t = Join<T, o2::soa::Table<o2::aod::Hash<"JOIN"_h>, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Cs...>>;
903 return output_t{{table.asArrowTable()}, table.offset()};
904}
905} // namespace o2::soa
906
907#endif // o2_framework_AnalysisHelpers_H_DEFINED
int32_t i
uint16_t pos
Definition RawData.h:3
uint32_t gfilter
Definition RawData.h:6
constexpr uint32_t runtime_hash(char const *str)
TBranch * ptr
Definition A.h:16
iterator const_iterator
Definition ASoA.h:3742
T::template iterator_template_o< FilteredIndexPolicy, self_t > iterator
Definition ASoA.h:3740
Helper to check if a type T is an iterator.
Definition ASoA.h:1278
GLint GLenum GLint x
Definition glcorearb.h:403
const GLfloat * m
Definition glcorearb.h:4066
GLuint64EXT * result
Definition glcorearb.h:5662
GLsizeiptr size
Definition glcorearb.h:659
GLuint index
Definition glcorearb.h:781
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei const GLfloat * value
Definition glcorearb.h:819
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
GLuint GLsizei const GLchar * label
Definition glcorearb.h:2519
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition glcorearb.h:1308
GLuint object
Definition glcorearb.h:4041
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
GLsizei GLenum * sources
Definition glcorearb.h:2516
GLuint counter
Definition glcorearb.h:3987
std::shared_ptr< gandiva::Filter > FilterPtr
Definition Expressions.h:47
std::variant< OriginValueMatcher, DescriptionValueMatcher, SubSpecificationTypeValueMatcher, std::unique_ptr< DataDescriptorMatcher >, ConstantValueMatcher, StartTimeValueMatcher > Node
std::shared_ptr< gandiva::Projector > createProjectorHelper(size_t nColumns, expressions::Projector *projectors, std::shared_ptr< arrow::Schema > schema, std::vector< std::shared_ptr< arrow::Field > > const &fields)
gandiva::Selection createSelection(std::shared_ptr< arrow::Table > const &table, Filter const &expression)
Function for creating gandiva selection from our internal filter tree.
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
auto spawner(std::shared_ptr< arrow::Table > const &fullTable, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector, std::shared_ptr< arrow::Schema > const &schema)
Expression-based column generator to materialize columns.
typename pack_element< I, T >::type pack_element_t
Definition Pack.h:56
constexpr std::size_t pack_size(pack< Ts... > const &)
template function to determine number of types in a pack
Definition Pack.h:28
constexpr auto transformBase()
This helper struct allows you to declare index tables to be created in a task.
memfun_type< decltype(&F::operator())>::type FFL(F const &func)
auto getTableFromFilter(soa::is_filtered_table auto const &table, soa::SelectionVector &&selection)
consteval auto typeWithRef() -> T
Helper to define output for a Table.
OutputObjHandlingPolicy
Policy enum to determine OutputObj handling when writing.
void initializePartitionCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema, expressions::Filter const &filter, gandiva::NodePtr &tree, gandiva::FilterPtr &gfilter)
std::shared_ptr< arrow::Table > makeArrowTable(const char *label, std::vector< std::shared_ptr< arrow::ChunkedArray > > &&columns, std::vector< std::shared_ptr< arrow::Field > > &&fields)
auto createFieldsFromColumns(framework::pack< C... >)
Definition ASoA.h:410
SelectionVector selectionToVector(gandiva::Selection const &sel)
Definition ASoA.cxx:48
constexpr auto tableRef2InputSpec()
constexpr auto tableRef2Output()
std::vector< int64_t > SelectionVector
Definition ASoA.h:415
auto Attach(T const &table)
constexpr auto tableRef2ConfigParamSpec()
auto Extend(T const &table)
On-the-fly adding of expression columns.
constexpr auto tableRef2OutputSpec()
constexpr auto tableRef2OutputRef()
@ C
Definition Defs.h:36
Defining DataPointCompositeObject explicitly as copiable.
std::shared_ptr< T > table
constexpr auto pack()
typename T::first_t H
T const & operator*() const
auto build(framework::pack< Cs... >, std::vector< std::shared_ptr< arrow::Table > > &&tables)
metadata::index_pack_t index_pack_t
typename T::rest_t Ts
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
T::table_t const & operator*() const
decltype(transformBase< T >())::metadata metadata
typename metadata::extension_table_t extension_t
std::array< o2::framework::expressions::Projector, N > projectors
std::shared_ptr< extension_t > extension
std::shared_ptr< arrow::Schema > inputSchema
std::shared_ptr< arrow::Schema > schema
typename metadata::base_table_t base_table_t
static constexpr bool delayed
static constexpr size_t N
typename metadata::placeholders_pack_t placeholders_pack_t
std::shared_ptr< gandiva::Projector > projector
static auto indexBuilder(const char *label, std::vector< std::shared_ptr< arrow::Table > > &&tables, framework::pack< C1, Cs... >)
static OutputSpec const spec()
aod::MetadataTrait< o2::aod::Hash< table_t::ref.desc_hash > >::metadata metadata
decltype(typeWithRef< T >()) table_t
O2 header for OutputObj metadata.
OutputObj(T &&t, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
void setObject(std::shared_ptr< T > t)
OutputObjSourceType sourceType
OutputRef ref(uint16_t index, uint16_t max)
std::shared_ptr< T > object
OutputObj(std::string const &label_, OutputObjHandlingPolicy policy_=OutputObjHandlingPolicy::AnalysisObject, OutputObjSourceType sourceType_=OutputObjSourceType::OutputObjSource)
OutputSpec const spec()
OutputObjHandlingPolicy policy
void setHash(uint32_t hash)
Partition(expressions::Node &&filter_, T const &table)
typename o2::soa::Filtered< T >::const_iterator filtered_const_iterator
void bindTable(T const &table)
filtered_const_iterator begin() const
auto rawSliceBy(o2::framework::Preslice< T1 > const &container, int value) const
auto sliceByCached(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
void intializeCaches(std::set< uint32_t > const &hashes, std::shared_ptr< arrow::Schema > const &schema)
Partition(expressions::Node &&filter_)
typename o2::soa::Filtered< T >::iterator filtered_iterator
o2::soa::Filtered< T > * operator->()
typename o2::soa::Filtered< T >::iterator iterator
auto sliceBy(o2::framework::PresliceBase< T1, Policy, OPT > const &container, int value) const
std::shared_ptr< arrow::Table > asArrowTable() const
o2::soa::RowViewSentinel end() const
void updatePlaceholders(InitContext &context)
std::unique_ptr< o2::soa::Filtered< T > > mFiltered
expressions::Filter filter
o2::soa::RowViewSentinel end()
auto sliceByCachedUnsorted(framework::expressions::BindingNode const &node, int value, o2::framework::SliceCache &cache) const
typename o2::soa::Filtered< T >::const_iterator const_iterator
gandiva::FilterPtr gfilter
void bindExternalIndices(Ts *... tables)
filtered_iterator begin()
T::table_t * operator->()
std::shared_ptr< gandiva::Projector > projector
std::shared_ptr< extension_t > extension
T::table_t const & operator*() const
static constexpr size_t N
typename metadata::base_table_t base_table_t
std::shared_ptr< arrow::Schema > schema
typename metadata::expression_pack_t expression_pack_t
constexpr auto pack()
std::array< o2::framework::expressions::Projector, N > projectors
decltype(transformBase< T >())::metadata metadata
std::shared_ptr< typename T::table_t > table
typename metadata::extension_table_t extension_t
Helper template for table transformations.
constexpr auto spec() const
constexpr auto output() const
static constexpr auto base_spec()
constexpr auto ref() const
int64_t lastIndex()
Last index inserted in the table.
decltype(FFL(std::declval< cursor_t >())) cursor
bool resetCursor(LifetimeHolder< TableBuilder > builder)
void setLabel(const char *label)
void operator()(Ts &&... args)
decltype([]() { if constexpr(soa::is_iterator< T >) { return typename T::parent_t{nullptr} persistent_table_t
decltype(std::declval< TableBuilder >().cursor< persistent_table_t >()) cursor_t
An expression tree node corresponding to a column binding.
A struct, containing the root of the expression tree.
a move-only header stack with serialized headers This is the flat buffer where all the headers in a m...
Definition Stack.h:33
constexpr size_t max
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))