Project
Loading...
Searching...
No Matches
AnalysisManagers.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#ifndef FRAMEWORK_ANALYSISMANAGERS_H
13#define FRAMEWORK_ANALYSISMANAGERS_H
17#include "Framework/ASoA.h"
25#include "Framework/Condition.h"
30
31namespace o2::framework
32{
33
34namespace
35{
36template <typename O>
37static inline auto extractOriginal(ProcessingContext& pc)
38{
39 return pc.inputs().get<TableConsumer>(aod::MetadataTrait<O>::metadata::tableLabel())->asArrowTable();
40}
41
42template <typename... Os>
43static inline std::vector<std::shared_ptr<arrow::Table>> extractOriginals(framework::pack<Os...>, ProcessingContext& pc)
44{
45 return {extractOriginal<Os>(pc)...};
46}
47
48template <size_t N, std::array<soa::TableRef, N> refs>
49static inline auto extractOriginals(ProcessingContext& pc)
50{
51 return [&]<size_t... Is>(std::index_sequence<Is...>) -> std::vector<std::shared_ptr<arrow::Table>> {
52 return {pc.inputs().get<TableConsumer>(o2::aod::label<refs[Is]>())->asArrowTable()...};
53 }(std::make_index_sequence<refs.size()>());
54}
55} // namespace
56
57namespace analysis_task_parsers
58{
59
61template <typename O>
62bool appendOption(std::vector<ConfigParamSpec>&, O&)
63{
64 return false;
65}
66
67template <is_configurable O>
68bool appendOption(std::vector<ConfigParamSpec>& options, O& option)
69{
70 return ConfigurableHelpers::appendOption(options, option);
71}
72
73template <is_configurable_group O>
74bool appendOption(std::vector<ConfigParamSpec>& options, O& optionGroup)
75{
76 if constexpr (requires { optionGroup.prefix; }) {
77 homogeneous_apply_refs<true>([prefix = optionGroup.prefix]<typename C>(C& option) { // apend group prefix if set
78 if constexpr (requires { option.name; }) {
79 option.name.insert(0, 1, '.');
80 option.name.insert(0, prefix);
81 }
82 return true;
83 },
84 optionGroup);
85 }
86 homogeneous_apply_refs<true>([&options](auto& option) { return appendOption(options, option); }, optionGroup);
87 return true;
88}
89
90template <typename O>
92{
93 return false;
94}
95
96template <is_configurable O>
97bool prepareOption(InitContext& context, O& configurable)
98{
99 if constexpr (variant_trait_v<typename O::type> != VariantType::Unknown) {
100 configurable.value = context.options().get<typename O::type>(configurable.name.c_str());
101 } else {
102 auto pt = context.options().get<boost::property_tree::ptree>(configurable.name.c_str());
103 configurable.value = RootConfigParamHelpers::as<typename O::type>(pt);
104 }
105 return true;
106}
107
108template <is_configurable_group O>
109bool prepareOption(InitContext& context, O& configurableGroup)
110{
111 homogeneous_apply_refs<true>([&context](auto&& configurable) { return prepareOption(context, configurable); }, configurableGroup);
112 return true;
113}
114
116template <typename C>
117bool appendCondition(std::vector<InputSpec>&, C&)
118{
119 return false;
120}
121
122template <is_condition C>
123bool appendCondition(std::vector<InputSpec>& inputs, C& condition)
124{
125 inputs.emplace_back(InputSpec{condition.path, "AODC", runtime_hash(condition.path.c_str()), Lifetime::Condition, ccdbParamSpec(condition.path)});
126 return true;
127}
128
129template <is_condition_group C>
130bool appendCondition(std::vector<InputSpec>& inputs, C& conditionGroup)
131{
132 homogeneous_apply_refs<true>([&inputs](auto& condition) { return appendCondition(inputs, condition); }, conditionGroup);
133 return true;
134}
135
137template <typename T>
138bool requestInputs(std::vector<InputSpec>&, T const&)
139{
140 return false;
141}
142
143template <is_spawns T>
144const char* controlOption()
145{
146 return "control:spawn";
147}
148
149template <is_builds T>
150const char* controlOption()
151{
152 return "control:build";
153}
154
155template <is_defines T>
156const char* controlOption()
157{
158 return "control:define";
159}
160
161template <typename T>
162 requires(is_spawns<T> || is_builds<T> || is_defines<T>)
163bool requestInputs(std::vector<InputSpec>& inputs, T const& entity)
164{
165 auto base_specs = entity.base_specs();
166 for (auto base_spec : base_specs) {
167 base_spec.metadata.push_back(ConfigParamSpec{std::string{controlOption<T>()}, VariantType::Bool, true, {"\"\""}});
168 DataSpecUtils::updateInputList(inputs, std::forward<InputSpec>(base_spec));
169 }
170 return true;
171}
172
173template <typename C>
175{
176 return false;
177}
178
179template <is_condition C>
180bool newDataframeCondition(InputRecord& record, C& condition)
181{
182 condition.instance = (typename C::type*)record.get<typename C::type*>(condition.path).get();
183 return true;
184}
185
186template <is_condition_group C>
187bool newDataframeCondition(InputRecord& record, C& conditionGroup)
188{
189 homogeneous_apply_refs<true>([&record](auto&& condition) { return newDataframeCondition(record, condition); }, conditionGroup);
190 return true;
191}
192
194template <typename T>
195bool appendOutput(std::vector<OutputSpec>&, T&, uint32_t)
196{
197 return false;
198}
199
200template <is_produces T>
201bool appendOutput(std::vector<OutputSpec>& outputs, T&, uint32_t)
202{
204 return true;
205}
206
207template <is_produces_group T>
208bool appendOutput(std::vector<OutputSpec>& outputs, T& producesGroup, uint32_t hash)
209{
210 homogeneous_apply_refs<true>([&outputs, hash](auto& produces) { return appendOutput(outputs, produces, hash); }, producesGroup);
211 return true;
212}
213
214template <is_histogram_registry T>
215bool appendOutput(std::vector<OutputSpec>& outputs, T& hr, uint32_t hash)
216{
217 hr.setHash(hash);
218 outputs.emplace_back(hr.spec());
219 return true;
220}
221
222template <is_outputobj T>
223bool appendOutput(std::vector<OutputSpec>& outputs, T& obj, uint32_t hash)
224{
225 obj.setHash(hash);
226 outputs.emplace_back(obj.spec());
227 return true;
228}
229
230template <typename T>
231 requires(is_spawns<T> || is_builds<T> || is_defines<T>)
232bool appendOutput(std::vector<OutputSpec>& outputs, T& entity, uint32_t)
233{
234 outputs.emplace_back(entity.spec());
235 return true;
236}
237
238template <typename T>
240{
241 return false;
242}
243
244template <is_histogram_registry T>
245bool postRunOutput(EndOfStreamContext& context, T& hr)
246{
247 auto& deviceSpec = context.services().get<o2::framework::DeviceSpec const>();
248 context.outputs().snapshot(hr.ref(deviceSpec.inputTimesliceId, deviceSpec.maxInputTimeslices), *(hr.getListOfHistograms()));
249 hr.clean();
250 return true;
251}
252
253template <is_outputobj T>
254bool postRunOutput(EndOfStreamContext& context, T& obj)
255{
256 auto& deviceSpec = context.services().get<o2::framework::DeviceSpec const>();
257 context.outputs().snapshot(obj.ref(deviceSpec.inputTimesliceId, deviceSpec.maxInputTimeslices), *obj);
258 return true;
259}
260
261template <typename T>
263{
264 return false;
265}
266
267template <is_produces T>
268bool prepareOutput(ProcessingContext& context, T& produces)
269{
270 produces.resetCursor(std::move(context.outputs().make<TableBuilder>(OutputForTable<typename T::persistent_table_t>::ref())));
271 return true;
272}
273
274template <is_produces_group T>
275bool prepareOutput(ProcessingContext& context, T& producesGroup)
276{
277 homogeneous_apply_refs<true>([&context](auto& produces) { return prepareOutput(context, produces); }, producesGroup);
278 return true;
279}
280
281template <is_spawns T>
282bool prepareOutput(ProcessingContext& context, T& spawns)
283{
284 using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::spawnable_t::ref.desc_hash>>::metadata;
285 auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals<metadata::sources.size(), metadata::sources>(context), std::span{metadata::base_table_t::originalLabels});
286 if (originalTable->schema()->fields().empty() == true) {
287 using base_table_t = typename T::base_table_t::table_t;
288 originalTable = makeEmptyTable<base_table_t>(o2::aod::label<metadata::extension_table_t::ref>());
289 }
290 using D = o2::aod::Hash<metadata::extension_table_t::ref.desc_hash>;
291
292 spawns.extension = std::make_shared<typename T::extension_t>(o2::framework::spawner<D>(originalTable,
293 o2::aod::label<metadata::extension_table_t::ref>(),
294 spawns.projectors.data(),
295 spawns.projector,
296 spawns.schema));
297 spawns.table = std::make_shared<typename T::spawnable_t::table_t>(soa::ArrowHelpers::joinTables({spawns.extension->asArrowTable(), originalTable}, std::span{T::spawnable_t::table_t::originalLabels}));
298 return true;
299}
300
301template <is_builds T>
302bool prepareOutput(ProcessingContext& context, T& builds)
303{
304 using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::buildable_t::ref.desc_hash>>::metadata;
305 return builds.template build<typename T::buildable_t::indexing_t>(builds.pack(), extractOriginals<metadata::sources.size(), metadata::sources>(context));
306}
307
308template <is_defines T>
309bool prepareOutput(ProcessingContext& context, T& defines)
310{
311 using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::spawnable_t::ref.desc_hash>>::metadata;
312 auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals<metadata::sources.size(), metadata::sources>(context), std::span{metadata::base_table_t::originalLabels});
313 if (originalTable->schema()->fields().empty() == true) {
314 using base_table_t = typename T::base_table_t::table_t;
315 originalTable = makeEmptyTable<base_table_t>(o2::aod::label<metadata::extension_table_t::ref>());
316 }
317 using D = o2::aod::Hash<metadata::extension_table_t::ref.desc_hash>;
318
319 defines.extension = std::make_shared<typename T::extension_t>(o2::framework::spawner<D>(originalTable,
320 o2::aod::label<metadata::extension_table_t::ref>(),
321 defines.projectors.data(),
322 defines.projector,
323 defines.schema));
324 defines.table = std::make_shared<typename T::spawnable_t::table_t>(soa::ArrowHelpers::joinTables({defines.extension->asArrowTable(), originalTable}, std::span{T::spawnable_t::table_t::originalLabels}));
325 return true;
326}
327
328template <typename T>
330{
331 return false;
332}
333
334template <is_produces T>
335bool finalizeOutput(ProcessingContext&, T& produces)
336{
337 produces.setLabel(o2::aod::label<T::persistent_table_t::ref>());
338 produces.release();
339 return true;
340}
341
342template <is_produces_group T>
343bool finalizeOutput(ProcessingContext& context, T& producesGroup)
344{
345 homogeneous_apply_refs<true>([&context](auto& produces) { return finalizeOutput(context, produces); }, producesGroup);
346 return true;
347}
348
349template <is_spawns T>
350bool finalizeOutput(ProcessingContext& context, T& spawns)
351{
352 context.outputs().adopt(spawns.output(), spawns.asArrowTable());
353 return true;
354}
355
356template <is_builds T>
357bool finalizeOutput(ProcessingContext& context, T& builds)
358{
359 context.outputs().adopt(builds.output(), builds.asArrowTable());
360 return true;
361}
362
363template <is_defines T>
364bool finalizeOutput(ProcessingContext& context, T& defines)
365{
366 context.outputs().adopt(defines.output(), defines.asArrowTable());
367 return true;
368}
369
371template <typename T>
372bool addService(std::vector<ServiceSpec>&, T&)
373{
374 return false;
375}
376
377template <is_service T>
378bool addService(std::vector<ServiceSpec>& specs, T&)
379{
381 auto p = typename T::service_t{};
382 auto loadableServices = PluginManager::parsePluginSpecString(p.loadSpec.c_str());
383 PluginManager::loadFromPlugin<ServiceSpec, ServicePlugin>(loadableServices, specs);
384 }
385 return true;
386}
387
388template <typename T>
390{
391 return false;
392}
393
394template <is_service T>
395bool prepareService(InitContext& context, T& service)
396{
397 using S = typename T::service_t;
398 if constexpr (requires { &S::instance; }) {
399 service.service = &(S::instance()); // Sigh...
400 return true;
401 } else {
402 service.service = &(context.services().get<S>());
403 return true;
404 }
405 return false;
406}
407
408template <typename T>
410{
411 return false;
412}
413
414template <is_service T>
415bool postRunService(EndOfStreamContext&, T& service)
416{
417 // FIXME: for the moment we only need endOfStream to be
418 // stateless. In the future we might want to pass it EndOfStreamContext
419 if constexpr (requires { &T::service_t::endOfStream; }) {
420 service.service->endOfStream();
421 return true;
422 }
423 return false;
424}
425
427template <typename T>
429{
430 return false;
431}
432
433template <expressions::is_filter T>
434bool updatePlaceholders(InitContext& context, T& filter)
435{
437 return true;
438}
439
440template <is_partition T>
441bool updatePlaceholders(InitContext& context, T& partition)
442{
443 partition.updatePlaceholders(context);
444 return true;
445}
446
447template <typename T>
448bool createExpressionTrees(std::vector<ExpressionInfo>&, T&)
449{
450 return false;
451}
452
453template <expressions::is_filter T>
454bool createExpressionTrees(std::vector<ExpressionInfo>& expressionInfos, T& filter)
455{
457 return true;
458}
459
460template <typename T>
462{
463 return false;
464}
465
466template <is_partition T>
467bool newDataframePartition(T& partition)
468{
469 partition.dataframeChanged = true;
470 return true;
471}
472
473template <typename P, typename... T>
474void setPartition(P&, T&...)
475{
476}
477
478template <is_partition P, typename... T>
479void setPartition(P& partition, T&... tables)
480{
481 ([&]() { if constexpr (std::same_as<typename P::content_t, T>) {partition.bindTable(tables);} }(), ...);
482}
483
484template <typename P, typename T>
486{
487}
488
489template <is_partition P, typename T>
490void bindInternalIndicesPartition(P& partition, T* table)
491{
492 if constexpr (o2::soa::is_binding_compatible_v<typename P::content_t, std::decay_t<T>>()) {
493 partition.bindInternalIndicesTo(table);
494 }
495}
496
497template <typename P, typename... T>
499{
500}
501
502template <is_partition P, typename... T>
503void bindExternalIndicesPartition(P& partition, T*... tables)
504{
505 partition.bindExternalIndices(tables...);
506}
507
509template <typename T>
511{
512 return false;
513}
514
515template <typename T>
517{
518 return false;
519}
520
521template <is_slice_cache T>
522bool initializeCache(ProcessingContext& context, T& cache)
523{
524 if (cache.ptr == nullptr) {
525 cache.ptr = &context.services().get<ArrowTableSlicingCache>();
526 }
527 return true;
528}
529
531template <typename C, typename TG, typename... Ts>
532void setGroupedCombination(C&, TG&, Ts&...)
533{
534}
535
536template <is_combinations_generator C, typename TG, typename... Ts>
537static void setGroupedCombination(C& comb, TG& grouping, std::tuple<Ts...>& associated)
538{
539 if constexpr (std::same_as<typename C::g_t, std::decay_t<TG>>) {
540 comb.setTables(grouping, associated);
541 }
542}
543
545template <typename T>
546 requires(!is_preslice<T>)
548{
549 return false;
550}
551
552template <is_preslice T>
553 requires std::same_as<typename T::policy_t, framework::PreslicePolicySorted>
554bool registerCache(T& preslice, Cache& bsks, Cache&)
555{
556 if constexpr (T::optional) {
557 if (preslice.binding == "[MISSING]") {
558 return true;
559 }
560 }
561 auto locate = std::find_if(bsks.begin(), bsks.end(), [&](auto const& entry) { return (entry.binding == preslice.bindingKey.binding) && (entry.key == preslice.bindingKey.key); });
562 if (locate == bsks.end()) {
563 bsks.emplace_back(preslice.getBindingKey());
564 } else if (locate->enabled == false) {
565 locate->enabled = true;
566 }
567 return true;
568}
569
570template <is_preslice T>
571 requires std::same_as<typename T::policy_t, framework::PreslicePolicyGeneral>
572bool registerCache(T& preslice, Cache&, Cache& bsksU)
573{
574 if constexpr (T::optional) {
575 if (preslice.binding == "[MISSING]") {
576 return true;
577 }
578 }
579 auto locate = std::find_if(bsksU.begin(), bsksU.end(), [&](auto const& entry) { return (entry.binding == preslice.bindingKey.binding) && (entry.key == preslice.bindingKey.key); });
580 if (locate == bsksU.end()) {
581 bsksU.emplace_back(preslice.getBindingKey());
582 } else if (locate->enabled == false) {
583 locate->enabled = true;
584 }
585 return true;
586}
587
588template <typename T>
589 requires(!is_preslice<T>)
591{
592 return false;
593}
594
595template <is_preslice T>
596static bool updateSliceInfo(T& preslice, ArrowTableSlicingCache& cache)
597 requires std::same_as<typename T::policy_t, framework::PreslicePolicySorted>
598{
599 if constexpr (T::optional) {
600 if (preslice.binding == "[MISSING]") {
601 return true;
602 }
603 }
604 preslice.updateSliceInfo(cache.getCacheFor(preslice.getBindingKey()));
605 return true;
606}
607
608template <is_preslice T>
609static bool updateSliceInfo(T& preslice, ArrowTableSlicingCache& cache)
610 requires std::same_as<typename T::policy_t, framework::PreslicePolicyGeneral>
611{
612 if constexpr (T::optional) {
613 if (preslice.binding == "[MISSING]") {
614 return true;
615 }
616 }
617 preslice.updateSliceInfo(cache.getCacheUnsortedFor(preslice.getBindingKey()));
618 return true;
619}
620
622template <typename T>
623static bool setProcessSwitch(std::pair<std::string, bool>, T&)
624{
625 return false;
626}
627
628template <is_process_configurable T>
629static bool setProcessSwitch(std::pair<std::string, bool> setting, T& pc)
630{
631 if (pc.name == setting.first) {
632 pc.value = setting.second;
633 return true;
634 }
635 return false;
636}
637
638} // namespace analysis_task_parsers
639} // namespace o2::framework
640
641#endif // ANALYSISMANAGERS_H
constexpr uint32_t runtime_hash(char const *str)
void snapshot(const Output &spec, T const &object)
decltype(auto) make(const Output &spec, Args... args)
ServiceRegistryRef services()
Definition InitContext.h:34
ConfigParamRegistry const & options()
Definition InitContext.h:33
The input API of the Data Processing Layer This class holds the inputs which are valid for processing...
decltype(auto) get(R binding, int part=0) const
DataAllocator & outputs()
The data allocator is used to allocate memory for the output data.
ServiceRegistryRef services()
The services registry associated with this processing context.
GLuint entry
Definition glcorearb.h:5735
GLsizeiptr size
Definition glcorearb.h:659
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition glcorearb.h:1308
bool prepareService(InitContext &, T &)
bool requestInputs(std::vector< InputSpec > &, T const &)
Table auto-creation handling.
void setGroupedCombination(C &, TG &, Ts &...)
Combinations handling.
bool initializeCache(ProcessingContext &, T &)
bool preInitializeCache(InitContext &, T &)
Cache handling.
bool prepareOption(InitContext &, O &)
bool appendOutput(std::vector< OutputSpec > &, T &, uint32_t)
Outputs handling.
bool finalizeOutput(ProcessingContext &, T &)
bool postRunOutput(EndOfStreamContext &, T &)
bool createExpressionTrees(std::vector< ExpressionInfo > &, T &)
bool prepareOutput(ProcessingContext &, T &)
bool appendOption(std::vector< ConfigParamSpec > &, O &)
Options handling.
bool registerCache(T &, Cache &, Cache &)
Preslice handling.
bool updateSliceInfo(T &, ArrowTableSlicingCache &)
bool appendCondition(std::vector< InputSpec > &, C &)
Conditions handling.
bool postRunService(EndOfStreamContext &, T &)
bool addService(std::vector< ServiceSpec > &, T &)
Service handling.
bool updatePlaceholders(InitContext &, T &)
Filter handling.
bool newDataframeCondition(InputRecord &, C &)
void updateExpressionInfos(expressions::Filter const &filter, std::vector< ExpressionInfo > &eInfos)
Function for attaching gandiva filters to to compatible task inputs.
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
std::vector< Entry > Cache
std::vector< ConfigParamSpec > ccdbParamSpec(std::string const &path, int runDependent, std::vector< CCDBMetadata > metadata={}, int qrate=0)
static bool appendOption(std::vector< ConfigParamSpec > &options, Configurable< T, K, IP > &what)
static void updateInputList(std::vector< InputSpec > &list, InputSpec &&input)
Updates list of InputSpecs by merging metadata.
static std::vector< LoadablePlugin > parsePluginSpecString(char const *str)
Parse a comma separated list of <library>:<plugin-name> plugin declarations.
static std::shared_ptr< arrow::Table > joinTables(std::vector< std::shared_ptr< arrow::Table > > &&tables, std::span< const char *const > labels)
Definition ASoA.cxx:67