Project
Loading...
Searching...
No Matches
AnalysisManagers.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#ifndef FRAMEWORK_ANALYSISMANAGERS_H
13#define FRAMEWORK_ANALYSISMANAGERS_H
17#include "Framework/ASoA.h"
25#include "Framework/Condition.h"
30
31namespace o2::framework
32{
33
34namespace
35{
36template <typename O>
37static inline auto extractOriginal(ProcessingContext& pc)
38{
39 return pc.inputs().get<TableConsumer>(aod::MetadataTrait<O>::metadata::tableLabel())->asArrowTable();
40}
41
42template <typename... Os>
43static inline std::vector<std::shared_ptr<arrow::Table>> extractOriginals(framework::pack<Os...>, ProcessingContext& pc)
44{
45 return {extractOriginal<Os>(pc)...};
46}
47
48template <size_t N, std::array<soa::TableRef, N> refs>
49static inline auto extractOriginals(ProcessingContext& pc)
50{
51 return [&]<size_t... Is>(std::index_sequence<Is...>) -> std::vector<std::shared_ptr<arrow::Table>> {
52 return {pc.inputs().get<TableConsumer>(o2::aod::label<refs[Is]>())->asArrowTable()...};
53 }(std::make_index_sequence<refs.size()>());
54}
55} // namespace
56
57namespace analysis_task_parsers
58{
59
61template <typename O>
62bool appendOption(std::vector<ConfigParamSpec>&, O&)
63{
64 return false;
65}
66
67template <is_configurable O>
68bool appendOption(std::vector<ConfigParamSpec>& options, O& option)
69{
70 return ConfigurableHelpers::appendOption(options, option);
71}
72
73template <is_configurable_group O>
74bool appendOption(std::vector<ConfigParamSpec>& options, O& optionGroup)
75{
76 if constexpr (requires { optionGroup.prefix; }) {
77 homogeneous_apply_refs<true>([prefix = optionGroup.prefix]<typename C>(C& option) { // apend group prefix if set
78 if constexpr (requires { option.name; }) {
79 option.name.insert(0, 1, '.');
80 option.name.insert(0, prefix);
81 }
82 return true;
83 },
84 optionGroup);
85 }
86 homogeneous_apply_refs<true>([&options](auto& option) { return appendOption(options, option); }, optionGroup);
87 return true;
88}
89
90template <typename O>
92{
93 return false;
94}
95
96template <is_configurable O>
97bool prepareOption(InitContext& context, O& configurable)
98{
99 if constexpr (variant_trait_v<typename O::type> != VariantType::Unknown) {
100 configurable.value = context.options().get<typename O::type>(configurable.name.c_str());
101 } else {
102 auto pt = context.options().get<boost::property_tree::ptree>(configurable.name.c_str());
103 configurable.value = RootConfigParamHelpers::as<typename O::type>(pt);
104 }
105 return true;
106}
107
108template <is_configurable_group O>
109bool prepareOption(InitContext& context, O& configurableGroup)
110{
111 homogeneous_apply_refs<true>([&context](auto&& configurable) { return prepareOption(context, configurable); }, configurableGroup);
112 return true;
113}
114
116template <typename C>
117bool appendCondition(std::vector<InputSpec>&, C&)
118{
119 return false;
120}
121
122template <is_condition C>
123bool appendCondition(std::vector<InputSpec>& inputs, C& condition)
124{
125 inputs.emplace_back(InputSpec{condition.path, "AODC", runtime_hash(condition.path.c_str()), Lifetime::Condition, ccdbParamSpec(condition.path)});
126 return true;
127}
128
129template <is_condition_group C>
130bool appendCondition(std::vector<InputSpec>& inputs, C& conditionGroup)
131{
132 homogeneous_apply_refs<true>([&inputs](auto& condition) { return appendCondition(inputs, condition); }, conditionGroup);
133 return true;
134}
135
137template <typename T>
138bool requestInputs(std::vector<InputSpec>&, T const&)
139{
140 return false;
141}
142
143template <is_spawns T>
144const char* controlOption()
145{
146 return "control:spawn";
147}
148
149template <is_builds T>
150const char* controlOption()
151{
152 return "control:build";
153}
154
155template <is_defines T>
156const char* controlOption()
157{
158 return "control:define";
159}
160
161template <typename T>
162 requires(is_spawns<T> || is_builds<T> || is_defines<T>)
163bool requestInputs(std::vector<InputSpec>& inputs, T const& entity)
164{
165 auto base_specs = entity.base_specs();
166 for (auto base_spec : base_specs) {
167 base_spec.metadata.push_back(ConfigParamSpec{std::string{controlOption<T>()}, VariantType::Bool, true, {"\"\""}});
168 DataSpecUtils::updateInputList(inputs, std::forward<InputSpec>(base_spec));
169 }
170 return true;
171}
172
173template <typename C>
175{
176 return false;
177}
178
179template <is_condition C>
180bool newDataframeCondition(InputRecord& record, C& condition)
181{
182 condition.instance = (typename C::type*)record.get<typename C::type*>(condition.path).get();
183 return true;
184}
185
186template <is_condition_group C>
187bool newDataframeCondition(InputRecord& record, C& conditionGroup)
188{
189 homogeneous_apply_refs<true>([&record](auto&& condition) { return newDataframeCondition(record, condition); }, conditionGroup);
190 return true;
191}
192
194template <typename T>
195bool appendOutput(std::vector<OutputSpec>&, T&, uint32_t)
196{
197 return false;
198}
199
200template <is_produces T>
201bool appendOutput(std::vector<OutputSpec>& outputs, T&, uint32_t)
202{
204 return true;
205}
206
207template <is_produces_group T>
208bool appendOutput(std::vector<OutputSpec>& outputs, T& producesGroup, uint32_t hash)
209{
210 homogeneous_apply_refs<true>([&outputs, hash](auto& produces) { return appendOutput(outputs, produces, hash); }, producesGroup);
211 return true;
212}
213
214template <is_histogram_registry T>
215bool appendOutput(std::vector<OutputSpec>& outputs, T& hr, uint32_t hash)
216{
217 hr.setHash(hash);
218 outputs.emplace_back(hr.spec());
219 return true;
220}
221
222template <is_outputobj T>
223bool appendOutput(std::vector<OutputSpec>& outputs, T& obj, uint32_t hash)
224{
225 obj.setHash(hash);
226 outputs.emplace_back(obj.spec());
227 return true;
228}
229
230template <typename T>
231 requires(is_spawns<T> || is_builds<T> || is_defines<T>)
232bool appendOutput(std::vector<OutputSpec>& outputs, T& entity, uint32_t)
233{
234 outputs.emplace_back(entity.spec());
235 return true;
236}
237
238template <typename T>
240{
241 return false;
242}
243
244template <is_histogram_registry T>
245bool postRunOutput(EndOfStreamContext& context, T& hr)
246{
247 auto& deviceSpec = context.services().get<o2::framework::DeviceSpec const>();
248 context.outputs().snapshot(hr.ref(deviceSpec.inputTimesliceId, deviceSpec.maxInputTimeslices), *(hr.getListOfHistograms()));
249 hr.clean();
250 return true;
251}
252
253template <is_outputobj T>
254bool postRunOutput(EndOfStreamContext& context, T& obj)
255{
256 auto& deviceSpec = context.services().get<o2::framework::DeviceSpec const>();
257 context.outputs().snapshot(obj.ref(deviceSpec.inputTimesliceId, deviceSpec.maxInputTimeslices), *obj);
258 return true;
259}
260
261template <typename T>
263{
264 return false;
265}
266
267template <is_produces T>
268bool prepareOutput(ProcessingContext& context, T& produces)
269{
270 produces.resetCursor(std::move(context.outputs().make<TableBuilder>(OutputForTable<typename T::persistent_table_t>::ref())));
271 return true;
272}
273
274template <is_produces_group T>
275bool prepareOutput(ProcessingContext& context, T& producesGroup)
276{
277 homogeneous_apply_refs<true>([&context](auto& produces) { return prepareOutput(context, produces); }, producesGroup);
278 return true;
279}
280
281template <is_spawns T>
282bool prepareOutput(ProcessingContext& context, T& spawns)
283{
284 using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::spawnable_t::ref.desc_hash>>::metadata;
285 auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals<metadata::sources.size(), metadata::sources>(context));
286 if (originalTable->schema()->fields().empty() == true) {
287 using base_table_t = typename T::base_table_t::table_t;
288 originalTable = makeEmptyTable<base_table_t>(o2::aod::label<metadata::extension_table_t::ref>());
289 }
290
291 spawns.extension = std::make_shared<typename T::extension_t>(o2::framework::spawner<o2::aod::Hash<metadata::extension_table_t::ref.desc_hash>>(originalTable, o2::aod::label<metadata::extension_table_t::ref>(), spawns.projector));
292 spawns.table = std::make_shared<typename T::spawnable_t::table_t>(soa::ArrowHelpers::joinTables({spawns.extension->asArrowTable(), originalTable}));
293 return true;
294}
295
296template <is_builds T>
297bool prepareOuput(ProcessingContext& context, T& builds)
298{
299 using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::buildable_t::ref.desc_hash>>::metadata;
300 return builds.template build<typename T::buildable_t::indexing_t>(builds.pack(), extractOriginals<metadata::sources.size(), metadata::sources>(context));
301}
302
303template <is_defines T>
304bool prepareOutput(ProcessingContext& context, T& defines)
305{
306 using metadata = o2::aod::MetadataTrait<o2::aod::Hash<T::spawnable_t::ref.desc_hash>>::metadata;
307 auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals<metadata::sources.size(), metadata::sources>(context));
308 if (originalTable->schema()->fields().empty() == true) {
309 using base_table_t = typename T::base_table_t::table_t;
310 originalTable = makeEmptyTable<base_table_t>(o2::aod::label<metadata::extension_table_t::ref>());
311 }
312
313 defines.extension = std::make_shared<typename T::extension_t>(o2::framework::spawner<o2::aod::Hash<metadata::extension_table_t::ref.desc_hash>>(originalTable, o2::aod::label<metadata::extension_table_t::ref>(), defines.projectors.data(), defines.projector));
314 defines.table = std::make_shared<typename T::spawnable_t::table_t>(soa::ArrowHelpers::joinTables({defines.extension->asArrowTable(), originalTable}));
315 return true;
316}
317
318template <typename T>
320{
321 return false;
322}
323
324template <is_produces T>
325bool finalizeOutput(ProcessingContext&, T& produces)
326{
327 produces.setLabel(o2::aod::label<T::persistent_table_t::ref>());
328 produces.release();
329 return true;
330}
331
332template <is_produces_group T>
333bool finalizeOutput(ProcessingContext& context, T& producesGroup)
334{
335 homogeneous_apply_refs<true>([&context](auto& produces) { return finalizeOutput(context, produces); }, producesGroup);
336 return true;
337}
338
339template <is_spawns T>
340bool finalizeOutput(ProcessingContext& context, T& spawns)
341{
342 context.outputs().adopt(spawns.output(), spawns.asArrowTable());
343 return true;
344}
345
346template <is_builds T>
347bool finalizeOutput(ProcessingContext& context, T& builds)
348{
349 context.outputs().adopt(builds.output(), builds.asArrowTable());
350 return true;
351}
352
353template <is_defines T>
354bool finalizeOutput(ProcessingContext& context, T& defines)
355{
356 context.outputs().adopt(defines.output(), defines.asArrowTable());
357 return true;
358}
359
361template <typename T>
362bool addService(std::vector<ServiceSpec>&, T&)
363{
364 return false;
365}
366
367template <is_service T>
368bool addService(std::vector<ServiceSpec>& specs, T&)
369{
371 auto p = typename T::service_t{};
372 auto loadableServices = PluginManager::parsePluginSpecString(p.loadSpec.c_str());
373 PluginManager::loadFromPlugin<ServiceSpec, ServicePlugin>(loadableServices, specs);
374 }
375 return true;
376}
377
378template <typename T>
380{
381 return false;
382}
383
384template <is_service T>
385bool prepareService(InitContext& context, T& service)
386{
387 using S = typename T::service_t;
388 if constexpr (requires { &S::instance; }) {
389 service.service = &(S::instance()); // Sigh...
390 return true;
391 } else {
392 service.service = &(context.services().get<S>());
393 return true;
394 }
395 return false;
396}
397
398template <typename T>
400{
401 return false;
402}
403
404template <is_service T>
405bool postRunService(EndOfStreamContext&, T& service)
406{
407 // FIXME: for the moment we only need endOfStream to be
408 // stateless. In the future we might want to pass it EndOfStreamContext
409 if constexpr (requires { &T::service_t::endOfStream; }) {
410 service.service->endOfStream();
411 return true;
412 }
413 return false;
414}
415
417template <typename T>
419{
420 return false;
421}
422
423template <expressions::is_filter T>
424bool updatePlaceholders(InitContext& context, T& filter)
425{
427 return true;
428}
429
430template <is_partition T>
431bool updatePlaceholders(InitContext& context, T& partition)
432{
433 partition.updatePlaceholders(context);
434 return true;
435}
436
437template <typename T>
438bool createExpressionTrees(std::vector<ExpressionInfo>&, T&)
439{
440 return false;
441}
442
443template <expressions::is_filter T>
444bool createExpressionTrees(std::vector<ExpressionInfo>& expressionInfos, T& filter)
445{
447 return true;
448}
449
450template <typename T>
452{
453 return false;
454}
455
456template <is_partition T>
457bool newDataframePartition(T& partition)
458{
459 partition.dataframeChanged = true;
460 return true;
461}
462
463template <typename P, typename... T>
464void setPartition(P&, T&...)
465{
466}
467
468template <is_partition P, typename... T>
469void setPartition(P& partition, T&... tables)
470{
471 ([&]() { if constexpr (std::same_as<typename P::content_t, T>) {partition.bindTable(tables);} }(), ...);
472}
473
474template <typename P, typename T>
476{
477}
478
479template <is_partition P, typename T>
480void bindInternalIndicesPartition(P& partition, T* table)
481{
482 if constexpr (o2::soa::is_binding_compatible_v<typename P::content_t, std::decay_t<T>>()) {
483 partition.bindInternalIndicesTo(table);
484 }
485}
486
487template <typename P, typename... T>
489{
490}
491
492template <is_partition P, typename... T>
493void bindExternalIndicesPartition(P& partition, T*... tables)
494{
495 partition.bindExternalIndices(tables...);
496}
497
499template <typename T>
501{
502 return false;
503}
504
505template <typename T>
507{
508 return false;
509}
510
511template <is_slice_cache T>
512bool initializeCache(ProcessingContext& context, T& cache)
513{
514 if (cache.ptr == nullptr) {
515 cache.ptr = &context.services().get<ArrowTableSlicingCache>();
516 }
517 return true;
518}
519
521template <typename C, typename TG, typename... Ts>
522void setGroupedCombination(C&, TG&, Ts&...)
523{
524}
525
526template <is_combinations_generator C, typename TG, typename... Ts>
527static void setGroupedCombination(C& comb, TG& grouping, std::tuple<Ts...>& associated)
528{
529 if constexpr (std::same_as<typename C::g_t, std::decay_t<TG>>) {
530 comb.setTables(grouping, associated);
531 }
532}
533
535template <typename T>
536 requires(!is_preslice<T>)
537bool registerCache(T&, std::vector<StringPair>&, std::vector<StringPair>&)
538{
539 return false;
540}
541
542template <is_preslice T>
543 requires std::same_as<typename T::policy_t, framework::PreslicePolicySorted>
544bool registerCache(T& preslice, std::vector<StringPair>& bsks, std::vector<StringPair>&)
545{
546 if constexpr (T::optional) {
547 if (preslice.binding == "[MISSING]") {
548 return true;
549 }
550 }
551 auto locate = std::find_if(bsks.begin(), bsks.end(), [&](auto const& entry) { return (entry.first == preslice.bindingKey.first) && (entry.second == preslice.bindingKey.second); });
552 if (locate == bsks.end()) {
553 bsks.emplace_back(preslice.getBindingKey());
554 }
555 return true;
556}
557
558template <is_preslice T>
559 requires std::same_as<typename T::policy_t, framework::PreslicePolicyGeneral>
560bool registerCache(T& preslice, std::vector<StringPair>&, std::vector<StringPair>& bsksU)
561{
562 if constexpr (T::optional) {
563 if (preslice.binding == "[MISSING]") {
564 return true;
565 }
566 }
567 auto locate = std::find_if(bsksU.begin(), bsksU.end(), [&](auto const& entry) { return (entry.first == preslice.bindingKey.first) && (entry.second == preslice.bindingKey.second); });
568 if (locate == bsksU.end()) {
569 bsksU.emplace_back(preslice.getBindingKey());
570 }
571 return true;
572}
573
574template <typename T>
575 requires(!is_preslice<T>)
577{
578 return false;
579}
580
581template <is_preslice T>
582static bool updateSliceInfo(T& preslice, ArrowTableSlicingCache& cache)
583 requires std::same_as<typename T::policy_t, framework::PreslicePolicySorted>
584{
585 if constexpr (T::optional) {
586 if (preslice.binding == "[MISSING]") {
587 return true;
588 }
589 }
590 preslice.updateSliceInfo(cache.getCacheFor(preslice.getBindingKey()));
591 return true;
592}
593
594template <is_preslice T>
595static bool updateSliceInfo(T& preslice, ArrowTableSlicingCache& cache)
596 requires std::same_as<typename T::policy_t, framework::PreslicePolicyGeneral>
597{
598 if constexpr (T::optional) {
599 if (preslice.binding == "[MISSING]") {
600 return true;
601 }
602 }
603 preslice.updateSliceInfo(cache.getCacheUnsortedFor(preslice.getBindingKey()));
604 return true;
605}
606
608template <typename T>
609static bool setProcessSwitch(std::pair<std::string, bool>, T&)
610{
611 return false;
612}
613
614template <is_process_configurable T>
615static bool setProcessSwitch(std::pair<std::string, bool> setting, T& pc)
616{
617 if (pc.name == setting.first) {
618 pc.value = setting.second;
619 return true;
620 }
621 return false;
622}
623
624} // namespace analysis_task_parsers
625} // namespace o2::framework
626
627#endif // ANALYSISMANAGERS_H
constexpr uint32_t runtime_hash(char const *str)
void snapshot(const Output &spec, T const &object)
decltype(auto) make(const Output &spec, Args... args)
ServiceRegistryRef services()
Definition InitContext.h:34
ConfigParamRegistry const & options()
Definition InitContext.h:33
The input API of the Data Processing Layer This class holds the inputs which are valid for processing...
decltype(auto) get(R binding, int part=0) const
DataAllocator & outputs()
The data allocator is used to allocate memory for the output data.
ServiceRegistryRef services()
The services registry associated with this processing context.
GLuint entry
Definition glcorearb.h:5735
GLsizeiptr size
Definition glcorearb.h:659
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition glcorearb.h:1308
bool prepareService(InitContext &, T &)
bool requestInputs(std::vector< InputSpec > &, T const &)
Table auto-creation handling.
void setGroupedCombination(C &, TG &, Ts &...)
Combinations handling.
bool initializeCache(ProcessingContext &, T &)
bool preInitializeCache(InitContext &, T &)
Cache handling.
bool prepareOption(InitContext &, O &)
bool appendOutput(std::vector< OutputSpec > &, T &, uint32_t)
Outputs handling.
bool finalizeOutput(ProcessingContext &, T &)
bool postRunOutput(EndOfStreamContext &, T &)
bool createExpressionTrees(std::vector< ExpressionInfo > &, T &)
bool prepareOutput(ProcessingContext &, T &)
bool appendOption(std::vector< ConfigParamSpec > &, O &)
Options handling.
bool prepareOuput(ProcessingContext &context, T &builds)
bool updateSliceInfo(T &, ArrowTableSlicingCache &)
bool appendCondition(std::vector< InputSpec > &, C &)
Conditions handling.
bool postRunService(EndOfStreamContext &, T &)
bool addService(std::vector< ServiceSpec > &, T &)
Service handling.
bool updatePlaceholders(InitContext &, T &)
Filter handling.
bool registerCache(T &, std::vector< StringPair > &, std::vector< StringPair > &)
Preslice handling.
bool newDataframeCondition(InputRecord &, C &)
void updateExpressionInfos(expressions::Filter const &filter, std::vector< ExpressionInfo > &eInfos)
Function for attaching gandiva filters to to compatible task inputs.
void updatePlaceholders(Filter &filter, InitContext &context)
Update placeholder nodes from context.
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
auto spawner(std::vector< std::shared_ptr< arrow::Table > > &&tables, const char *name, o2::framework::expressions::Projector *projectors, std::shared_ptr< gandiva::Projector > &projector)
Expression-based column generator to materialize columns.
std::vector< ConfigParamSpec > ccdbParamSpec(std::string const &path, int runDependent, std::vector< CCDBMetadata > metadata={}, int qrate=0)
static bool appendOption(std::vector< ConfigParamSpec > &options, Configurable< T, K, IP > &what)
static void updateInputList(std::vector< InputSpec > &list, InputSpec &&input)
Updates list of InputSpecs by merging metadata.
static std::vector< LoadablePlugin > parsePluginSpecString(char const *str)
Parse a comma separated list of <library>:<plugin-name> plugin declarations.
static std::shared_ptr< arrow::Table > joinTables(std::vector< std::shared_ptr< arrow::Table > > &&tables)
Definition ASoA.cxx:67