Project
Loading...
Searching...
No Matches
NoiseCalibratorSpec.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
13
14#include "CCDB/CcdbApi.h"
23
24#include <fairlogger/Logger.h>
29
30using namespace o2::framework;
31using namespace o2::utils;
32
33namespace o2
34{
35namespace mft
36{
37
38NoiseCalibratorSpec::NoiseCalibratorSpec(bool useDigits, std::shared_ptr<o2::base::GRPGeomRequest> req)
39 : mDigits(useDigits), mCCDBRequest(req)
40{
41}
42
44{
46 auto probT = ic.options().get<float>("prob-threshold");
47 auto probTRelErr = ic.options().get<float>("prob-rel-err");
48 LOGP(info, "Setting the probability threshold to {} with relative error {}", probT, probTRelErr);
49 mStopMeOnly = ic.options().get<bool>("stop-me-only");
50 mPath = ic.options().get<std::string>("path-CCDB");
51 mPathMerge = ic.options().get<std::string>("path-CCDB-merge");
52
53 mMeta = ic.options().get<std::string>("meta");
54 mStart = ic.options().get<int64_t>("tstart");
55 mEnd = ic.options().get<int64_t>("tend");
56
57 mCalibrator = std::make_unique<CALIBRATOR>(probT, probTRelErr);
58
59 mPathDcs = ic.options().get<std::string>("path-DCS");
60 mOutputType = ic.options().get<std::string>("send-to-server");
61 mNoiseMapForDcs.clear();
63}
64
66{
67 updateTimeDependentParams(pc);
68 if (mDigits) {
69 const auto digits = pc.inputs().get<gsl::span<o2::itsmft::Digit>>("digits");
70 const auto rofs = pc.inputs().get<gsl::span<o2::itsmft::ROFRecord>>("digitsROF");
71 const auto tfcounter = o2::header::get<o2::framework::DataProcessingHeader*>(pc.inputs().get("digitsROF").header)->startTime;
72
73 if (mCalibrator->processTimeFrame(tfcounter, digits, rofs)) {
74 LOG(info) << "Minimum number of noise counts has been reached !";
75 if (mOutputType.compare("CCDB") == 0) {
76 LOG(info) << "Sending an object to Production-CCDB";
77 sendOutputCcdb(pc.outputs());
78 LOG(info) << "Sending an object to Production-CCDBMerge";
79 sendOutputCcdbMerge(pc.outputs());
80 } else if (mOutputType.compare("DCS") == 0) {
81 LOG(info) << "Sending an object to DCS-CCDB";
82 sendOutputDcs(pc.outputs());
83 } else {
84 LOG(info) << "Sending an object to Production-CCDB and DCS-CCDB";
85 sendOutputCcdbDcs(pc.outputs());
86 LOG(info) << "Sending an object to Production-CCDBMerge";
87 sendOutputCcdbMerge(pc.outputs());
88 }
89 pc.services().get<ControlService>().readyToQuit(mStopMeOnly ? QuitRequest::Me : QuitRequest::All);
90 }
91 } else {
92 const auto compClusters = pc.inputs().get<gsl::span<o2::itsmft::CompClusterExt>>("compClusters");
93 gsl::span<const unsigned char> patterns = pc.inputs().get<gsl::span<unsigned char>>("patterns");
94 const auto rofs = pc.inputs().get<gsl::span<o2::itsmft::ROFRecord>>("ROframes");
95 const auto tfcounter = o2::header::get<o2::framework::DataProcessingHeader*>(pc.inputs().get("ROframes").header)->startTime;
96
97 if (mCalibrator->processTimeFrame(tfcounter, compClusters, patterns, rofs)) {
98 LOG(info) << "Minimum number of noise counts has been reached !";
99 if (mOutputType.compare("CCDB") == 0) {
100 LOG(info) << "Sending an object to Production-CCDB";
101 sendOutputCcdb(pc.outputs());
102 LOG(info) << "Sending an object to Production-CCDBMerge";
103 sendOutputCcdbMerge(pc.outputs());
104 } else if (mOutputType.compare("DCS") == 0) {
105 LOG(info) << "Sending an object to DCS-CCDB";
106 sendOutputDcs(pc.outputs());
107 } else {
108 LOG(info) << "Sending an object to Production-CCDB and DCS-CCDB";
109 sendOutputCcdbDcs(pc.outputs());
110 LOG(info) << "Sending an object to Production-CCDBMerge";
111 sendOutputCcdbMerge(pc.outputs());
112 }
113 pc.services().get<ControlService>().readyToQuit(mStopMeOnly ? QuitRequest::Me : QuitRequest::All);
114 }
115 }
116}
117
118void NoiseCalibratorSpec::setOutputDcs(const o2::itsmft::NoiseMap& payload)
119{
120 for (int iChip = 0; iChip < 936; ++iChip) {
121 for (int iRow = 0; iRow < 512; ++iRow) {
122 for (int iCol = 0; iCol < 1024; ++iCol) {
123
124 if (!payload.isNoisy(iChip, iRow, iCol)) {
125 continue;
126 }
127 std::array<int, 3> noise = {iChip, iRow, iCol};
128 mNoiseMapForDcs.emplace_back(noise);
129 }
130 }
131 }
132}
133
134void NoiseCalibratorSpec::sendOutputCcdbDcs(DataAllocator& output)
135{
136
137 LOG(info) << "CCDB-DCS mode";
138
139 static bool done = false;
140 if (done) {
141 return;
142 }
143 done = true;
144
145 mCalibrator->finalize();
146
147 long tstart = mStart;
148 if (tstart == -1) {
150 }
151 long tend = mEnd;
152 if (tend == -1) {
153 constexpr long SECONDSPERYEAR = 365 * 24 * 60 * 60;
154 tend = o2::ccdb::getFutureTimestamp(SECONDSPERYEAR);
155 }
156
157 std::map<std::string, std::string> meta;
158 auto toKeyValPairs = [&meta](std::vector<std::string> const& tokens) {
159 for (auto& token : tokens) {
160 auto keyval = Str::tokenize(token, '=', false);
161 if (keyval.size() != 2) {
162 LOG(error) << "Illegal command-line key/value string: " << token;
163 continue;
164 }
165 Str::trim(keyval[1]);
166 meta[keyval[0]] = keyval[1];
167 }
168 };
169 toKeyValPairs(Str::tokenize(mMeta, ';', true));
170
171 long startTF, endTF;
172
173 const auto& payload = mCalibrator->getNoiseMap();
174 // const auto& payload = mCalibrator->getNoiseMap(starTF, endTF); //For TimeSlot calibration
175
176 o2::ccdb::CcdbObjectInfo info(mPath, "NoiseMap", "noise.root", meta, tstart, tend);
177 auto flName = o2::ccdb::CcdbApi::generateFileName("noise");
178 auto image = o2::ccdb::CcdbApi::createObjectImage(&payload, &info);
179 info.setFileName(flName);
180 LOG(info) << "Sending object " << info.getPath() << "/" << info.getFileName()
181 << " of size " << image->size()
182 << " bytes, valid for " << info.getStartValidityTimestamp()
183 << " : " << info.getEndValidityTimestamp();
184
186 output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "MFT_NoiseMap", 0}, *image.get());
187 output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "MFT_NoiseMap", 0}, info);
188
189 setOutputDcs(payload);
190
191 o2::ccdb::CcdbObjectInfo infoDcs(mPathDcs, "NoiseMap", "noise.root", meta, tstart, tend);
192 auto flNameDcs = o2::ccdb::CcdbApi::generateFileName("noise");
193 auto imageDcs = o2::ccdb::CcdbApi::createObjectImage(&mNoiseMapForDcs, &infoDcs);
194 infoDcs.setFileName(flNameDcs);
195 LOG(info) << "Sending object " << infoDcs.getPath() << "/" << infoDcs.getFileName()
196 << " of size " << imageDcs->size()
197 << " bytes, valid for " << infoDcs.getStartValidityTimestamp()
198 << " : " << infoDcs.getEndValidityTimestamp();
199
200 using clbUtilsDcs = o2::calibration::Utils;
201 output.snapshot(Output{clbUtilsDcs::gDataOriginCDBPayload, "MFT_NoiseMap", 1}, *imageDcs.get());
202 output.snapshot(Output{clbUtilsDcs::gDataOriginCDBWrapper, "MFT_NoiseMap", 1}, infoDcs);
203}
204
205void NoiseCalibratorSpec::sendOutputCcdb(DataAllocator& output)
206{
207
208 LOG(info) << "CCDB mode";
209
210 static bool done = false;
211 if (done) {
212 return;
213 }
214 done = true;
215
216 mCalibrator->finalize();
217
218 long tstart = mStart;
219 if (tstart == -1) {
221 }
222 long tend = mEnd;
223 if (tend == -1) {
224 constexpr long SECONDSPERYEAR = 365 * 24 * 60 * 60;
225 tend = o2::ccdb::getFutureTimestamp(SECONDSPERYEAR);
226 }
227
228 std::map<std::string, std::string> meta;
229 auto toKeyValPairs = [&meta](std::vector<std::string> const& tokens) {
230 for (auto& token : tokens) {
231 auto keyval = Str::tokenize(token, '=', false);
232 if (keyval.size() != 2) {
233 LOG(error) << "Illegal command-line key/value string: " << token;
234 continue;
235 }
236 Str::trim(keyval[1]);
237 meta[keyval[0]] = keyval[1];
238 }
239 };
240 toKeyValPairs(Str::tokenize(mMeta, ';', true));
241
242 long startTF, endTF;
243
244 const auto& payload = mCalibrator->getNoiseMap();
245 // const auto& payload = mCalibrator->getNoiseMap(starTF, endTF); //For TimeSlot calibration
246
247 o2::ccdb::CcdbObjectInfo info(mPath, "NoiseMap", "noise.root", meta, tstart, tend);
248 auto flName = o2::ccdb::CcdbApi::generateFileName("noise");
249 auto image = o2::ccdb::CcdbApi::createObjectImage(&payload, &info);
250 info.setFileName(flName);
251 LOG(info) << "Sending object CCDB " << info.getPath() << "/" << info.getFileName()
252 << " of size " << image->size()
253 << " bytes, valid for " << info.getStartValidityTimestamp()
254 << " : " << info.getEndValidityTimestamp();
255
257 output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "MFT_NoiseMap", 0}, *image.get());
258 output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "MFT_NoiseMap", 0}, info);
259}
260
261void NoiseCalibratorSpec::sendOutputCcdbMerge(DataAllocator& output)
262{
263
264 LOG(info) << "CCDB-Merge mode";
265
266 static bool done = false;
267 if (done) {
268 return;
269 }
270 done = true;
271
272 mCalibrator->finalize();
273
274 long tstart = mStart;
275 if (tstart == -1) {
277 }
278 long tend = mEnd;
279 if (tend == -1) {
280 constexpr long SECONDSPERYEAR = 365 * 24 * 60 * 60;
281 tend = o2::ccdb::getFutureTimestamp(SECONDSPERYEAR);
282 }
283
284 std::map<std::string, std::string> meta;
285 auto toKeyValPairs = [&meta](std::vector<std::string> const& tokens) {
286 for (auto& token : tokens) {
287 auto keyval = Str::tokenize(token, '=', false);
288 if (keyval.size() != 2) {
289 LOG(error) << "Illegal command-line key/value string: " << token;
290 continue;
291 }
292 Str::trim(keyval[1]);
293 meta[keyval[0]] = keyval[1];
294 }
295 };
296 toKeyValPairs(Str::tokenize(mMeta, ';', true));
297
298 long startTF, endTF;
299
300 auto payload = mCalibrator->getNoiseMap();
301 // const auto& payload = mCalibrator->getNoiseMap(starTF, endTF); //For TimeSlot calibration
302 map<string, string> headers;
303 map<std::string, std::string> filter;
304 auto* payloadPrev1 = api.retrieveFromTFileAny<o2::itsmft::NoiseMap>(mPath, filter, -1, &headers);
305 long validtime = std::stol(headers["Valid-From"]);
306 auto mergedPL = payload;
307 if (validtime > 0) {
308 validtime = validtime - 1;
309 auto* payloadPrev2 = api.retrieveFromTFileAny<o2::itsmft::NoiseMap>(mPath, filter, validtime, &headers);
310 auto bufferPL = payloadPrev2->merge(payloadPrev1);
311 mergedPL = payload.merge(&bufferPL);
312 }
313 o2::ccdb::CcdbObjectInfo info(mPathMerge, "NoiseMap", "noise.root", meta, tstart, tend);
314 auto flName = o2::ccdb::CcdbApi::generateFileName("noise");
315 auto image = o2::ccdb::CcdbApi::createObjectImage(&mergedPL, &info);
316 info.setFileName(flName);
317 LOG(info) << "Sending object ccdb-merge " << info.getPath() << "/" << info.getFileName()
318 << " of size " << image->size()
319 << " bytes, valid for " << info.getStartValidityTimestamp()
320 << " : " << info.getEndValidityTimestamp();
321
323 output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "MFT_NoiseMap", 0}, *image.get());
324 output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "MFT_NoiseMap", 0}, info);
325}
326
327void NoiseCalibratorSpec::sendOutputDcs(DataAllocator& output)
328{
329
330 LOG(info) << "DCS mode";
331
332 static bool done = false;
333 if (done) {
334 return;
335 }
336 done = true;
337
338 mCalibrator->finalize();
339
340 long tstart = mStart;
341 if (tstart == -1) {
343 }
344 long tend = mEnd;
345 if (tend == -1) {
346 constexpr long SECONDSPERYEAR = 365 * 24 * 60 * 60;
347 tend = o2::ccdb::getFutureTimestamp(SECONDSPERYEAR);
348 }
349
350 std::map<std::string, std::string> meta;
351 auto toKeyValPairs = [&meta](std::vector<std::string> const& tokens) {
352 for (auto& token : tokens) {
353 auto keyval = Str::tokenize(token, '=', false);
354 if (keyval.size() != 2) {
355 LOG(error) << "Illegal command-line key/value string: " << token;
356 continue;
357 }
358 Str::trim(keyval[1]);
359 meta[keyval[0]] = keyval[1];
360 }
361 };
362 toKeyValPairs(Str::tokenize(mMeta, ';', true));
363
364 long startTF, endTF;
365
366 const auto& payload = mCalibrator->getNoiseMap();
367 // const auto& payload = mCalibrator->getNoiseMap(starTF, endTF); //For TimeSlot calibration
368
369 setOutputDcs(payload);
370
371 o2::ccdb::CcdbObjectInfo infoDcs(mPathDcs, "NoiseMap", "noise.root", meta, tstart, tend);
372 auto flNameDcs = o2::ccdb::CcdbApi::generateFileName("noise");
373 auto imageDcs = o2::ccdb::CcdbApi::createObjectImage(&mNoiseMapForDcs, &infoDcs);
374 infoDcs.setFileName(flNameDcs);
375 LOG(info) << "Sending object " << infoDcs.getPath() << "/" << infoDcs.getFileName()
376 << " of size " << imageDcs->size()
377 << " bytes, valid for " << infoDcs.getStartValidityTimestamp()
378 << " : " << infoDcs.getEndValidityTimestamp();
379
380 using clbUtilsDcs = o2::calibration::Utils;
381 output.snapshot(Output{clbUtilsDcs::gDataOriginCDBPayload, "MFT_NoiseMap", 0}, *imageDcs.get());
382 output.snapshot(Output{clbUtilsDcs::gDataOriginCDBWrapper, "MFT_NoiseMap", 0}, infoDcs);
383}
384
386{
387 if (mOutputType.compare("CCDB") == 0) {
388 LOG(info) << "Sending an object to Production-CCDB";
389 sendOutputCcdb(ec.outputs());
390 LOG(info) << "Sending an object to Production-CCDB-Merge";
391 sendOutputCcdbMerge(ec.outputs());
392 } else if (mOutputType.compare("DCS") == 0) {
393 LOG(info) << "Sending an object to DCS-CCDB";
394 sendOutputDcs(ec.outputs());
395 } else {
396 LOG(info) << "Sending an object to Production-CCDB and DCS-CCDB";
397 sendOutputCcdbDcs(ec.outputs());
398 sendOutputCcdbMerge(ec.outputs());
399 }
400}
401
403void NoiseCalibratorSpec::updateTimeDependentParams(ProcessingContext& pc)
404{
406 if (!mDigits) {
407 pc.inputs().get<o2::itsmft::TopologyDictionary*>("cldict"); // just to trigger the finaliseCCDB
408 }
409}
410
413{
415 if (matcher == ConcreteDataMatcher("MFT", "CLUSDICT", 0)) {
416 LOG(info) << "cluster dictionary updated";
417 mCalibrator->setClusterDictionary((const o2::itsmft::TopologyDictionary*)obj);
418 }
419}
420
422{
424 std::vector<InputSpec> inputs;
425 if (useDigits) {
426 inputs.emplace_back("digits", detOrig, "DIGITS", 0, Lifetime::Timeframe);
427 inputs.emplace_back("digitsROF", detOrig, "DIGITSROF", 0, Lifetime::Timeframe);
428 } else {
429 inputs.emplace_back("compClusters", detOrig, "COMPCLUSTERS", 0, Lifetime::Timeframe);
430 inputs.emplace_back("patterns", detOrig, "PATTERNS", 0, Lifetime::Timeframe);
431 inputs.emplace_back("ROframes", detOrig, "CLUSTERSROF", 0, Lifetime::Timeframe);
432 inputs.emplace_back("cldict", "MFT", "CLUSDICT", 0, Lifetime::Condition, ccdbParamSpec("MFT/Calib/ClusterDictionary"));
433 }
434 auto ccdbRequest = std::make_shared<o2::base::GRPGeomRequest>(false, // orbitResetTime
435 false, // GRPECS=true
436 false, // GRPLHCIF
437 false, // GRPMagField
438 false, // askMatLUT
440 inputs);
442 std::vector<OutputSpec> outputs;
443 outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBPayload, "MFT_NoiseMap"}, Lifetime::Sporadic);
444 outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBWrapper, "MFT_NoiseMap"}, Lifetime::Sporadic);
445
446 return DataProcessorSpec{
447 "mft-noise-calibrator",
448 inputs,
449 outputs,
450 AlgorithmSpec{adaptFromTask<NoiseCalibratorSpec>(useDigits, ccdbRequest)},
451 Options{
452 {"prob-threshold", VariantType::Float, 1.e-6f, {"Probability threshold for noisy pixels"}},
453 {"prob-rel-err", VariantType::Float, 0.2f, {"Relative error on channel noise to apply the threshold"}},
454 {"tstart", VariantType::Int64, -1ll, {"Start of validity timestamp"}},
455 {"tend", VariantType::Int64, -1ll, {"End of validity timestamp"}},
456 {"path-CCDB", VariantType::String, "/MFT/Calib/NoiseMap", {"Path to write to in CCDB"}},
457 {"path-CCDB-merge", VariantType::String, "/MFT/Calib/NoiseMapMerged", {"Path to write merged file to in CCDB"}},
458 {"path-DCS", VariantType::String, "/MFT/Config/NoiseMap", {"Path to write to in DCS"}},
459 {"meta", VariantType::String, "", {"meta data to write in CCDB"}},
460 {"send-to-server", VariantType::String, "CCDB-DCS", {"meta data to write in DCS-CCDB"}},
461 {"stop-me-only", VariantType::Bool, false, {"At sufficient statistics stop only this device, otherwise whole workflow"}}}};
462}
463
464} // namespace mft
465} // namespace o2
Definition of the ITS/MFT clusterer settings.
Definition of the ITSMFT compact cluster.
Definition of the ITSMFT digit.
Definition of the Names Generator class.
Utils and constants for calibration and related workflows.
bool done
Definition of the ITSMFT ROFrame (trigger) record.
void output(const std::map< std::string, ChannelStat > &channels)
Definition rawdump.cxx:197
void checkUpdates(o2::framework::ProcessingContext &pc)
bool finaliseCCDB(o2::framework::ConcreteDataMatcher &matcher, void *obj)
static GRPGeomHelper & instance()
void setRequest(std::shared_ptr< GRPGeomRequest > req)
static std::string getCCDBServer()
Definition NameConf.cxx:110
static std::string generateFileName(const std::string &inp)
Definition CcdbApi.cxx:815
void init(std::string const &hosts)
Definition CcdbApi.cxx:165
static std::unique_ptr< std::vector< char > > createObjectImage(const T *obj, CcdbObjectInfo *info=nullptr)
Definition CcdbApi.h:103
std::enable_if<!std::is_base_of< o2::conf::ConfigurableParam, T >::value, T * >::type retrieveFromTFileAny(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp=-1, std::map< std::string, std::string > *headers=nullptr, std::string const &etag="", const std::string &createdNotAfter="", const std::string &createdNotBefore="") const
Definition CcdbApi.h:660
ConfigParamRegistry const & options()
Definition InitContext.h:33
decltype(auto) get(R binding, int part=0) const
DataAllocator & outputs()
The data allocator is used to allocate memory for the output data.
InputRecord & inputs()
The inputs associated with this processing context.
ServiceRegistryRef services()
The services registry associated with this processing context.
NoiseMap class for the ITS and MFT.
Definition NoiseMap.h:39
NoiseMap merge(const NoiseMap *prev)
Definition NoiseMap.h:205
bool isNoisy(int chip, int row, int col) const
Definition NoiseMap.h:151
NoiseCalibratorSpec(bool digits=false, std::shared_ptr< o2::base::GRPGeomRequest > req={})
void endOfStream(EndOfStreamContext &ec) final
This is invoked whenever we have an EndOfStream event.
void run(ProcessingContext &pc) final
void init(InitContext &ic) final
void finaliseCCDB(ConcreteDataMatcher &matcher, void *obj) final
GLeglImageOES image
Definition glcorearb.h:4021
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition glcorearb.h:1308
constexpr o2::header::DataOrigin gDataOriginMFT
Definition DataHeader.h:572
long getCurrentTimestamp()
returns the timestamp in long corresponding to "now"
long getFutureTimestamp(int secondsInFuture)
returns the timestamp in long corresponding to "now + secondsInFuture"
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
std::vector< ConfigParamSpec > ccdbParamSpec(std::string const &path, int runDependent, std::vector< CCDBMetadata > metadata={}, int qrate=0)
std::vector< ConfigParamSpec > Options
DataProcessorSpec getNoiseCalibratorSpec(bool useDigits)
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
static constexpr o2::header::DataOrigin gDataOriginCDBWrapper
Definition Utils.h:44
static constexpr o2::header::DataOrigin gDataOriginCDBPayload
Definition Utils.h:43
static void trim(std::string &s)
Definition StringUtils.h:71
static std::vector< std::string > tokenize(const std::string &src, char delim, bool trimToken=true, bool skipEmpty=true)
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"
std::vector< Digit > digits