Project
Loading...
Searching...
No Matches
RawFileReader.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#ifndef DETECTOR_BASE_RAWFILEREADER_H
13#define DETECTOR_BASE_RAWFILEREADER_H
14
18
19#include <cstdio>
20#include <unordered_map>
21#include <map>
22#include <tuple>
23#include <vector>
24#include <string>
25#include <utility>
26#include <Rtypes.h>
28#include "Headers/DataHeader.h"
31
32namespace o2
33{
34namespace raw
35{
36
39
40struct ReaderInp {
41 std::string inifile{};
42 std::string rawChannelConfig{};
43 std::string dropTF{};
44 std::string metricChannel{};
45 std::string onlyDet{};
46 size_t spSize = 1024L * 1024L;
47 size_t bufferSize = 1024L * 1024L;
48 size_t minSHM = 0;
49 int loop = 1;
51 uint32_t errMap = 0xffffffff;
53 uint32_t maxTF = 0xffffffff;
54 bool partPerSP = true;
55 bool cache = false;
56 bool autodetectTF0 = false;
57 bool preferCalcTF = false;
58 bool sup0xccdb = false;
59};
60
62{
63 using LinkSpec_t = uint64_t; // = (origin<<32) | LinkSubSpec
64 public:
65 //================================================================================
68 static constexpr std::string_view CardNames[] = {"CRU", "RORC"};
69 //================================================================================
83
84 enum class FirstTFDetection : int { Disabled,
85 Pending,
86 Done };
87
88 static constexpr std::string_view ErrNames[] = {
89 // long names for error codes
90 "Wrong RDH.packetCounter increment", // ErrWrongPacketCounterIncrement
91 "Wrong RDH.pageCnt increment", // ErrWrongPageCounterIncrement
92 "RDH.stop set of 1st HBF page", // ErrHBFStopOnFirstPage
93 "New HBF starts w/o closing old one", // ErrHBFNoStop
94 "Data does not start with TF/HBF", // ErrWrongFirstPage
95 "Number of HBFs per TF not as expected", // ErrWrongHBFsPerTF
96 "Number of TFs is less than expected", // ErrWrongNumberOfTF
97 "Wrong HBF orbit increment", // ErrHBFJump
98 "TF does not start by new superpage", // ErrNoSuperPageForTF
99 "No SOX found on 1st page", // ErrNoSOX
100 "Mismatch between flagged and calculated new TF start" // ErrMismatchTF
101 };
102 static constexpr std::string_view ErrNamesShort[] = {
103 // short names for error codes
104 "packet-increment", // ErrWrongPacketCounterIncrement
105 "page-increment", // ErrWrongPageCounterIncrement
106 "stop-on-page0", // ErrHBFStopOnFirstPage
107 "missing-stop", // ErrHBFNoStop
108 "starts-with-tf", // ErrWrongFirstPage
109 "hbf-per-tf", // ErrWrongHBFsPerTF
110 "tf-per-link", // ErrWrongNumberOfTF
111 "hbf-jump", // ErrHBFJump
112 "no-spage-for-tf", // ErrNoSuperPageForTF
113 "no-sox", // ErrNoSOX
114 "tf-start-mismatch" // ErrMismatchTF
115 };
116 static constexpr bool ErrCheckDefaults[] = {
117 true, // ErrWrongPacketCounterIncrement
118 true, // ErrWrongPageCounterIncrement
119 false, // ErrHBFStopOnFirstPage
120 true, // ErrHBFNoStop
121 true, // ErrWrongFirstPage
122 true, // ErrWrongHBFsPerTF
123 true, // ErrWrongNumberOfTF
124 true, // ErrHBFJump
125 false, // ErrNoSuperPageForTF
126 false, // ErrNoSOX
127 true, // ErrMismatchTF
128 };
129 //================================================================================
130
133 using OrigDescCard = std::tuple<o2::header::DataOrigin, o2::header::DataDescription, ReadoutCardType>;
134 using InputsMap = std::map<OrigDescCard, std::vector<std::string>>;
135
136 //=====================================================================================
137
138 // reference on blocks making single message part
139 struct PartStat {
140 int size; // total size
141 int nBlocks; // number of consecutive LinkBlock objects
142 };
143
144 // info on the smallest block of data to be read when fetching the HBF
145 struct LinkBlock {
146 enum { StartTF = 0x1,
147 StartHB = 0x1 << 1,
148 StartSP = 0x1 << 2,
149 EndHB = 0x1 << 3 };
150 size_t offset = 0;
153 IR ir = 0;
156 std::unique_ptr<char[]> dataCache;
157 LinkBlock() = default;
158 LinkBlock(int fid, size_t offs) : offset(offs), fileID(fid) {}
159 void setFlag(uint8_t fl, bool v = true)
160 {
161 if (v) {
162 flags |= fl;
163 } else {
164 flags &= ~fl;
165 }
166 }
167 bool testFlag(uint8_t fl) const { return (flags & fl) == fl; }
168 void print(const std::string& pref = "") const;
169 };
170
171 //=====================================================================================
172 struct LinkData {
175 LinkSpec_t spec = 0;
181 bool cruDetector = true;
182 bool continuousRO = true;
183
186 int nErrors = 0;
187 std::vector<LinkBlock> blocks;
188 std::vector<std::pair<int, uint32_t>> tfStartBlock;
189 //
190 // transient info during processing
191 bool openHB = false;
192 int nHBFinTF = 0;
194
195 LinkData() = default;
196 template <typename H>
197 LinkData(const H& rdh, RawFileReader* r) : rdhl(rdh), reader(r)
198 {
199 }
200 bool preprocessCRUPage(const RDHAny& rdh, bool newSPage);
201 size_t getLargestSuperPage() const;
202 size_t getLargestTF() const;
203 size_t getNextHBFSize() const;
204 size_t getNextTFSize() const;
205 size_t getNextTFSuperPagesStat(std::vector<PartStat>& parts) const;
206 int getNHBFinTF() const;
207
208 size_t readNextHBF(char* buff);
209 size_t readNextTF(char* buff);
210 size_t readNextSuperPage(char* buff, const PartStat* pstat = nullptr);
211 size_t skipNextHBF();
212 size_t skipNextTF();
213
214 bool rewindToTF(uint32_t tf);
215 void print(bool verbose = false, const std::string& pref = "") const;
216 std::string describe() const;
217
218 private:
219 RawFileReader* reader = nullptr;
220 };
221
222 //=====================================================================================
223
224 RawFileReader(const std::string& config = "", int verbosity = 0, size_t buffsize = 50 * 1024UL, const std::string& onlyDet = {});
226
227 void loadFromInputsMap(const InputsMap& inp);
228 bool init();
229 void clear();
231 bool addFile(const std::string& sname) { return addFile(sname, mDefDataOrigin, mDefDataDescription, mDefCardType); }
232 void setDefaultReadoutCardType(ReadoutCardType t = CRU) { mDefCardType = t; }
233 void setDefaultDataOrigin(const std::string& orig) { mDefDataOrigin = getDataOrigin(orig); }
234 void setDefaultDataDescription(const std::string& desc) { mDefDataDescription = getDataDescription(desc); }
235 void setDefaultDataOrigin(const o2::header::DataOrigin o) { mDefDataOrigin = o; }
236 void setDefaultDataDescription(const o2::header::DataDescription d) { mDefDataDescription = d; }
237 int getNLinks() const { return mLinksData.size(); }
238 int getNFiles() const { return mFiles.size(); }
239
240 uint32_t getNextTFToRead() const { return mNextTF2Read; }
241 void setNextTFToRead(uint32_t tf) { mNextTF2Read = tf; }
242
243 const std::vector<int>& getLinksOrder() const { return mOrderedIDs; }
244 const LinkData& getLink(int i) const { return mLinksData[mOrderedIDs[i]]; }
245 const LinkData& getLinkWithSpec(LinkSpec_t s) const { return mLinksData[mLinkEntries.at(s)]; }
246 LinkData& getLink(int i) { return mLinksData[mOrderedIDs[i]]; }
248 LinkSpec_t getLinkSpec(int i) const { return getLink(i).spec; }
249
250 void printStat(bool verbose = false) const;
251
252 void setVerbosity(int v = 1) { mVerbosity = v; }
253 void setCheckErrors(uint32_t m = 0xffffffff) { mCheckErrors = m & ((0x1 << NErrorsDefined) - 1); }
254 int getVerbosity() const { return mVerbosity; }
255 uint32_t getCheckErrors() const { return mCheckErrors; }
256 bool isProcessingStopped() const { return mStopProcessing; }
257
258 void setNominalSPageSize(int n = 0x1 << 20) { mNominalSPageSize = n > (0x1 << 15) ? n : (0x1 << 15); }
259 int getNominalSPageSize() const { return mNominalSPageSize; }
260
261 void setBufferSize(size_t s) { mBufferSize = s < sizeof(RDHAny) ? sizeof(RDHAny) : s; }
262 size_t getBufferSize() const { return mBufferSize; }
263
264 void setMaxTFToRead(uint32_t n) { mMaxTFToRead = n; }
265 bool isEmpty() const { return mEmpty; }
266 uint32_t getMaxTFToRead() const { return mMaxTFToRead; }
267 uint32_t getNTimeFrames() const { return mNTimeFrames; }
268 uint32_t getOrbitMin() const { return mOrbitMin; }
269 uint32_t getOrbitMax() const { return mOrbitMax; }
270
271 bool getCacheData() const { return mCacheData; }
272 void setCacheData(bool v) { mCacheData = v; }
273
274 o2::header::DataOrigin getDefaultDataOrigin() const { return mDefDataOrigin; }
275 o2::header::DataDescription getDefaultDataSpecification() const { return mDefDataDescription; }
276 ReadoutCardType getDefaultReadoutCardType() const { return mDefCardType; }
277
279 void setTFAutodetect(FirstTFDetection v) { mFirstTFAutodetect = v; }
280 void setPreferCalculatedTFStart(bool v) { mPreferCalculatedTFStart = v; }
281 FirstTFDetection getTFAutodetect() const { return mFirstTFAutodetect; }
282
284
285 static o2::header::DataOrigin getDataOrigin(const std::string& ors);
286 static o2::header::DataDescription getDataDescription(const std::string& ors);
287 static InputsMap parseInput(const std::string& confUri, const std::string& onlyDet = {}, bool verbose = false);
288 static std::string nochk_opt(ErrTypes e);
289 static std::string nochk_expl(ErrTypes e);
290
291 private:
292 int getLinkLocalID(const RDHAny& rdh, int fileID);
293 bool preprocessFile(int ifl);
294 static LinkSpec_t createSpec(o2::header::DataOrigin orig, LinkSubSpec_t ss) { return (LinkSpec_t(orig) << 32) | ss; }
295
296 static constexpr o2::header::DataOrigin DEFDataOrigin = o2::header::gDataOriginFLP;
297 static constexpr o2::header::DataDescription DEFDataDescription = o2::header::gDataDescriptionRawData;
298 static constexpr ReadoutCardType DEFCardType = CRU;
299 o2::header::DataOrigin mDefDataOrigin = DEFDataOrigin;
300 o2::header::DataDescription mDefDataDescription = DEFDataDescription;
301 ReadoutCardType mDefCardType = CRU;
302 std::vector<std::string> mFileNames;
303 std::vector<FILE*> mFiles;
304 std::vector<std::unique_ptr<char[]>> mFileBuffers;
305 std::vector<OrigDescCard> mDataSpecs;
306 bool mInitDone = false;
307 bool mEmpty = true;
308 std::unordered_map<LinkSpec_t, int> mLinkEntries;
309 std::vector<LinkData> mLinksData;
310 std::vector<int> mOrderedIDs;
311 uint32_t mMaxTFToRead = 0xffffffff;
312 uint32_t mNTimeFrames = 0;
313 uint32_t mNextTF2Read = 0;
314 uint32_t mOrbitMin = 0xffffffff;
315 uint32_t mOrbitMax = 0;
316 size_t mBufferSize = 5 * 1024UL;
317 int mNominalSPageSize = 0x1 << 20;
318 int mCurrentFileID = 0;
319 long int mPosInFile = 0;
320 bool mMultiLinkFile = false;
321 bool mCacheData = false;
322 bool mStopProcessing = false;
323 uint32_t mCheckErrors = 0;
324 FirstTFDetection mFirstTFAutodetect = FirstTFDetection::Disabled;
325 bool mPreferCalculatedTFStart = false;
326 int mVerbosity = 0;
327 ClassDefNV(RawFileReader, 1);
328};
329
330} // namespace raw
331} // namespace o2
332
333#endif //DETECTOR_BASE_RAWFILEREADER_H
#define verbosity
header::DataOrigin origin
uint64_t orbit
Definition RawEventData.h:6
void print() const
unsigned short int uint16_t
int32_t i
o2::raw::RawFileWriter * raw
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char e
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events d
Definition of the RAW Data Header.
Static class with identifiers, bitmasks and names for ALICE detectors.
Definition DetID.h:60
const std::vector< int > & getLinksOrder() const
uint32_t getOrbitMax() const
void printStat(bool verbose=false) const
uint32_t getNextTFToRead() const
void setVerbosity(int v=1)
static InputsMap parseInput(const std::string &confUri, const std::string &onlyDet={}, bool verbose=false)
ReadoutCardType getDefaultReadoutCardType() const
std::tuple< o2::header::DataOrigin, o2::header::DataDescription, ReadoutCardType > OrigDescCard
void setNextTFToRead(uint32_t tf)
void setDefaultDataDescription(const std::string &desc)
static std::string nochk_expl(ErrTypes e)
void setTFAutodetect(FirstTFDetection v)
FirstTFDetection getTFAutodetect() const
void setBufferSize(size_t s)
void setDefaultDataOrigin(const std::string &orig)
std::map< OrigDescCard, std::vector< std::string > > InputsMap
void setNominalSPageSize(int n=0x1<< 20)
uint32_t getMaxTFToRead() const
void setDefaultDataDescription(const o2::header::DataDescription d)
bool addFile(const std::string &sname)
static constexpr std::string_view CardNames[]
LinkSubSpec_t getLinkSubSpec(int i) const
void setDefaultDataOrigin(const o2::header::DataOrigin o)
uint32_t getCheckErrors() const
void loadFromInputsMap(const InputsMap &inp)
int getNominalSPageSize() const
void setMaxTFToRead(uint32_t n)
bool isProcessingStopped() const
LinkSpec_t getLinkSpec(int i) const
size_t getBufferSize() const
static constexpr bool ErrCheckDefaults[]
static constexpr std::string_view ErrNamesShort[]
uint32_t getNTimeFrames() const
static o2::header::DataDescription getDataDescription(const std::string &ors)
static std::string nochk_opt(ErrTypes e)
void setCheckErrors(uint32_t m=0xffffffff)
const LinkData & getLink(int i) const
LinkData & getLink(int i)
void setIROfSOX(const o2::InteractionRecord &ir)
void setDefaultReadoutCardType(ReadoutCardType t=CRU)
o2::header::DataDescription getDefaultDataSpecification() const
static o2::header::DataOrigin getDataOrigin(const std::string &ors)
const LinkData & getLinkWithSpec(LinkSpec_t s) const
void imposeFirstTF(uint32_t orbit)
static constexpr std::string_view ErrNames[]
uint32_t getOrbitMin() const
o2::header::DataOrigin getDefaultDataOrigin() const
bool addFile(const std::string &sname, o2::header::DataOrigin origin, o2::header::DataDescription desc, ReadoutCardType t=CRU)
void setPreferCalculatedTFStart(bool v)
GLdouble n
Definition glcorearb.h:1982
const GLfloat * m
Definition glcorearb.h:4066
GLsizeiptr size
Definition glcorearb.h:659
const GLdouble * v
Definition glcorearb.h:832
GLintptr offset
Definition glcorearb.h:660
constexpr o2::header::DataDescription gDataDescriptionInvalid
Definition DataHeader.h:600
constexpr o2::header::DataOrigin gDataOriginFLP
Definition DataHeader.h:565
constexpr o2::header::DataOrigin gDataOriginInvalid
Definition DataHeader.h:564
constexpr o2::header::DataDescription gDataDescriptionRawData
Definition DataHeader.h:601
RAWDataHeaderV7 RAWDataHeader
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
std::unique_ptr< GPUReconstructionTimeframe > tf
bool testFlag(uint8_t fl) const
std::unique_ptr< char[]> dataCache
different flags
void setFlag(uint8_t fl, bool v=true)
LinkBlock()=default
optional cache for fast access
uint16_t fileID
ir starting the block
LinkBlock(int fid, size_t offs)
bool preprocessCRUPage(const RDHAny &rdh, bool newSPage)
LinkSubSpec_t subspec
Link subspec augmented by its origin.
o2::header::DataOrigin origin
size_t readNextSuperPage(char *buff, const PartStat *pstat=nullptr)
uint32_t nTimeFrames
subspec according to DataDistribution
o2::header::DataDescription description
o2::InteractionRecord irOfSOX
RDH with the running info of the last RDH seen.
LinkData(const H &rdh, RawFileReader *r)
std::vector< LinkBlock > blocks
size_t getNextTFSuperPagesStat(std::vector< PartStat > &parts) const
LinkData()=default
next block which should be read
std::vector< std::pair< int, uint32_t > > tfStartBlock
bool continuousRO
CRU vs RORC detector.
std::string rawChannelConfig
std::string inifile
std::string onlyDet
std::string metricChannel
o2::InteractionRecord ir(0, 0)