Project
Loading...
Searching...
No Matches
DataInputDirector.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11#ifndef O2_FRAMEWORK_DATAINPUTDIRECTOR_H_
12#define O2_FRAMEWORK_DATAINPUTDIRECTOR_H_
13
14#include "TFile.h"
15
19#include "AODInputFile.h"
21
22#include <arrow/filesystem/filesystem.h>
23#include <arrow/dataset/dataset.h>
24
25#include <map>
26#include <regex>
27#include <stdexcept>
28#include <vector>
29#include "rapidjson/fwd.h"
30
32{
33class Monitoring;
34}
35
36namespace o2::framework
37{
38
39class AODReadError : public std::runtime_error
40{
41 public:
42 enum class Kind {
43 InvalidTimeframe, // skip the timeframe
44 FileOpen, // skip the file, its timeframes are unknown
45 };
46
47 AODReadError(Kind kind, std::string const& what) : std::runtime_error(what), mKind(kind) {}
48 Kind getKind() const { return mKind; }
49
50 private:
51 Kind mKind;
52};
53
55 std::string fileName;
57 std::vector<uint64_t> listOfTimeFrameNumbers;
58 std::vector<bool> alreadyRead;
60};
61
62FileNameHolder makeFileNameHolder(std::string fileName);
63
65 o2::monitoring::Monitoring* monitoring = nullptr;
67 std::string parentFileReplacement = "";
68 std::vector<std::pair<std::string, int>> parentLevelToOrigin = {};
69 // Optional registry of pre-opened TFiles (keyed by name) used to bypass
70 // TFile::Open for testing with in-memory TMemFile instances.
71 std::vector<std::pair<std::string, TFile*>> openFiles = {};
72 // Which reader opens the files: "root", "native", or "auto" (native, with a
73 // per-file fallback to ROOT for what the native reader cannot read).
74 std::string readerBackend = "root";
76
77 int levelForOrigin(std::string_view origin) const
78 {
79 for (auto& [o, level] : parentLevelToOrigin) {
80 if (o == origin) {
81 return level;
82 }
83 }
84 return -1;
85 }
86};
87
89{
93
94 public:
95 std::string tablename = "";
96 std::string treename = "";
97 std::shared_ptr<data_matcher::DataDescriptorMatcher> matcher;
98
99 DataInputDescriptor(bool alienSupport, int level, DataInputDirectorContext& context);
101
102 void printOut() const;
103
104 // setters
105 void setInputfilesFile(std::string dffn) { minputfilesFile = dffn; }
106 void setInputfilesFile(std::string* dffnptr) { minputfilesFilePtr = dffnptr; }
107 void setFilenamesRegex(std::string fn) { mFilenameRegex = fn; }
108 void setFilenamesRegex(std::string* fnptr) { mFilenameRegexPtr = fnptr; }
109
110 void setDefaultInputfiles(std::vector<FileNameHolder> difnptr) { mdefaultFilenamesPtr = difnptr; }
111
113 int fillInputfiles();
114 bool setFile(int counter, int wantedParentLevel, std::string_view wantedOrigin);
115
116 // getters
117 std::string getInputfilesFilename() const;
118 std::string getFilenamesRegexString() const;
119 std::regex getFilenamesRegex();
120 int getNumberInputfiles() { return mfilenames.size(); }
121 int getNumberTimeFrames() const { return mtotalNumberTimeFrames; }
122 int findDFNumber(int file, std::string dfName);
123
124 uint64_t getTimeFrameNumber(int counter, int numTF, int wantedParentLevel, std::string_view wantedOrigin);
125 arrow::dataset::FileSource getFileFolder(int counter, int numTF, int wantedParentLevel, std::string_view wantedOrigin);
126 // Start tracking the file/DF pairs participating in one output timeframe.
127 void beginTimeFrame();
128 // Commit or discard a timeframe, then close retained parents and report statistics.
129 void finishTimeFrame(bool skipped = false);
130 // Open the current file to populate the parent map, then return the parent descriptor and
131 // the TF index within it that corresponds to numTF at this level. Returns {nullptr, -1} on failure.
132 std::pair<std::shared_ptr<DataInputDescriptor>, int> navigateToLevel(int counter, int numTF, int wantedParentLevel, std::string_view wantedOrigin);
133 std::shared_ptr<DataInputDescriptor> getParentFile(int counter, int numTF, std::string treename, int wantedParentLevel, std::string_view wantedOrigin);
136
137 bool readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, std::string treename, size_t& totalSizeCompressed, size_t& totalSizeUncompressed);
138
139 void printFileOpening();
140 void printFileStatistics();
141 void closeInputFile();
142 bool isAlienSupportOn() { return mAlienSupport; }
143
144 private:
145 void recordTimeFrameRead(int counter, int numTF);
146 void releaseParentFile();
147 void reportFileStatistics(int counter, std::string monitoringInfo);
148
150 std::string minputfilesFile;
151 std::string* minputfilesFilePtr = nullptr;
152 std::string mFilenameRegex;
153 std::string* mFilenameRegexPtr = nullptr;
154 std::vector<FileNameHolder> mfilenames;
155 std::vector<FileNameHolder> mdefaultFilenamesPtr;
156 std::shared_ptr<AODInputFile> mCurrentFile;
157 std::shared_ptr<arrow::fs::FileSystem> mCurrentFilesystem;
158 int mCurrentFileID = -1;
159 // File-local DF indices visited during the current output timeframe.
160 std::vector<std::pair<int, int>> mTimeFrameReads;
161 bool mAlienSupport = false;
162
163 DataInputDirectorContext& mContext;
164 // folder name (DF_XXX) --> parent file (absolute path), empty if the file has none
165 std::map<std::string, std::string> mParentFileMap;
166 std::shared_ptr<DataInputDescriptor> mParentFile = nullptr;
167 std::vector<std::shared_ptr<DataInputDescriptor>> mRetainedParents;
168 bool mTimeFrameActive = false;
169 std::vector<std::pair<int, std::string>> mPendingFileStatistics;
170 int mLevel = 0; // level of parent files
171
172 int mtotalNumberTimeFrames = 0;
173
174 uint64_t mIOTime = 0;
175 uint64_t mCurrentFileStartedAt = 0;
176};
177
179{
183
184 public:
185 DataInputDirector(std::vector<std::string> inputFiles, DataInputDirectorContext&& context);
187
188 void reset();
189 void printOut();
190 bool atEnd(int counter);
191
192 // setters
193 void setInputfilesFile(std::string iffn) { minputfilesFile = iffn; }
194 void setFilenamesRegex(std::string dfn) { mFilenameRegex = dfn; }
195 bool readJson(std::string const& fnjson);
196 void closeInputFiles();
197
198 // getters
200 int getNumberInputDescriptors() { return mdataInputDescriptors.size(); }
202
203 bool readTree(DataAllocator& outputs, header::DataHeader dh, int counter, int numTF, size_t& totalSizeCompressed, size_t& totalSizeUncompressed, bool wasAOD);
205 arrow::dataset::FileSource getFileFolder(header::DataHeader dh, int counter, int numTF);
206 // Start tracking the file/DF pairs participating in one output timeframe.
207 void beginTimeFrame();
208 // Commit or discard a timeframe, then close retained parents and report statistics.
209 void finishTimeFrame(bool skipped = false);
211
214
216
217 private:
219 std::string minputfilesFile;
220 std::string* const minputfilesFilePtr = &minputfilesFile;
221 std::string mFilenameRegex;
222 std::string* const mFilenameRegexPtr = &mFilenameRegex;
223 std::shared_ptr<DataInputDescriptor> mdefaultDataInputDescriptor = nullptr;
224 std::vector<FileNameHolder> mdefaultInputFiles;
225 std::vector<DataInputDescriptor> mdataInputDescriptors;
226
227 bool mDebugMode = false;
228 bool mAlienSupport = false;
229
230 bool readJsonDocument(rapidjson::Document* doc);
231 bool isValid();
232};
233
234} // namespace o2::framework
235
236#endif // O2_FRAMEWORK_DATAINPUTDIRECTOR_H_
header::DataOrigin origin
configStandalone Light OpenGL display display_light Apply smoothing to points Enable Z buffer Highlight links Highlight seeds Highlight sector tracks Highlight final tracks Show individual Show related Highlight grid Show only individual collision Color clusters belonging to track objects Set line Enable drawing TPC data Enable drawing TOF data Enable drawing magnetic field Set field line step count Invert colors configStandalone Heavy OpenGL display display_heavy Show tracks with ITS contribution Show tracks with TOF contribution Enable propagation of loopers Show clusters at nominal x of pad row for early transformed data Split CE tracks when they cross the central Mark clusters Mark adjacent clusters Hide rejected tracks Apply filter on tracks to be displayed Separate increase X h configStandalone Camera window renderer OpenGL display display_camera Orient the camera such that the y axis is always upwards Full Screen MultiSample Anti Downsampling Anti Enable Vertical Sync Target frame or if the backend cannot limit to the refresh Use OpenGL indirect draws to reduce number of draw calls Display h configStandalone Vulkan display display_vulkan Have uniform buffers in host accessible device memory h configStandalone Internal display display_internal Number of updates the Qt GUI may send ahead of the last one the display has before it waits for the display Opacity of the console background Number of console lines the history is shortened to this when it reaches twice the size Time in seconds an info message fades out after setInfoTime h OpenGL display display Font size Do not use Freetype for font Renderer for event Frontend to use for event ROOT macros used as track filter Console script file
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
o2::monitoring::Monitoring Monitoring
AODReadError(Kind kind, std::string const &what)
uint64_t getTimeFrameNumber(int counter, int numTF, int wantedParentLevel, std::string_view wantedOrigin)
std::shared_ptr< DataInputDescriptor > getParentFile(int counter, int numTF, std::string treename, int wantedParentLevel, std::string_view wantedOrigin)
bool readTree(DataAllocator &outputs, header::DataHeader dh, int counter, int numTF, std::string treename, size_t &totalSizeCompressed, size_t &totalSizeUncompressed)
arrow::dataset::FileSource getFileFolder(int counter, int numTF, int wantedParentLevel, std::string_view wantedOrigin)
void setDefaultInputfiles(std::vector< FileNameHolder > difnptr)
std::shared_ptr< data_matcher::DataDescriptorMatcher > matcher
void setInputfilesFile(std::string dffn)
void setInputfilesFile(std::string *dffnptr)
void setFilenamesRegex(std::string *fnptr)
void addFileNameHolder(FileNameHolder fn)
void finishTimeFrame(bool skipped=false)
bool setFile(int counter, int wantedParentLevel, std::string_view wantedOrigin)
int findDFNumber(int file, std::string dfName)
DataInputDescriptor(DataInputDescriptor const &)=default
std::pair< std::shared_ptr< DataInputDescriptor >, int > navigateToLevel(int counter, int numTF, int wantedParentLevel, std::string_view wantedOrigin)
DataInputDescriptor * getDataInputDescriptor(header::DataHeader dh)
arrow::dataset::FileSource getFileFolder(header::DataHeader dh, int counter, int numTF)
void setInputfilesFile(std::string iffn)
int getTimeFramesInFile(header::DataHeader dh, int counter)
uint64_t getTimeFrameNumber(header::DataHeader dh, int counter, int numTF)
int getLevelForOrigin(header::DataOrigin origin) const
void setFilenamesRegex(std::string dfn)
void finishTimeFrame(bool skipped=false)
bool readJson(std::string const &fnjson)
bool readTree(DataAllocator &outputs, header::DataHeader dh, int counter, int numTF, size_t &totalSizeCompressed, size_t &totalSizeUncompressed, bool wasAOD)
GLint level
Definition glcorearb.h:275
Defining ITS Vertex explicitly as messageable.
Definition Cartesian.h:288
RuntimeErrorRef runtime_error(const char *)
FileNameHolder makeFileNameHolder(std::string fileName)
o2::monitoring::Monitoring * monitoring
std::vector< std::pair< std::string, int > > parentLevelToOrigin
std::vector< std::pair< std::string, TFile * > > openFiles
int levelForOrigin(std::string_view origin) const
std::vector< uint64_t > listOfTimeFrameNumbers
the main header struct
Definition DataHeader.h:623