Project
Loading...
Searching...
No Matches
BinFileOp.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12// @brief Polimorphic class to access either local or grid files with fixed sed ot methods
13
15#include "Framework/Logger.h"
17#include <sys/mman.h>
18#include <TFileCacheRead.h>
19
20namespace o2::rawdd
21{
22
23//_____________________________________________________________________
24BinFileOp* BinFileOp::open(const std::string& name)
25{
26 BinFileOp* ptr = o2::utils::Str::beginsWith(name, "alien://") ? static_cast<BinFileOp*>(new BinFileOpGrid(name)) : static_cast<BinFileOp*>(new BinFileOpLocal(name));
27 return ptr->isGood() ? ptr : nullptr;
28}
29
30//_____________________________________________________________________
32{
34 if (!mFileMap.is_open()) {
35 LOG(error) << "Failed to open TF file for reading (mmap).";
36 return;
37 }
39 mFileOffset = 0;
40
41#if __linux__
42 madvise((void*)mFileMap.data(), mFileMap.size(), MADV_HUGEPAGE | MADV_SEQUENTIAL | MADV_DONTDUMP);
43#endif
44}
45
47{
48 if (!mFileMap.is_open()) {
49#if __linux__
50 madvise((void*)mFileMap.data(), mFileMap.size(), MADV_DONTNEED);
51#endif
52 mFileMap.close();
53 }
54}
55
57{
58 if (!mFileMap.is_open()) {
59 return false;
60 }
61 assert(mFileOffset <= mFileSize);
62 const size_t lToRead = std::min(len, mFileSize - mFileOffset);
63 if (lToRead != len) {
64 LOGP(error, "BinFileOpLocal: request to read beyond the file end. pos={} size={} len={}, closing the file {}", mFileOffset, mFileSize, len, mFileName);
65 mFileMap.close();
66 mFileOffset = 0;
67 mFileSize = 0;
68 return false;
69 }
70 std::memcpy(reinterpret_cast<char*>(ptr), mFileMap.data() + mFileOffset, lToRead);
71 mFileOffset += lToRead;
72 return true;
73}
74
75unsigned char* BinFileOpLocal::bufferize(size_t& s)
76{
77 if (s > MaxBuffSize) {
78 LOGP(fatal, "Requested buffer size {} exceeds max allowed {}", s, MaxBuffSize);
79 }
80 s = std::min(s, distance_to_eof());
82 return const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(mFileMap.data() + mFileOffset));
83}
84
86{
87 const size_t lToIgnore = std::min(len, std::size_t(mFileSize - mFileOffset));
88 if (len != lToIgnore) {
89 LOGP(error, "BinFileOpLocal: request to ignore bytes beyond the file end. pos={} size={} len={}, closing the file {}", mFileOffset, mFileSize, len, mFileName);
90 mFileMap.close();
91 mFileOffset = 0;
92 mFileSize = 0;
93 return false;
94 }
95 mFileOffset += lToIgnore;
96 assert(mFileOffset <= mFileSize);
97 return true;
98}
99
100//_____________________________________________________________________
102{
103 mFile.reset(TFile::Open(fmt::format("{}?filetype=raw", name).c_str()));
104 if (!isGood()) {
105 LOGP(error, "Failed to open file {} for reading.", name);
106 return;
107 }
108 mFileSize = mFile->GetSize();
109 set_position(0);
110 mBuffer.reserve(MaxBuffSize);
111}
112
114{
115 assert(pos <= mFileSize);
116 mFileOffset = std::min(pos, mFileSize);
117 mFile->Seek(mFileOffset);
118}
119
121{
122 if (!isGood()) {
123 return false;
124 }
125 assert(mFileOffset <= mFileSize);
126 const size_t lToRead = std::min(len, distance_to_eof());
127 if (lToRead != len) {
128 LOGP(error, "BinFileOpGrid: request to read beyond the file end. pos={} size={} len={}, closing the file {}", mFileOffset, mFileSize, len, mFileName);
129 mFile.reset();
130 mFileOffset = 0;
131 mFileSize = 0;
132 return false;
133 }
134
135 if (len < MaxBuffSize) {
136 auto pos = position();
137 LOGP(debug, "read_advance(fast) {} from {}", lToRead, position());
138 void* lptr = bufferize(len);
139 std::memcpy(ptr, lptr, len);
141 } else { // too large chunk to bufferize, read directly
142 LOGP(debug, "read_advance(slow) {} from {}", lToRead, position());
143 if (mFile->ReadBuffer(reinterpret_cast<char*>(ptr), lToRead)) {
144 LOGP(error, "BinFileOpGrid: failed to read {} bytes from position {} of file {}, closing it", lToRead, mFileOffset, mFileName);
145 }
146 mFileOffset += lToRead;
147 }
148 return true;
149}
150
151unsigned char* BinFileOpGrid::bufferize(size_t& s)
152{
153 if (s > MaxBuffSize) {
154 LOGP(fatal, "Requested buffer size {} exceeds max allowed {}", s, MaxBuffSize);
155 }
156 if (mBufferizedPos <= position() && mBufferizedPos + mBuffer.size() >= position() + s) {
157 LOGP(debug, "bufferize(fast) {} from {}", s, position());
158 return mBuffer.data() + (position() - mBufferizedPos);
159 }
160 s = std::min(distance_to_eof(), s);
162 mBuffer.resize(std::min(MaxBuffSize, distance_to_eof()));
163 if (!mFile->ReadBuffer((char*)mBuffer.data(), mBuffer.size())) {
164 LOGP(debug, "bufferize(slow) {} from {}", s, position());
165 set_position(mBufferizedPos); // go back
166 return mBuffer.data();
167 }
168 mBuffer.clear();
169 LOGP(error, "BinFileOpGrid:bufferize failed to read {} bytes from position {} of file {}, closing it", s, mFileOffset, mFileName);
171 mBufferizedPos = -1UL;
172 s = 0;
173 return nullptr;
174}
175
177{
178 const size_t lToIgnore = std::min(len, distance_to_eof());
179 if (len != lToIgnore) {
180 LOGP(error, "BinFileOpGrid: request to ignore bytes beyond the file end. pos={} size={} len={}, closing the file {}", mFileOffset, mFileSize, len, mFileName);
181 mFile.reset();
182 mFileOffset = 0;
183 mFileSize = 0;
184 return false;
185 }
186 set_position(mFileOffset + lToIgnore);
187 return true;
188}
189
190} // namespace o2::rawdd
std::ostringstream debug
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
char error[CURL_ERROR_SIZE]
uint16_t pos
Definition RawData.h:3
TBranch * ptr
bool ignore_nbytes(size_t len) override
std::vector< unsigned char > mBuffer
Definition BinFileOp.h:103
size_t position() const
Definition BinFileOp.h:98
unsigned char * bufferize(size_t &s) override
std::unique_ptr< TFile > mFile
Definition BinFileOp.h:102
bool isGood() const override
Definition BinFileOp.h:95
bool read_advance(void *ptr, size_t len) override
void set_position(size_t pos) override
BinFileOpGrid(const std::string &name)
bool read_advance(void *ptr, size_t len) override
Definition BinFileOp.cxx:56
bool ignore_nbytes(size_t len) override
Definition BinFileOp.cxx:85
size_t position() const
Definition BinFileOp.h:76
boost::iostreams::mapped_file_source mFileMap
Definition BinFileOp.h:80
BinFileOpLocal(const std::string &name)
Definition BinFileOp.cxx:31
unsigned char * bufferize(size_t &s) override
Definition BinFileOp.cxx:75
static constexpr size_t MaxBuffSize
Definition BinFileOp.h:30
size_t distance_to_eof() const
Definition BinFileOp.h:42
std::string mFileName
Definition BinFileOp.h:50
static BinFileOp * open(const std::string &name)
Definition BinFileOp.cxx:24
virtual bool isGood() const =0
size_t size() const
Definition BinFileOp.h:40
GLuint const GLchar * name
Definition glcorearb.h:781
GLenum GLenum GLsizei len
Definition glcorearb.h:4232
static bool beginsWith(const std::string &s, const std::string &start)
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"