Project
Loading...
Searching...
No Matches
FileFetcher.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
18#include "Framework/Logger.h"
19#include <filesystem>
20#include <fmt/chrono.h>
21#include <fstream>
22#include <memory>
23#include <thread>
24#include <chrono>
25#include <cstdlib>
26#include <ctime>
27#include <locale>
28#include <TGrid.h>
29#include <TSystem.h>
30
31using namespace o2::utils;
32using namespace std::chrono_literals;
33namespace fs = std::filesystem;
34
35//____________________________________________________________
36FileFetcher::FileFetcher(const std::string& input, const std::string& selRegex, const std::string& remRegex,
37 const std::string& copyCmd, const std::string& copyDir)
38 : mCopyCmd(copyCmd), mCopyDirName(copyDir)
39{
40 if (!selRegex.empty()) {
41 mSelRegex = std::make_unique<std::regex>(selRegex.c_str());
42 }
43 if (!remRegex.empty()) {
44 mRemRegex = std::make_unique<std::regex>(remRegex);
45 }
46 mNoRemoteCopy = mCopyCmd == "no-copy";
47
48 // parse input list
49 mCopyDirName = o2::utils::Str::create_unique_path(mCopyDirName, 8);
50 processInput(input);
51 LOGP(info, "Input contains {} files, {} remote", getNFiles(), mNRemote);
52 if (mNRemote) {
53 if (mNoRemoteCopy) { // make sure the copy command is provided, unless copy was explicitly forbidden
54 LOGP(info, "... but their local copying is explicitly forbidden");
55 } else {
56 if (mCopyCmd.find("?src") == std::string::npos || mCopyCmd.find("?dst") == std::string::npos) {
57 throw std::runtime_error(fmt::format("remote files asked but copy cmd \"{}\" is not valid", mCopyCmd));
58 }
59 try {
61 } catch (...) {
62 throw std::runtime_error(fmt::format("failed to create scratch directory {}", mCopyDirName));
63 }
64 mCopyCmdLogFile = fmt::format("{}/{}", mCopyDirName, "copy-cmd.log");
65 LOGP(info, "FileFetcher tmp scratch directory is set to {}", mCopyDirName);
66 }
67 }
68}
69
70//____________________________________________________________
76
77//____________________________________________________________
78void FileFetcher::processInput(const std::string& input)
79{
80 auto ftokens = o2::utils::Str::tokenize(input, ',', true); // in case multiple inputs are provided
81 processInput(ftokens);
82}
83
84//____________________________________________________________
85void FileFetcher::processInput(const std::vector<std::string>& input)
86{
87 for (auto inp : input) {
89 if (fs::is_directory(inp)) {
90 processDirectory(inp);
91 } else if (mSelRegex && !std::regex_match(inp, *mSelRegex.get())) { // provided selector does not match, treat as a txt file with list
92 // Avoid reading a multigiB data file as a list of inputs
93 // bringing down the system.
94 std::filesystem::path p(inp);
95 if (std::filesystem::file_size(p) > 10000000) {
96 LOGP(error, "file list {} larger than 10MB. Is this a data file?", inp);
97 continue;
98 }
99
100 std::ifstream listFile(inp);
101 if (!listFile.good()) {
102 LOGP(error, "file {} pretends to be a list of inputs but does not exist", inp);
103 continue;
104 }
105 std::string line;
106 std::vector<std::string> newInput;
107 while (getline(listFile, line)) {
109 if (line[0] == '#' || line.empty()) { // ignore commented file or empty line
110 continue;
111 }
112 newInput.push_back(line);
113 }
114 processInput(newInput);
115 } else { // should be local or remote data file
116 addInputFile(inp);
117 }
118 }
119}
120
121//____________________________________________________________
122void FileFetcher::processDirectory(const std::string& name)
123{
124 std::vector<std::string> vs;
125 for (auto const& entry : fs::directory_iterator{name}) {
126 const auto& fnm = entry.path().native();
127 if (fs::is_regular_file(fnm) && (!mSelRegex || std::regex_match(fnm, *mSelRegex.get()))) {
128 vs.push_back(fnm);
129 }
130 }
131 std::sort(vs.begin(), vs.end());
132 for (const auto& s : vs) {
133 addInputFile(s); // local files only
134 }
135}
136
137//____________________________________________________________
138bool FileFetcher::addInputFile(const std::string& fname)
139{
140 static bool alienErrorPrinted = false;
141 if (mRemRegex && std::regex_match(fname, *mRemRegex.get())) {
142 mInputFiles.emplace_back(FileRef{fname, mNoRemoteCopy ? fname : createCopyName(fname), true, false});
143 if (fname.find("alien:") == 0) {
144 if (!gGrid && !TGrid::Connect("alien://") && !alienErrorPrinted) {
145 LOG(error) << "File name starts with alien but connection to Grid failed";
146 alienErrorPrinted = true;
147 }
148 }
149 mNRemote++;
150 } else if (fs::exists(fname)) { // local file
151 mInputFiles.emplace_back(FileRef{fname, "", false, false});
152 } else {
153 LOGP(error, "file {} pretends to be local but does not exist", fname);
154 return false;
155 }
156 return true;
157}
158
159//____________________________________________________________
160std::string FileFetcher::createCopyName(const std::string& fname) const
161{
162 std::string cpnam{}, cpnamP = fname;
163 for (auto& c : cpnamP) {
164 if (!std::isalnum(c) && c != '.' && c != '-') {
165 c = '_';
166 }
167 }
168 while (1) {
169 cpnam = fmt::format("{}/{}_{}", mCopyDirName, o2::utils::Str::getRandomString(12), cpnamP);
170 if (!fs::exists(cpnam)) {
171 break;
172 }
173 }
174 return cpnam;
175}
176
177//____________________________________________________________
178size_t FileFetcher::popFromQueue(bool discard)
179{
180 // remove file from the queue, if requested and if it was copied, remove copy
181 std::lock_guard<std::mutex> lock(mMtx);
182 const auto* ptr = mQueue.frontPtr();
183 if (mQueue.empty()) {
184 return -1ul;
185 }
186 auto id = mQueue.front();
187 mQueue.pop();
188 LOGP(info, "removing processed {} from the queue, discard flag is {}", mInputFiles[id].getLocalName(), discard);
189 if (discard) {
190 discardFile(mInputFiles[id].getLocalName());
191 }
192 return id;
193}
194
195//____________________________________________________________
196size_t FileFetcher::nextInQueue() const
197{
198 return mQueue.empty() ? -1ul : mQueue.front();
199}
200
201//____________________________________________________________
203{
204 if (mQueue.empty()) {
205 return {};
206 }
207 return mQueue.empty() ? "" : mInputFiles[mQueue.front()].getLocalName();
208}
209
210//____________________________________________________________
212{
213 if (mRunning) {
214 return;
215 }
216 mRunning = true;
217 mFetcherThread = std::thread(&FileFetcher::fetcher, this);
218}
219
220//____________________________________________________________
222{
223 mRunning = false;
224 std::lock_guard<std::mutex> lock(mMtxStop);
225 if (mFetcherThread.joinable()) {
226 mFetcherThread.join();
227 }
228 if (mFailure) {
229 LOGP(fatal, "too many failures in file fetching: {} in {} attempts for {} files in {} loops, abort", mNFilesProc - mNFilesProcOK, mNFilesProc, getNFiles(), mNLoops);
230 }
231}
232
233//____________________________________________________________
235{
236 if (mRunning) {
237 throw std::runtime_error("FileFetcher thread is still active, cannot cleanup");
238 }
239 if (mNRemote && o2::utils::Str::pathExists(mCopyDirName)) {
240 try {
241 LOGP(info, "FileFetcher scratch directory {} contents (modification date, size in bytes, name):", mCopyDirName);
242 for (const auto& entry : fs::directory_iterator(mCopyDirName)) {
243 const auto fileTime = entry.last_write_time();
244 const auto systemTime = std::chrono::time_point_cast<std::chrono::system_clock::duration>(
245 fileTime - fs::file_time_type::clock::now() + std::chrono::system_clock::now());
246 const auto modificationTime = std::chrono::system_clock::to_time_t(systemTime);
247 std::tm localTime{};
248 localtime_r(&modificationTime, &localTime);
249 const auto size = entry.is_regular_file() ? std::to_string(entry.file_size()) : "-";
250 LOGP(info, "{:%Y-%m-%d %H:%M:%S} {} {}", localTime, size, entry.path().filename().string());
251 }
252 } catch (const std::exception& e) {
253 LOGP(error, "FileFetcher failed to list scratch directory {}: {}", mCopyDirName, e.what());
254 }
255 try {
256 fs::remove_all(mCopyDirName);
257 } catch (...) {
258 LOGP(error, "FileFetcher failed to remove sctrach directory {}", mCopyDirName);
259 }
260 }
261}
262
263//____________________________________________________________
264void FileFetcher::fetcher()
265{
266 // data fetching/copying thread
267 size_t fileEntry = -1ul;
268
269 if (!getNFiles()) {
270 mRunning = false;
271 return;
272 }
273
274 // BOOST requires a locale set
275 try {
276 std::locale loc("");
277 } catch (const std::exception& e) {
278 setenv("LC_ALL", "C", 1);
279 try {
280 std::locale loc("");
281 LOG(info) << "Setting locale";
282 } catch (const std::exception& e) {
283 LOG(info) << "Setting locale failed: " << e.what();
284 return;
285 }
286 }
287
288 while (mRunning) {
289 mNLoops = mNFilesProc / getNFiles();
290 if (mNLoops > mMaxLoops) {
291 LOGP(info, "Finished file fetching: {} of {} files fetched successfully in {} iterations", mNFilesProcOK, mNFilesProc, mMaxLoops);
292 mRunning = false;
293 break;
294 }
295 if (getQueueSize() >= mMaxInQueue) {
296 std::this_thread::sleep_for(5ms);
297 continue;
298 }
299 fileEntry = (fileEntry + 1) % getNFiles();
300 if (fileEntry == 0 && mNLoops > 0) {
301 LOG(info) << "Fetcher starts new iteration " << mNLoops;
302 }
303 mNFilesProc++;
304 auto& fileRef = mInputFiles[fileEntry];
305 if (fileRef.copied || !fileRef.remote || mNoRemoteCopy) {
306 mQueue.push(fileEntry);
307 mNFilesProcOK++;
308 } else { // need to copy
309 if (copyFile(fileEntry)) {
310 fileRef.copied = true;
311 mQueue.push(fileEntry);
312 mNFilesProcOK++;
313 } else {
314 if (mFailThreshold < 0.f) { // cut on abs number of failures
315 if (mNFilesProc - mNFilesProcOK > -mNFilesProcOK) {
316 mFailure = true;
317 }
318 } else if (mFailThreshold > 0.f) {
319 float fracFail = mNLoops ? (mNFilesProc - mNFilesProcOK) / float(mNFilesProc) : (mNFilesProc - mNFilesProcOK) / float(getNFiles());
320 mFailure = fracFail > mFailThreshold;
321 }
322 if (mFailure) {
323 mRunning = false;
324 break;
325 }
326 }
327 }
328 }
329}
330
331//____________________________________________________________
332void FileFetcher::discardFile(const std::string& fname)
333{
334 // delete file if it is copied.
335 auto ent = mCopied.find(fname);
336 if (ent != mCopied.end()) {
337 mInputFiles[ent->second - 1].copied = false;
338 fs::remove(fname);
339 mCopied.erase(fname);
340 LOGP(info, "removing processed file {} from the disk", fname);
341 }
342}
343
344//____________________________________________________________
345bool FileFetcher::copyFile(size_t id)
346{
347 // copy remote file to local setCopyDirName. Adaptation for Gvozden's code from SubTimeFrameFileSource::DataFetcherThread()
348 bool aliencpMode = false;
349 std::string uuid{};
350 std::vector<std::string> logsToClean;
351 std::string dbgset{};
352 if (mCopyCmd.find("alien") != std::string::npos) {
353 if (!gGrid && !TGrid::Connect("alien://")) {
354 LOG(error) << "Copy command refers to alien but connection to Grid failed";
355 }
356 uuid = mInputFiles[id].getOrigName();
357 for (auto& c : uuid) {
358 if (!std::isalnum(c) && c != '-') {
359 c = '_';
360 }
361 }
362 if (!(getenv("ALIENPY_DEBUG") && std::stoi(getenv("ALIENPY_DEBUG")) == 1)) {
363 logsToClean.push_back(fmt::format("log_alienpy_{}.txt", uuid));
364 dbgset += fmt::format("ALIENPY_DEBUG=1 ALIENPY_DEBUG_FILE={} ", logsToClean.back());
365 }
366 if (!(getenv("XRD_LOGLEVEL") && strcmp(getenv("XRD_LOGLEVEL"), "Dump") == 0)) {
367 logsToClean.push_back(fmt::format("log_xrd_{}.txt", uuid));
368 dbgset += fmt::format("XRD_LOGLEVEL=Dump XRD_LOGFILE={} ", logsToClean.back());
369 }
370 LOGP(debug, "debug setting for for {}: {}", mInputFiles[id].getOrigName(), dbgset);
371 }
372 auto realCmd = std::regex_replace(std::regex_replace(mCopyCmd, std::regex(R"(\?src)"), mInputFiles[id].getOrigName()), std::regex(R"(\?dst)"), mInputFiles[id].getLocalName());
373 auto fullCmd = fmt::format(R"(sh -c "{}{}" >> {} 2>&1)", dbgset, realCmd, mCopyCmdLogFile);
374 LOG(info) << "Executing " << fullCmd;
375 const auto sysRet = gSystem->Exec(fullCmd.c_str());
376 if (sysRet != 0) {
377 LOGP(warning, "FileFetcher: non-zero exit code {} for cmd={}", sysRet, realCmd);
378 std::string logCmd = fmt::format(R"(sh -c "cp {} log_aliencp_{}.txt")", mCopyCmdLogFile, uuid);
379 gSystem->Exec(logCmd.c_str());
380 } else { // on success cleanup debug log files
381 for (const auto& log : logsToClean) {
382 if (fs::exists(log)) {
383 fs::remove(log);
384 }
385 }
386 }
387 if (!fs::is_regular_file(mInputFiles[id].getLocalName()) || fs::is_empty(mInputFiles[id].getLocalName()) || sysRet != 0) {
388 LOGP(alarm, "FileFetcher: failed for copy command {}", realCmd);
389 return false;
390 }
391 mCopied[mInputFiles[id].getLocalName()] = id + 1;
392 return true;
393}
std::ostringstream debug
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char e
configStandalone Reconstruction rec_tpc float
char error[CURL_ERROR_SIZE]
uint32_t c
Definition RawData.h:2
TBranch * ptr
const T * frontPtr() const
Definition FIFO.h:82
void pop()
Definition FIFO.h:56
const T & front() const
Definition FIFO.h:64
void push(Args &&... args)
Definition FIFO.h:50
bool empty() const
Definition FIFO.h:43
void discardFile(const std::string &fname)
size_t getQueueSize() const
Definition FileFetcher.h:77
size_t getNFiles() const
Definition FileFetcher.h:75
size_t popFromQueue(bool discard=false)
std::string getNextFileInQueue() const
FileFetcher(const std::string &input, const std::string &selRegex="", const std::string &remRegex="", const std::string &copyCmd="", const std::string &copyDir="/tmp")
GLsizeiptr size
Definition glcorearb.h:659
GLuint const GLchar * name
Definition glcorearb.h:781
GLuint id
Definition glcorearb.h:650
int32_t const char int32_t line
void createDirectoriesIfAbsent(std::string const &path)
std::string to_string(gsl::span< T, Size > span)
Definition common.h:52
static void trim(std::string &s)
Definition StringUtils.h:70
static bool pathExists(const std::string_view p)
static std::vector< std::string > tokenize(const std::string &src, char delim, bool trimToken=true, bool skipEmpty=true)
static std::string create_unique_path(const std::string_view prefix="", int length=16)
static std::string getRandomString(int length)
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"