20#include <fmt/chrono.h>
32using namespace std::chrono_literals;
33namespace fs = std::filesystem;
37 const std::string& copyCmd,
const std::string& copyDir)
38 : mCopyCmd(copyCmd), mCopyDirName(copyDir)
40 if (!selRegex.empty()) {
41 mSelRegex = std::make_unique<std::regex>(selRegex.c_str());
43 if (!remRegex.empty()) {
44 mRemRegex = std::make_unique<std::regex>(remRegex);
46 mNoRemoteCopy = mCopyCmd ==
"no-copy";
51 LOGP(info,
"Input contains {} files, {} remote",
getNFiles(), mNRemote);
54 LOGP(info,
"... but their local copying is explicitly forbidden");
56 if (mCopyCmd.find(
"?src") == std::string::npos || mCopyCmd.find(
"?dst") == std::string::npos) {
57 throw std::runtime_error(fmt::format(
"remote files asked but copy cmd \"{}\" is not valid", mCopyCmd));
62 throw std::runtime_error(fmt::format(
"failed to create scratch directory {}", mCopyDirName));
64 mCopyCmdLogFile = fmt::format(
"{}/{}", mCopyDirName,
"copy-cmd.log");
65 LOGP(info,
"FileFetcher tmp scratch directory is set to {}", mCopyDirName);
78void FileFetcher::processInput(
const std::string& input)
81 processInput(ftokens);
85void FileFetcher::processInput(
const std::vector<std::string>& input)
87 for (
auto inp : input) {
89 if (fs::is_directory(inp)) {
90 processDirectory(inp);
91 }
else if (mSelRegex && !std::regex_match(inp, *mSelRegex.get())) {
94 std::filesystem::path p(inp);
95 if (std::filesystem::file_size(p) > 10000000) {
96 LOGP(
error,
"file list {} larger than 10MB. Is this a data file?", inp);
100 std::ifstream listFile(inp);
101 if (!listFile.good()) {
102 LOGP(
error,
"file {} pretends to be a list of inputs but does not exist", inp);
106 std::vector<std::string> newInput;
107 while (getline(listFile, line)) {
109 if (line[0] ==
'#' ||
line.empty()) {
112 newInput.push_back(line);
114 processInput(newInput);
122void FileFetcher::processDirectory(
const std::string&
name)
124 std::vector<std::string> vs;
125 for (
auto const&
entry : fs::directory_iterator{
name}) {
126 const auto& fnm =
entry.path().native();
127 if (fs::is_regular_file(fnm) && (!mSelRegex || std::regex_match(fnm, *mSelRegex.get()))) {
131 std::sort(vs.begin(), vs.end());
132 for (
const auto&
s : vs) {
138bool FileFetcher::addInputFile(
const std::string& fname)
140 static bool alienErrorPrinted =
false;
141 if (mRemRegex && std::regex_match(fname, *mRemRegex.get())) {
142 mInputFiles.emplace_back(FileRef{fname, mNoRemoteCopy ? fname : createCopyName(fname),
true,
false});
143 if (fname.find(
"alien:") == 0) {
144 if (!gGrid && !TGrid::Connect(
"alien://") && !alienErrorPrinted) {
145 LOG(
error) <<
"File name starts with alien but connection to Grid failed";
146 alienErrorPrinted =
true;
150 }
else if (fs::exists(fname)) {
151 mInputFiles.emplace_back(FileRef{fname,
"",
false,
false});
153 LOGP(
error,
"file {} pretends to be local but does not exist", fname);
160std::string FileFetcher::createCopyName(
const std::string& fname)
const
162 std::string cpnam{}, cpnamP = fname;
163 for (
auto&
c : cpnamP) {
164 if (!std::isalnum(
c) &&
c !=
'.' &&
c !=
'-') {
170 if (!fs::exists(cpnam)) {
181 std::lock_guard<std::mutex> lock(mMtx);
183 if (mQueue.
empty()) {
186 auto id = mQueue.
front();
188 LOGP(info,
"removing processed {} from the queue, discard flag is {}", mInputFiles[
id].getLocalName(), discard);
196size_t FileFetcher::nextInQueue()
const
198 return mQueue.
empty() ? -1ul : mQueue.
front();
204 if (mQueue.
empty()) {
207 return mQueue.
empty() ?
"" : mInputFiles[mQueue.
front()].getLocalName();
217 mFetcherThread = std::thread(&FileFetcher::fetcher,
this);
224 std::lock_guard<std::mutex> lock(mMtxStop);
225 if (mFetcherThread.joinable()) {
226 mFetcherThread.join();
229 LOGP(fatal,
"too many failures in file fetching: {} in {} attempts for {} files in {} loops, abort", mNFilesProc - mNFilesProcOK, mNFilesProc,
getNFiles(), mNLoops);
237 throw std::runtime_error(
"FileFetcher thread is still active, cannot cleanup");
241 LOGP(info,
"FileFetcher scratch directory {} contents (modification date, size in bytes, name):", mCopyDirName);
242 for (
const auto&
entry : fs::directory_iterator(mCopyDirName)) {
243 const auto fileTime =
entry.last_write_time();
244 const auto systemTime = std::chrono::time_point_cast<std::chrono::system_clock::duration>(
245 fileTime - fs::file_time_type::clock::now() + std::chrono::system_clock::now());
246 const auto modificationTime = std::chrono::system_clock::to_time_t(systemTime);
248 localtime_r(&modificationTime, &localTime);
250 LOGP(info,
"{:%Y-%m-%d %H:%M:%S} {} {}", localTime,
size,
entry.path().filename().string());
252 }
catch (
const std::exception&
e) {
253 LOGP(
error,
"FileFetcher failed to list scratch directory {}: {}", mCopyDirName,
e.what());
256 fs::remove_all(mCopyDirName);
258 LOGP(
error,
"FileFetcher failed to remove sctrach directory {}", mCopyDirName);
264void FileFetcher::fetcher()
267 size_t fileEntry = -1ul;
277 }
catch (
const std::exception&
e) {
278 setenv(
"LC_ALL",
"C", 1);
281 LOG(info) <<
"Setting locale";
282 }
catch (
const std::exception&
e) {
283 LOG(info) <<
"Setting locale failed: " <<
e.what();
290 if (mNLoops > mMaxLoops) {
291 LOGP(info,
"Finished file fetching: {} of {} files fetched successfully in {} iterations", mNFilesProcOK, mNFilesProc, mMaxLoops);
296 std::this_thread::sleep_for(5ms);
299 fileEntry = (fileEntry + 1) %
getNFiles();
300 if (fileEntry == 0 && mNLoops > 0) {
301 LOG(info) <<
"Fetcher starts new iteration " << mNLoops;
304 auto& fileRef = mInputFiles[fileEntry];
305 if (fileRef.copied || !fileRef.remote || mNoRemoteCopy) {
306 mQueue.
push(fileEntry);
309 if (copyFile(fileEntry)) {
310 fileRef.copied =
true;
311 mQueue.
push(fileEntry);
314 if (mFailThreshold < 0.f) {
315 if (mNFilesProc - mNFilesProcOK > -mNFilesProcOK) {
318 }
else if (mFailThreshold > 0.f) {
319 float fracFail = mNLoops ? (mNFilesProc - mNFilesProcOK) /
float(mNFilesProc) : (mNFilesProc - mNFilesProcOK) /
float(
getNFiles());
320 mFailure = fracFail > mFailThreshold;
335 auto ent = mCopied.find(fname);
336 if (ent != mCopied.end()) {
337 mInputFiles[ent->second - 1].copied =
false;
339 mCopied.erase(fname);
340 LOGP(info,
"removing processed file {} from the disk", fname);
345bool FileFetcher::copyFile(
size_t id)
348 bool aliencpMode =
false;
350 std::vector<std::string> logsToClean;
351 std::string dbgset{};
352 if (mCopyCmd.find(
"alien") != std::string::npos) {
353 if (!gGrid && !TGrid::Connect(
"alien://")) {
354 LOG(
error) <<
"Copy command refers to alien but connection to Grid failed";
356 uuid = mInputFiles[
id].getOrigName();
357 for (
auto&
c : uuid) {
358 if (!std::isalnum(
c) &&
c !=
'-') {
362 if (!(getenv(
"ALIENPY_DEBUG") && std::stoi(getenv(
"ALIENPY_DEBUG")) == 1)) {
363 logsToClean.push_back(fmt::format(
"log_alienpy_{}.txt", uuid));
364 dbgset += fmt::format(
"ALIENPY_DEBUG=1 ALIENPY_DEBUG_FILE={} ", logsToClean.back());
366 if (!(getenv(
"XRD_LOGLEVEL") && strcmp(getenv(
"XRD_LOGLEVEL"),
"Dump") == 0)) {
367 logsToClean.push_back(fmt::format(
"log_xrd_{}.txt", uuid));
368 dbgset += fmt::format(
"XRD_LOGLEVEL=Dump XRD_LOGFILE={} ", logsToClean.back());
370 LOGP(
debug,
"debug setting for for {}: {}", mInputFiles[
id].getOrigName(), dbgset);
372 auto realCmd = std::regex_replace(std::regex_replace(mCopyCmd, std::regex(R
"(\?src)"), mInputFiles[id].getOrigName()), std::regex(R
"(\?dst)"), mInputFiles[id].getLocalName());
373 auto fullCmd = fmt::format(R
"(sh -c "{}{}" >> {} 2>&1)", dbgset, realCmd, mCopyCmdLogFile);
374 LOG(info) << "Executing " << fullCmd;
375 const auto sysRet = gSystem->Exec(fullCmd.c_str());
377 LOGP(warning,
"FileFetcher: non-zero exit code {} for cmd={}", sysRet, realCmd);
378 std::string logCmd = fmt::format(R
"(sh -c "cp {} log_aliencp_{}.txt")", mCopyCmdLogFile, uuid);
379 gSystem->Exec(logCmd.c_str());
381 for (
const auto& log : logsToClean) {
382 if (fs::exists(log)) {
387 if (!fs::is_regular_file(mInputFiles[
id].getLocalName()) || fs::is_empty(mInputFiles[
id].getLocalName()) || sysRet != 0) {
388 LOGP(alarm,
"FileFetcher: failed for copy command {}", realCmd);
391 mCopied[mInputFiles[
id].getLocalName()] =
id + 1;
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char e
configStandalone Reconstruction rec_tpc float
char error[CURL_ERROR_SIZE]
const T * frontPtr() const
void push(Args &&... args)
void discardFile(const std::string &fname)
size_t getQueueSize() const
size_t popFromQueue(bool discard=false)
std::string getNextFileInQueue() const
FileFetcher(const std::string &input, const std::string &selRegex="", const std::string &remRegex="", const std::string ©Cmd="", const std::string ©Dir="/tmp")
GLuint const GLchar * name
int32_t const char int32_t line
void createDirectoriesIfAbsent(std::string const &path)
std::string to_string(gsl::span< T, Size > span)
static void trim(std::string &s)
static bool pathExists(const std::string_view p)
static std::vector< std::string > tokenize(const std::string &src, char delim, bool trimToken=true, bool skipEmpty=true)
static std::string create_unique_path(const std::string_view prefix="", int length=16)
static std::string getRandomString(int length)
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"