Project
Loading...
Searching...
No Matches
FileFetcher.h
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#ifndef ALICEO2_FILEFETCHER_H_
16#define ALICEO2_FILEFETCHER_H_
17
18#include "CommonUtils/FIFO.h"
19#include <unordered_map>
20#include <string>
21#include <thread>
22#include <Rtypes.h>
23#include <mutex>
24#include <regex>
25
26namespace o2
27{
28namespace utils
29{
30
32{
33 public:
34 struct FileRef {
35 std::string origName{};
36 std::string localName{}; // local alias for for remote files
37 bool remote = false;
38 bool copied = false;
39
40 const auto& getLocalName() const { return remote ? localName : origName; }
41 const auto& getOrigName() const { return origName; }
42 };
43
44 /*
45 * Create file fetcher with predefined cache and async copy of remote files
46 *
47 * @param input: comma-separated list of input data files and/or files with list of data files and/or directories
48 * @param selRegex: regex expression to select files needed for input
49 * @param remRegex: optional regex expression to recognize remote files
50 * @param copyCmd: optional command to copy remote files in format "<operation> ?src ?dst"
51 * @param copyCmd: base directory for copied remote files
52 */
53 FileFetcher(const std::string& input,
54 const std::string& selRegex = "",
55 const std::string& remRegex = "",
56 const std::string& copyCmd = "",
57 const std::string& copyDir = "/tmp");
59
60 const auto& getFileRef(size_t i) const { return mInputFiles[i]; }
61 void setFailThreshold(float f) { mFailThreshold = f; }
62 float getFailThreshold() const { return mFailThreshold; }
63 void setMaxFilesInQueue(size_t s) { mMaxInQueue = s > 0 ? s : 1; }
64 void setMaxLoops(size_t v) { mMaxLoops = v; }
65 bool isRunning() const { return mRunning; }
66 bool isFailed() const { return mFailure; }
67 void start();
68 void stop();
69 void cleanup();
70 size_t getNLoops() const { return mNLoops; }
71 size_t getNFilesProc() const { return mNFilesProc; }
72 size_t getNFilesProcOK() const { return mNFilesProcOK; }
73 size_t getMaxFilesInQueue() const { return mMaxInQueue; }
74 size_t getNRemoteFiles() const { return mNRemote; }
75 size_t getNFiles() const { return mInputFiles.size(); }
76 size_t popFromQueue(bool discard = false);
77 size_t getQueueSize() const { return mQueue.size(); }
78 std::string getNextFileInQueue() const;
79 void discardFile(const std::string& fname);
80
81 private:
82 size_t nextInQueue() const;
83 void processInput(const std::string& input);
84 void processInput(const std::vector<std::string>& input);
85 void processDirectory(const std::string& name);
86 bool addInputFile(const std::string& fname);
87 std::string createCopyName(const std::string& fname) const;
88 bool copyFile(size_t id);
89 bool isRemote(const std::string& fname) const;
90 void fetcher();
91
92 private:
93 FIFO<size_t> mQueue{};
94 std::string mCopyDirName{"/tmp"};
95 std::string mCopyCmdLogFile{};
96 std::string mCopyCmd{};
97 std::unique_ptr<std::regex> mSelRegex;
98 std::unique_ptr<std::regex> mRemRegex;
99 std::unordered_map<std::string, size_t> mCopied{};
100 std::vector<FileRef> mInputFiles{};
101 size_t mNRemote{0};
102 size_t mMaxInQueue{5};
103 bool mRunning = false;
104 bool mNoRemoteCopy = false;
105 bool mFailure = false;
106 size_t mMaxLoops = 0;
107 size_t mNLoops = 0;
108 size_t mNFilesProc = 0;
109 size_t mNFilesProcOK = 0;
110 float mFailThreshold = 0.f; // throw if too many failed fetches (>0 : fraction to total, <0 abs number)
111 mutable std::mutex mMtx;
112 std::mutex mMtxStop;
113 std::thread mFetcherThread{};
114
115 ClassDefNV(FileFetcher, 1);
116};
117
118} // namespace utils
119} // namespace o2
120
121#endif
int32_t i
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
size_t size() const
Definition FIFO.h:31
const auto & getFileRef(size_t i) const
Definition FileFetcher.h:60
size_t getNRemoteFiles() const
Definition FileFetcher.h:74
bool isFailed() const
Definition FileFetcher.h:66
size_t getNLoops() const
Definition FileFetcher.h:70
void discardFile(const std::string &fname)
size_t getNFilesProc() const
Definition FileFetcher.h:71
void setMaxLoops(size_t v)
Definition FileFetcher.h:64
size_t getQueueSize() const
Definition FileFetcher.h:77
float getFailThreshold() const
Definition FileFetcher.h:62
size_t getNFiles() const
Definition FileFetcher.h:75
size_t getMaxFilesInQueue() const
Definition FileFetcher.h:73
void setFailThreshold(float f)
Definition FileFetcher.h:61
size_t popFromQueue(bool discard=false)
std::string getNextFileInQueue() const
void setMaxFilesInQueue(size_t s)
Definition FileFetcher.h:63
size_t getNFilesProcOK() const
Definition FileFetcher.h:72
bool isRunning() const
Definition FileFetcher.h:65
const GLdouble * v
Definition glcorearb.h:832
GLuint const GLchar * name
Definition glcorearb.h:781
GLdouble f
Definition glcorearb.h:310
a couple of static helper functions to create timestamp values for CCDB queries or override obsolete ...
Common utility functions.
const auto & getOrigName() const
Definition FileFetcher.h:41
const auto & getLocalName() const
Definition FileFetcher.h:40