Project
Loading...
Searching...
No Matches
DirectoryLoader.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
15
19#include <filesystem>
20#include <algorithm>
21#include <climits>
22#include <map>
23#include <iostream>
24#include <forward_list>
25#include <regex>
26#include <fairlogger/Logger.h>
27
28using namespace std;
29using namespace o2::event_visualisation;
30
31deque<string> DirectoryLoader::load(const std::string& path, const std::string& marker, const std::vector<std::string>& ext)
32{
33 deque<string> result;
34 try {
35 for (const auto& entry : std::filesystem::directory_iterator(path)) {
36 if (std::find(ext.begin(), ext.end(), entry.path().extension()) != ext.end()) {
37 result.push_back(entry.path().filename());
38 }
39 }
40 } catch (std::filesystem::filesystem_error const& ex) {
41 LOGF(error, "filesystem problem during DirectoryLoader::load: %s", ex.what());
42 }
43 // comparison with safety if marker not in the filename (-1+1 gives 0)
44 std::sort(result.begin(), result.end(),
45 [marker](const std::string& a, const std::string& b) {
46 return a.substr(a.find_first_of(marker) + 1) < b.substr(b.find_first_of(marker) + 1);
47 });
48
49 return result;
50}
51
52bool DirectoryLoader::canCreateNextFile(const std::vector<std::string>& paths, const std::string& marker, const std::vector<std::string>& ext, long long millisec, long capacityAllowed)
53{
54 deque<string> result;
55 std::map<std::string, std::string> fullPath;
56 for (const auto& path : paths) {
57 try {
58 for (const auto& entry : std::filesystem::directory_iterator(path)) {
59 if (std::find(ext.begin(), ext.end(), entry.path().extension()) != ext.end()) {
60 result.push_back(entry.path().filename());
61 fullPath[entry.path().filename()] = entry.path();
62 }
63 }
64 } catch (std::filesystem::filesystem_error const& ex) {
65 LOGF(error, "filesystem problem during DirectoryLoader::canCreateNextFile: %s", ex.what());
66 }
67 }
68
69 // comparison with safety if marker not in the filename (-1+1 gives 0)
70 if (result.size() > 1) {
71 std::ranges::sort(result.begin(), result.end(),
72 [marker](const std::string& a, const std::string& b) {
73 return a.substr(a.find_first_of(marker) + 1) > b.substr(b.find_first_of(marker) + 1);
74 });
75 }
76
77 unsigned long accumulatedSize = 0L;
78 const std::regex delimiter{"_"};
79 for (auto const& file : result) {
80 std::vector<std::string> c(std::sregex_token_iterator(file.begin(), file.end(), delimiter, -1), {});
81 if (std::stoll(c[1]) < millisec) {
82 break;
83 }
84 try {
85 accumulatedSize += filesystem::file_size(fullPath[file]);
86 } catch (std::filesystem::filesystem_error const& ex) {
87 LOGF(info, "problem scanning folder: %s", ex.what());
88 }
89 if (accumulatedSize > capacityAllowed) {
90 return false;
91 }
92 }
93 return true;
94}
95
96deque<string> DirectoryLoader::load(const std::vector<std::string>& paths, const std::string& marker, const std::vector<std::string>& ext)
97{
98 deque<string> result;
99 try {
100 for (const auto& path : paths) {
101 for (const auto& entry : std::filesystem::directory_iterator(path)) {
102 if (std::find(ext.begin(), ext.end(), entry.path().extension()) != ext.end()) {
103 result.push_back(entry.path().filename());
104 }
105 }
106 }
107 } catch (std::filesystem::filesystem_error const& ex) {
108 LOGF(error, "filesystem problem during DirectoryLoader::load: %s", ex.what());
109 }
110 // comparison with safety if marker not in the filename (-1+1 gives 0)
111 std::sort(result.begin(), result.end(),
112 [marker](const std::string& a, const std::string& b) {
113 return a.substr(a.find_first_of(marker) + 1) < b.substr(b.find_first_of(marker) + 1);
114 });
115
116 return result;
117}
118
119std::vector<std::string> DirectoryLoader::allFolders(const std::string& location)
120{
121 std::vector<std::string> folders;
123 auto const pos = location.find_last_of('_');
124 folders.push_back(location.substr(0, pos) + "_PHYSICS");
125 folders.push_back(location.substr(0, pos) + "_COSMICS");
126 folders.push_back(location.substr(0, pos) + "_SYNTHETIC");
127 } else {
128 folders.push_back(location);
129 }
130 return folders;
131}
132
133void DirectoryLoader::reduceNumberOfFiles(const std::string& path, const std::deque<std::string>& files, std::size_t filesInFolder)
134{
135 if (filesInFolder == -1) {
136 return; // do not reduce
137 }
138 const auto items = files.size() - std::min(files.size(), filesInFolder);
139 for (int i = 0; i < items; i++) {
140 std::remove((path + "/" + files[i]).c_str()); // delete file
141 }
142}
143
144template <typename TP>
145std::time_t to_time_t(TP tp)
146{
147 using namespace std::chrono;
148 auto sctp = time_point_cast<system_clock::duration>(tp - TP::clock::now() + system_clock::now());
149 return system_clock::to_time_t(sctp);
150}
151
152int DirectoryLoader::getNumberOfFiles(const std::string& path, std::vector<std::string>& ext)
153{
154 int res = 0;
155 try {
156 for (const auto& entry : std::filesystem::directory_iterator(path)) {
157 if (std::find(ext.begin(), ext.end(), entry.path().extension()) != ext.end()) {
158 res++;
159 }
160 }
161 } catch (std::filesystem::filesystem_error const& ex) {
162 LOGF(error, "filesystem problem during DirectoryLoader::getNumberOfFiles: %s", ex.what());
163 }
164 return res;
165}
166std::string DirectoryLoader::getLatestFile(const std::string& path, std::vector<std::string>& ext)
167{
168 std::string oldest_file_name = "";
169 std::time_t oldest_file_time = LONG_MAX;
170 for (const auto& entry : std::filesystem::directory_iterator(path)) {
171 if (std::find(ext.begin(), ext.end(), entry.path().extension()) != ext.end()) {
172 const auto file_time = entry.last_write_time();
173 if (const std::time_t tt = to_time_t(file_time); tt < oldest_file_time) {
174 oldest_file_time = tt;
175 oldest_file_name = entry.path().filename().string();
176 }
177 }
178 }
179 return oldest_file_name;
180}
181
182void DirectoryLoader::removeOldestFiles(const std::string& path, std::vector<std::string>& ext, const int remaining)
183{
184 try {
185 while (getNumberOfFiles(path, ext) > remaining) {
186 LOGF(info, "removing oldest file in folder: %s : %s", path, getLatestFile(path, ext));
187 filesystem::remove(path + "/" + getLatestFile(path, ext));
188 }
189 } catch (std::filesystem::filesystem_error const& ex) {
190 LOGF(error, "filesystem problem during DirectoryLoader::removeOldestFiles: %s", ex.what());
191 }
192}
std::time_t to_time_t(TP tp)
Loading content of the Folder and returning sorted.
int32_t i
configStandalone Light OpenGL display display_light Apply smoothing to points Enable Z buffer Highlight links Highlight seeds Highlight sector tracks Highlight final tracks Show individual Show related Highlight grid Show only individual collision Color clusters belonging to track objects Set line Enable drawing TPC data Enable drawing TOF data Enable drawing magnetic field Set field line step count Invert colors configStandalone Heavy OpenGL display display_heavy Show tracks with ITS contribution Show tracks with TOF contribution Enable propagation of loopers Show clusters at nominal x of pad row for early transformed data Split CE tracks when they cross the central Mark clusters Mark adjacent clusters Hide rejected tracks Apply filter on tracks to be displayed Separate increase X h configStandalone Camera window renderer OpenGL display display_camera Orient the camera such that the y axis is always upwards Full Screen MultiSample Anti Downsampling Anti Enable Vertical Sync Target frame or if the backend cannot limit to the refresh Use OpenGL indirect draws to reduce number of draw calls Display h configStandalone Vulkan display display_vulkan Have uniform buffers in host accessible device memory h configStandalone Internal display display_internal Number of updates the Qt GUI may send ahead of the last one the display has before it waits for the display Opacity of the console background Number of console lines the history is shortened to this when it reaches twice the size Time in seconds an info message fades out after setInfoTime h OpenGL display display Font size Do not use Freetype for font Renderer for event Frontend to use for event ROOT macros used as track filter Console script file
char error[CURL_ERROR_SIZE]
uint16_t pos
Definition RawData.h:3
uint32_t res
Definition RawData.h:0
uint32_t c
Definition RawData.h:2
static void removeOldestFiles(const std::string &path, std::vector< std::string > &ext, int remaining)
static bool canCreateNextFile(const std::vector< std::string > &paths, const std::string &marker, const std::vector< std::string > &ext, long long millisec, long capacityAllowed)
static void reduceNumberOfFiles(const std::string &path, const std::deque< std::string > &files, std::size_t filesInFolder)
static std::vector< std::string > allFolders(const std::string &location)
static std::deque< std::string > load(const std::string &path, const std::string &marker, const std::vector< std::string > &ext)
GLuint64EXT * result
Definition glcorearb.h:5662
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei const GLuint * paths
Definition glcorearb.h:5475
GLsizei const GLchar *const * path
Definition glcorearb.h:3591
GLint location
Definition glcorearb.h:800
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
static DeploymentMode deploymentMode()