Project
Loading...
Searching...
No Matches
CcdbApi.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
16
17#include "CCDB/CcdbApi.h"
18#include "CCDB/CCDBDownloader.h"
19#include <curl/curl.h>
20#include "CCDB/CCDBQuery.h"
21
27#include <chrono>
28#include <memory>
29#include <ranges>
30#include <sstream>
31#include <TFile.h>
32#include <TGrid.h>
33#include <TSystem.h>
34#include <TStreamerInfo.h>
35#include <TMemFile.h>
36#include <TH1F.h>
37#include <TTree.h>
38#include <fairlogger/Logger.h>
39#include <TError.h>
40#include <TClass.h>
42#include <algorithm>
43#include <filesystem>
44#include <boost/algorithm/string.hpp>
45#include <iostream>
46#include <mutex>
47#include <boost/interprocess/sync/named_semaphore.hpp>
48#include <regex>
49#include <cstdio>
50#include <string>
51#include <string_view>
52#include <utility>
53#include <TAlienUserAgent.h>
54#if defined(_LIBCPP_VERSION)
56#endif
57#include <unordered_set>
58#include "rapidjson/document.h"
59#include "rapidjson/writer.h"
60#include "rapidjson/stringbuffer.h"
61
62namespace o2::ccdb
63{
64
65using namespace std;
66
67std::mutex gIOMutex; // to protect TMemFile IO operations
68unique_ptr<TJAlienCredentials> CcdbApi::mJAlienCredentials = nullptr;
69
70namespace
71{
76std::string_view trimHeaderValue(std::string_view value)
77{
78 constexpr std::string_view whitespace = " \t\r\n";
79 const auto first = value.find_first_not_of(whitespace);
80 return first == std::string_view::npos
81 ? std::string_view{}
82 : value.substr(first, value.find_last_not_of(whitespace) - first + 1);
83}
84
91const std::vector<std::pair<std::string, std::string>>& gateTokenTable()
92{
93 static const auto table = []() {
94 std::vector<std::pair<std::string, std::string>> entries;
95 const char* spec = getenv("ALICEO2_CCDB_AUTH_TOKENS");
96 std::string_view rest = spec ? spec : "";
97 while (!rest.empty()) {
98 const auto sep = rest.find(';');
99 const auto entry = trimHeaderValue(rest.substr(0, sep));
100 rest = (sep == std::string_view::npos) ? std::string_view{} : rest.substr(sep + 1);
101 const auto eq = entry.find('=');
102 if (eq == std::string_view::npos) {
103 continue;
104 }
105 auto url = trimHeaderValue(entry.substr(0, eq));
106 // Trimmed: a stray newline in a token makes the request malformed, which
107 // a strict broker rejects with an opaque 400 rather than an auth error.
108 const auto token = trimHeaderValue(entry.substr(eq + 1));
109 while (url.size() > 1 && url.back() == '/') { // normalise, so the boundary test below is exact
110 url.remove_suffix(1);
111 }
112 if (!url.empty() && !token.empty()) {
113 entries.emplace_back(std::string(url), std::string("Authorization: Bearer ").append(token));
114 }
115 }
116 std::sort(entries.begin(), entries.end(),
117 [](const auto& a, const auto& b) { return a.first.size() > b.first.size(); });
118 return entries;
119 }();
120 return table;
121}
122
131curl_slist* appendGateToken(curl_slist* list, std::string_view url)
132{
133 for (const auto& [prefix, header] : gateTokenTable()) {
134 if (url.substr(0, prefix.size()) == prefix &&
135 (url.size() == prefix.size() || url[prefix.size()] == '/')) {
136 return curl_slist_append(list, header.c_str());
137 }
138 }
139 return list;
140}
141} // namespace
142
150{
151 public:
152 CCDBSemaphore(std::string const& cachepath, std::string const& path);
154
155 private:
156 boost::interprocess::named_semaphore* mSem = nullptr;
157 std::string mSemName{}; // name under which semaphore is kept by the OS kernel
158};
159
160// Small registry class with the purpose that a static object
161// ensures cleanup of registered semaphores even when programs
162// "crash".
164{
165 public:
166 SemaphoreRegistry() = default;
168 void add(CCDBSemaphore const* ptr);
169 void remove(CCDBSemaphore const* ptr);
170
171 private:
172 std::unordered_set<CCDBSemaphore const*> mStore;
173};
174static SemaphoreRegistry gSemaRegistry;
175
177{
178 using namespace o2::framework;
179 setUniqueAgentID();
180
182 mIsCCDBDownloaderPreferred = 0;
183 if (deploymentMode == DeploymentMode::OnlineDDS && deploymentMode == DeploymentMode::OnlineECS && deploymentMode == DeploymentMode::OnlineAUX && deploymentMode == DeploymentMode::FST) {
184 mIsCCDBDownloaderPreferred = 1;
185 }
186 if (getenv("ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI")) { // todo rename ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI to ALICEO2_PREFER_MULTIHANDLE_CCDBAPI
187 mIsCCDBDownloaderPreferred = atoi(getenv("ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI"));
188 }
189 mDownloader = new CCDBDownloader();
190}
191
193{
194 curl_global_cleanup();
195 delete mDownloader;
196}
197
198void CcdbApi::setUniqueAgentID()
199{
200 mUniqueAgentID = TAlienUserAgent::BasedOnEnvironment().ToString();
201}
202
204{
205#ifdef __APPLE__
206 LOG(debug) << "On macOS we simply rely on TGrid::Connect(\"alien\").";
207 return true;
208#endif
209 if (getenv("ALICEO2_CCDB_NOTOKENCHECK") && atoi(getenv("ALICEO2_CCDB_NOTOKENCHECK"))) {
210 return true;
211 }
212 if (getenv("JALIEN_TOKEN_CERT")) {
213 return true;
214 }
215 auto returncode = system("LD_PRELOAD= alien-token-info &> /dev/null");
216 if (returncode == -1) {
217 LOG(error) << "...";
218 }
219 return returncode == 0;
220}
221
222void CcdbApi::curlInit()
223{
224 // todo : are there other things to initialize globally for curl ?
225 curl_global_init(CURL_GLOBAL_DEFAULT);
226 CcdbApi::mJAlienCredentials = std::make_unique<TJAlienCredentials>();
227 CcdbApi::mJAlienCredentials->loadCredentials();
228 CcdbApi::mJAlienCredentials->selectPreferedCredentials();
229
230 // allow to configure the socket timeout of CCDBDownloader (for some tuning studies)
231 if (getenv("ALICEO2_CCDB_SOCKET_TIMEOUT")) {
232 auto timeoutMS = atoi(getenv("ALICEO2_CCDB_SOCKET_TIMEOUT"));
233 if (timeoutMS >= 0) {
234 LOG(info) << "Setting socket timeout to " << timeoutMS << " milliseconds";
235 mDownloader->setKeepaliveTimeoutTime(timeoutMS);
236 }
237 }
238}
239
240void CcdbApi::init(std::string const& host)
241{
242 if (host.empty()) {
243 throw std::invalid_argument("Empty url passed CcdbApi, cannot initialize. Aborting.");
244 }
245
246 // if host is prefixed with "file://" this is a local snapshot
247 // in this case we init the API in snapshot (readonly) mode
248 constexpr const char* SNAPSHOTPREFIX = "file://";
249 mUrl = host;
250
251 if (host.substr(0, 7).compare(SNAPSHOTPREFIX) == 0) {
252 auto path = host.substr(7);
253 initInSnapshotMode(path);
254 } else {
255 initHostsPool(host);
256 curlInit();
257 }
258 // The environment option ALICEO2_CCDB_LOCALCACHE allows
259 // to reduce the number of queries to the server, by collecting the objects in a local
260 // cache folder, and serving from this folder for repeated queries.
261 // This is useful for instance for MC GRID productions in which we spawn
262 // many isolated processes, all querying the CCDB (for potentially the same objects and same timestamp).
263 // In addition, we can monitor exactly which objects are fetched and what is their content.
264 // One can also distribute so obtained caches to sites without network access.
265 //
266 // THE INFORMATION BELOW IS TEMPORARILY WRONG: the functionality of checking the validity if IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE
267 // is NOT set is broken. At the moment the code is modified to behave as if the IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE is always set
268 // whenever the ALICEO2_CCDB_LOCALCACHE is defined.
269 //
270 // When used with the DPL CCDB fetcher (i.e. loadFileToMemory is called), in order to prefer the available snapshot w/o its validity
271 // check an extra variable IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE must be defined, otherwhise the object will be fetched from the
272 // server after the validity check and new snapshot will be created if needed
273
274 std::string snapshotReport{};
275 const char* cachedir = getenv("ALICEO2_CCDB_LOCALCACHE");
276 namespace fs = std::filesystem;
277 if (cachedir) {
278 if (cachedir[0] == 0) {
279 mSnapshotCachePath = fs::weakly_canonical(fs::absolute("."));
280 } else {
281 mSnapshotCachePath = fs::weakly_canonical(fs::absolute(cachedir));
282 }
283 snapshotReport = fmt::format("(cache snapshots to dir={}", mSnapshotCachePath);
284 }
285 if (cachedir) { // || getenv("IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE")) {
286 mPreferSnapshotCache = true;
287 if (mSnapshotCachePath.empty()) {
288 LOGP(fatal, "IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE is defined but the ALICEO2_CCDB_LOCALCACHE is not");
289 }
290 snapshotReport += ", prefer if available";
291 }
292 if (!snapshotReport.empty()) {
293 snapshotReport += ')';
294 }
295
296 mNeedAlienToken = (host.find("https://") != std::string::npos) || (host.find("alice-ccdb.cern.ch") != std::string::npos) || (host.find("ccdb-test.cern.ch") != std::string::npos);
297 // Set the curl timeout. It can be forced with an env var or it has different defaults based on the deployment mode.
298 if (getenv("ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD")) {
299 auto timeout = atoi(getenv("ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD"));
300 if (timeout >= 0) { // if valid int
301 mCurlTimeoutDownload = timeout;
302 }
303 } else { // set a default depending on the deployment mode
305 if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS ||
308 mCurlTimeoutDownload = 15;
309 } else if (deploymentMode == o2::framework::DeploymentMode::Grid ||
310 deploymentMode == o2::framework::DeploymentMode::FST) {
311 mCurlTimeoutDownload = 15;
312 } else if (deploymentMode == o2::framework::DeploymentMode::Local) {
313 mCurlTimeoutDownload = 5;
314 }
315 }
316
317 if (getenv("ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD")) {
318 auto timeout = atoi(getenv("ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD"));
319 if (timeout >= 0) { // if valid int
320 mCurlTimeoutUpload = timeout;
321 }
322 } else { // set a default depending on the deployment mode
324 if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS ||
327 mCurlTimeoutUpload = 3;
328 } else if (deploymentMode == o2::framework::DeploymentMode::Grid ||
329 deploymentMode == o2::framework::DeploymentMode::FST) {
330 mCurlTimeoutUpload = 20;
331 } else if (deploymentMode == o2::framework::DeploymentMode::Local) {
332 mCurlTimeoutUpload = 20;
333 }
334 }
335 if (mDownloader) {
336 mDownloader->setRequestTimeoutTime(mCurlTimeoutDownload * 1000L);
337 }
338
339 LOGP(debug, "Curl timeouts are set to: download={:2}, upload={:2} seconds", mCurlTimeoutDownload, mCurlTimeoutUpload);
340
341 LOGP(info, "Init CcdApi with UserAgentID: {}, Host: {}{}, Curl timeouts: upload:{} download:{}", mUniqueAgentID, host,
342 mInSnapshotMode ? "(snapshot readonly mode)" : snapshotReport.c_str(), mCurlTimeoutUpload, mCurlTimeoutDownload);
343}
344
346{
347 mDownloader->runLoop(noWait);
348}
349
350// A helper function used in a few places. Updates a ROOT file with meta/header information.
351void CcdbApi::updateMetaInformationInLocalFile(std::string const& filename, std::map<std::string, std::string> const* headers, CCDBQuery const* querysummary)
352{
353 std::lock_guard<std::mutex> guard(gIOMutex);
354 auto oldlevel = gErrorIgnoreLevel;
355 gErrorIgnoreLevel = 6001; // ignoring error messages here (since we catch with IsZombie)
356 TFile snapshotfile(filename.c_str(), "UPDATE");
357 // The assumption is that the blob is a ROOT file
358 if (!snapshotfile.IsZombie()) {
359 if (querysummary && !snapshotfile.Get(CCDBQUERY_ENTRY)) {
360 snapshotfile.WriteObjectAny(querysummary, TClass::GetClass(typeid(*querysummary)), CCDBQUERY_ENTRY);
361 }
362 if (headers && !snapshotfile.Get(CCDBMETA_ENTRY)) {
363 snapshotfile.WriteObjectAny(headers, TClass::GetClass(typeid(*headers)), CCDBMETA_ENTRY);
364 }
365 snapshotfile.Write();
366 snapshotfile.Close();
367 }
368 gErrorIgnoreLevel = oldlevel;
369}
370
376std::string sanitizeObjectName(const std::string& objectName)
377{
378 std::string tmpObjectName = objectName;
379 tmpObjectName.erase(std::remove_if(tmpObjectName.begin(), tmpObjectName.end(),
380 [](auto const& c) -> bool { return (!std::isalnum(c) && c != '_' && c != '/' && c != '.'); }),
381 tmpObjectName.end());
382 return tmpObjectName;
383}
384
385std::unique_ptr<std::vector<char>> CcdbApi::createObjectImage(const void* obj, std::type_info const& tinfo, CcdbObjectInfo* info)
386{
387 // Create a binary image of the object, if CcdbObjectInfo pointer is provided, register there
388 // the assigned object class name and the filename
389 std::lock_guard<std::mutex> guard(gIOMutex);
391 std::string tmpFileName = generateFileName(className);
392 if (info) {
393 info->setFileName(tmpFileName);
394 info->setObjectType(className);
395 }
396 return o2::utils::MemFileHelper::createFileImage(obj, tinfo, tmpFileName, CCDBOBJECT_ENTRY);
397}
398
399std::unique_ptr<std::vector<char>> CcdbApi::createObjectImage(const TObject* rootObject, CcdbObjectInfo* info)
400{
401 // Create a binary image of the object, if CcdbObjectInfo pointer is provided, register there
402 // the assigned object class name and the filename
403 std::string className = rootObject->GetName();
404 std::string tmpFileName = generateFileName(className);
405 if (info) {
406 info->setFileName(tmpFileName);
407 info->setObjectType("TObject"); // why TObject and not the actual name?
408 }
409 std::lock_guard<std::mutex> guard(gIOMutex);
410 return o2::utils::MemFileHelper::createFileImage(*rootObject, tmpFileName, CCDBOBJECT_ENTRY);
411}
412
413int CcdbApi::storeAsTFile_impl(const void* obj, std::type_info const& tinfo, std::string const& path,
414 std::map<std::string, std::string> const& metadata,
415 long startValidityTimestamp, long endValidityTimestamp,
416 std::vector<char>::size_type maxSize) const
417{
418 // We need the TClass for this type; will verify if dictionary exists
419 if (!obj) {
420 LOGP(error, "nullptr is provided for object {}/{}/{}", path, startValidityTimestamp, endValidityTimestamp);
421 return -1;
422 }
423 CcdbObjectInfo info;
424 auto img = createObjectImage(obj, tinfo, &info);
425 return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(),
426 path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize);
427}
428
429int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::string& filename, const std::string& objectType,
430 const std::string& path, const std::map<std::string, std::string>& metadata,
431 long startValidityTimestamp, long endValidityTimestamp, std::vector<char>::size_type maxSize) const
432{
433 if (maxSize > 0 && size > maxSize) {
434 LOGP(alarm, "Object will not be uploaded to {} since its size {} exceeds max allowed {}", path, size, maxSize);
435 return -1;
436 }
437 int returnValue = 0;
438
439 // Prepare URL
440 long sanitizedStartValidityTimestamp = startValidityTimestamp;
441 if (startValidityTimestamp == -1) {
442 LOGP(info, "Start of Validity not set, current timestamp used.");
443 sanitizedStartValidityTimestamp = getCurrentTimestamp();
444 }
445 long sanitizedEndValidityTimestamp = endValidityTimestamp;
446 if (endValidityTimestamp == -1) {
447 LOGP(info, "End of Validity not set, start of validity plus 1 day used.");
448 sanitizedEndValidityTimestamp = getFutureTimestamp(60 * 60 * 24 * 1);
449 }
450 if (mInSnapshotMode) { // write local file
451 if (filename.empty() || buffer == nullptr || size == 0) {
452 LOGP(alarm, "Snapshot mode does not support headers-only upload");
453 return -3;
454 }
455 auto pthLoc = getSnapshotDir(mSnapshotTopPath, path);
457 auto flLoc = getSnapshotFile(mSnapshotTopPath, path, filename);
458 // add the timestamps to the end
459 auto pent = flLoc.find_last_of('.');
460 if (pent == std::string::npos) {
461 pent = flLoc.size();
462 }
463 flLoc.insert(pent, fmt::format("_{}_{}", startValidityTimestamp, endValidityTimestamp));
464 ofstream outf(flLoc.c_str(), ios::out | ios::binary);
465 outf.write(buffer, size);
466 outf.close();
467 if (!outf.good()) {
468 throw std::runtime_error(fmt::format("Failed to write local CCDB file {}", flLoc));
469 } else {
470 std::map<std::string, std::string> metaheader(metadata);
471 // add time validity information
472 metaheader["Valid-From"] = std::to_string(startValidityTimestamp);
473 metaheader["Valid-Until"] = std::to_string(endValidityTimestamp);
474 updateMetaInformationInLocalFile(flLoc.c_str(), &metaheader);
475 std::string metaStr{};
476 for (const auto& mentry : metadata) {
477 metaStr += fmt::format("{}={};", mentry.first, mentry.second);
478 }
479 metaStr += "$USER_META;";
480 LOGP(info, "Created local snapshot {}", flLoc);
481 LOGP(info, R"(Upload with: o2-ccdb-upload --host "$ccdbhost" -p {} -f {} -k {} --starttimestamp {} --endtimestamp {} -m "{}")",
482 path, flLoc, CCDBOBJECT_ENTRY, startValidityTimestamp, endValidityTimestamp, metaStr);
483 }
484 return returnValue;
485 }
486
487 // Curl preparation
488 CurlHandle* curl = nullptr;
489 curl = curl_easy_init();
490
491 // checking that all metadata keys do not contain invalid characters
492 checkMetadataKeys(metadata);
493
494 if (curl != nullptr) {
495 auto mime = curl_mime_init(curl);
496 auto field = curl_mime_addpart(mime);
497 curl_mime_name(field, "send");
498 if (!filename.empty()) {
499 curl_mime_filedata(field, filename.c_str());
500 }
501 if (buffer != nullptr && size > 0) {
502 curl_mime_data(field, buffer, size);
503 } else {
504 curl_mime_data(field, "", 0);
505 }
506
507 curlSetSSLOptions(curl);
508
509 curl_easy_setopt(curl, CURLOPT_MIMEPOST, mime);
510 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
511 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
512 curl_easy_setopt(curl, CURLOPT_TIMEOUT, mCurlTimeoutUpload);
513
514 CURLcode res = CURL_LAST;
515
516 for (size_t hostIndex = 0; hostIndex < hostsPool.size() && res > 0; hostIndex++) {
517 std::string fullUrl = getFullUrlForStorage(curl, path, objectType, metadata, sanitizedStartValidityTimestamp, sanitizedEndValidityTimestamp, hostIndex);
518 LOG(debug3) << "Full URL Encoded: " << fullUrl;
519 /* what URL that receives this POST */
520 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
521
522 // Per host: the gate token is per endpoint (see appendGateToken).
523 struct curl_slist* headerlist = curl_slist_append(nullptr, "Expect:");
524 headerlist = appendGateToken(headerlist, fullUrl);
525 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist);
526
527 /* Perform the request, res will get the return code */
528 res = static_cast<CURLcode>(CURL_perform(curl));
529 /* Check for errors */
530 if (res != CURLE_OK) {
531 if (res == CURLE_OPERATION_TIMEDOUT) {
532 LOGP(alarm, "curl_easy_perform() timed out. Consider increasing the timeout using the env var `ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD` (seconds), current one is {}", mCurlTimeoutUpload);
533 } else { // generic message
534 LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res));
535 }
536 returnValue = res;
537 }
538 curl_slist_free_all(headerlist);
539 }
540
541 /* always cleanup */
542 curl_easy_cleanup(curl);
543
544 /* free mime */
545 curl_mime_free(mime);
546 } else {
547 LOGP(alarm, "curl initialization failure");
548 returnValue = -2;
549 }
550 return returnValue;
551}
552
553int CcdbApi::storeAsTFile(const TObject* rootObject, std::string const& path, std::map<std::string, std::string> const& metadata,
554 long startValidityTimestamp, long endValidityTimestamp, std::vector<char>::size_type maxSize) const
555{
556 // Prepare file
557 if (!rootObject) {
558 LOGP(error, "nullptr is provided for object {}/{}/{}", path, startValidityTimestamp, endValidityTimestamp);
559 return -1;
560 }
561 CcdbObjectInfo info;
562 auto img = createObjectImage(rootObject, &info);
563 return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(), path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize);
564}
565
566std::string CcdbApi::getFullUrlForStorage(CurlHandle* curl, const std::string& path, const std::string& objtype,
567 const std::map<std::string, std::string>& metadata,
568 long startValidityTimestamp, long endValidityTimestamp, int hostIndex) const
569{
570 // Prepare timestamps
571 std::string startValidityString = getTimestampString(startValidityTimestamp < 0 ? getCurrentTimestamp() : startValidityTimestamp);
572 std::string endValidityString = getTimestampString(endValidityTimestamp < 0 ? getFutureTimestamp(60 * 60 * 24 * 1) : endValidityTimestamp);
573 // Get url
574 std::string url = getHostUrl(hostIndex);
575 // Build URL
576 std::string fullUrl = url + "/" + path + "/" + startValidityString + "/" + endValidityString + "/";
577 // Add type as part of metadata
578 // we need to URL encode the object type, since in case it has special characters (like the "<", ">" for templated classes) it won't work otherwise
579 char* objtypeEncoded = curl_easy_escape(curl, objtype.c_str(), objtype.size());
580 fullUrl += "ObjectType=" + std::string(objtypeEncoded) + "/";
581 curl_free(objtypeEncoded);
582 // Add general metadata
583 for (auto& kv : metadata) {
584 std::string mfirst = kv.first;
585 std::string msecond = kv.second;
586 // same trick for the metadata as for the object type
587 char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
588 char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
589 fullUrl += std::string(mfirstEncoded) + "=" + std::string(msecondEncoded) + "/";
590 curl_free(mfirstEncoded);
591 curl_free(msecondEncoded);
592 }
593 return fullUrl;
594}
595
596// todo make a single method of the one above and below
597std::string CcdbApi::getFullUrlForRetrieval(CurlHandle* curl, const std::string& path, const std::map<std::string, std::string>& metadata, long timestamp, int hostIndex) const
598{
599 if (mInSnapshotMode) {
600 return getSnapshotFile(mSnapshotTopPath, path);
601 }
602
603 // Prepare timestamps
604 std::string validityString = getTimestampString(timestamp < 0 ? getCurrentTimestamp() : timestamp);
605 // Get host url
606 std::string hostUrl = getHostUrl(hostIndex);
607 // Build URL
608 std::string fullUrl = hostUrl + "/" + path + "/" + validityString + "/";
609 // Add metadata
610 for (auto& kv : metadata) {
611 std::string mfirst = kv.first;
612 std::string msecond = kv.second;
613 // trick for the metadata in case it contains special characters
614 char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
615 char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
616 fullUrl += std::string(mfirstEncoded) + "=" + std::string(msecondEncoded) + "/";
617 curl_free(mfirstEncoded);
618 curl_free(msecondEncoded);
619 }
620 return fullUrl;
621}
622
627 char* memory;
628 unsigned int size;
629};
630
640static size_t WriteMemoryCallback(void* contents, size_t size, size_t nmemb, void* userp)
641{
642 size_t realsize = size * nmemb;
643 auto* mem = (struct MemoryStruct*)userp;
644
645 mem->memory = (char*)realloc(mem->memory, mem->size + realsize + 1);
646 if (mem->memory == nullptr) {
647 printf("not enough memory (realloc returned NULL)\n");
648 return 0;
649 }
650
651 memcpy(&(mem->memory[mem->size]), contents, realsize);
652 mem->size += realsize;
653 mem->memory[mem->size] = 0;
654
655 return realsize;
656}
657
669static size_t WriteToFileCallback(void* ptr, size_t size, size_t nmemb, FILE* stream)
670{
671 size_t written = fwrite(ptr, size, nmemb, stream);
672 return written;
673}
674
682static CURLcode ssl_ctx_callback(CurlHandle*, void*, void* parm)
683{
684 std::string msg((const char*)parm);
685 int start = 0, end = msg.find('\n');
686
687 if (msg.length() > 0 && end == -1) {
688 LOG(warn) << msg;
689 } else if (end > 0) {
690 while (end > 0) {
691 LOG(warn) << msg.substr(start, end - start);
692 start = end + 1;
693 end = msg.find('\n', start);
694 }
695 }
696 return CURLE_OK;
697}
698
700{
701 CredentialsKind cmk = mJAlienCredentials->getPreferedCredentials();
702
703 /* NOTE: return early, the warning should be printed on SSL callback if needed */
704 if (cmk == cNOT_FOUND) {
705 return;
706 }
707
708 TJAlienCredentialsObject cmo = mJAlienCredentials->get(cmk);
709
710 char* CAPath = getenv("X509_CERT_DIR");
711 if (CAPath) {
712 curl_easy_setopt(curl_handle, CURLOPT_CAPATH, CAPath);
713 }
714 curl_easy_setopt(curl_handle, CURLOPT_CAINFO, nullptr);
715 curl_easy_setopt(curl_handle, CURLOPT_SSLCERT, cmo.certpath.c_str());
716 curl_easy_setopt(curl_handle, CURLOPT_SSLKEY, cmo.keypath.c_str());
717
718 // NOTE: for lazy logging only
719 curl_easy_setopt(curl_handle, CURLOPT_SSL_CTX_FUNCTION, ssl_ctx_callback);
720 curl_easy_setopt(curl_handle, CURLOPT_SSL_CTX_DATA, mJAlienCredentials->getMessages().c_str());
721
722 // CURLcode ret = curl_easy_setopt(curl_handle, CURLOPT_SSL_CTX_FUNCTION, *ssl_ctx_callback);
723}
724
725using CurlWriteCallback = size_t (*)(void*, size_t, size_t, void*);
726
727void CcdbApi::initCurlOptionsForRetrieve(CurlHandle* curlHandle, void* chunk, CurlWriteCallback writeCallback, bool followRedirect) const
728{
729 curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
730 curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, chunk);
731 curl_easy_setopt(curlHandle, CURLOPT_FOLLOWLOCATION, followRedirect ? 1L : 0L);
732}
733
734namespace
735{
736template <typename MapType = std::map<std::string, std::string>>
737size_t header_map_callback(char* buffer, size_t size, size_t nitems, void* userdata)
738{
739 auto* headers = static_cast<MapType*>(userdata);
740 auto header = std::string(buffer, size * nitems);
741 std::string::size_type index = header.find(':', 0);
742 if (index != std::string::npos) {
743 const auto key = boost::algorithm::trim_copy(header.substr(0, index));
744 const auto value = boost::algorithm::trim_copy(header.substr(index + 1));
745 LOGP(debug, "Adding #{} {} -> {}", headers->size(), key, value);
746 bool insert = true;
747 if (key == "Content-Length") {
748 auto cl = headers->find("Content-Length");
749 if (cl != headers->end()) {
750 if (std::stol(cl->second) < stol(value)) {
751 headers->erase(key);
752 } else {
753 insert = false;
754 }
755 }
756 }
757
758 // Keep only the first ETag encountered
759 if (key == "ETag") {
760 auto cl = headers->find("ETag");
761 if (cl != headers->end()) {
762 insert = false;
763 }
764 }
765
766 // Keep only the first Content-Type encountered
767 if (key == "Content-Type") {
768 auto cl = headers->find("Content-Type");
769 if (cl != headers->end()) {
770 insert = false;
771 }
772 }
773
774 if (insert) {
775 headers->insert(std::make_pair(key, value));
776 }
777 }
778 return size * nitems;
779}
780} // namespace
781
782void CcdbApi::initCurlHTTPHeaderOptionsForRetrieve(CurlHandle* curlHandle, curl_slist*& option_list, long timestamp, std::map<std::string, std::string>* headers, std::string const& etag,
783 const std::string& createdNotAfter, const std::string& createdNotBefore, std::string_view url) const
784{
785 // struct curl_slist* list = nullptr;
786 if (!etag.empty()) {
787 option_list = curl_slist_append(option_list, ("If-None-Match: " + etag).c_str());
788 }
789
790 if (!createdNotAfter.empty()) {
791 option_list = curl_slist_append(option_list, ("If-Not-After: " + createdNotAfter).c_str());
792 }
793
794 if (!createdNotBefore.empty()) {
795 option_list = curl_slist_append(option_list, ("If-Not-Before: " + createdNotBefore).c_str());
796 }
797
798 if (headers != nullptr) {
799 option_list = curl_slist_append(option_list, ("If-None-Match: " + to_string(timestamp)).c_str());
800 curl_easy_setopt(curlHandle, CURLOPT_HEADERFUNCTION, header_map_callback<>);
801 curl_easy_setopt(curlHandle, CURLOPT_HEADERDATA, headers);
802 }
803
804 option_list = appendGateToken(option_list, url);
805
806 // Unconditionally, nullptr included: the handle is reused across hosts, and
807 // skipping the set would leave a previous host's freed list installed.
808 curl_easy_setopt(curlHandle, CURLOPT_HTTPHEADER, option_list);
809
810 curl_easy_setopt(curlHandle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
811}
812
813bool CcdbApi::receiveToFile(FILE* fileHandle, std::string const& path, std::map<std::string, std::string> const& metadata,
814 long timestamp, std::map<std::string, std::string>* headers, std::string const& etag,
815 const std::string& createdNotAfter, const std::string& createdNotBefore, bool followRedirect) const
816{
817 return receiveObject((void*)fileHandle, path, metadata, timestamp, headers, etag, createdNotAfter, createdNotBefore, followRedirect, (CurlWriteCallback)&WriteToFileCallback);
818}
819
820bool CcdbApi::receiveToMemory(void* chunk, std::string const& path, std::map<std::string, std::string> const& metadata,
821 long timestamp, std::map<std::string, std::string>* headers, std::string const& etag,
822 const std::string& createdNotAfter, const std::string& createdNotBefore, bool followRedirect) const
823{
824 return receiveObject((void*)chunk, path, metadata, timestamp, headers, etag, createdNotAfter, createdNotBefore, followRedirect, (CurlWriteCallback)&WriteMemoryCallback);
825}
826
827bool CcdbApi::receiveObject(void* dataHolder, std::string const& path, std::map<std::string, std::string> const& metadata,
828 long timestamp, std::map<std::string, std::string>* headers, std::string const& etag,
829 const std::string& createdNotAfter, const std::string& createdNotBefore, bool followRedirect, CurlWriteCallback writeCallback) const
830{
831 CurlHandle* curlHandle;
832
833 curlHandle = curl_easy_init();
834 curl_easy_setopt(curlHandle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
835
836 if (curlHandle != nullptr) {
837
838 curlSetSSLOptions(curlHandle);
839 initCurlOptionsForRetrieve(curlHandle, dataHolder, writeCallback, followRedirect);
840 long responseCode = 0;
841 CURLcode curlResultCode = CURL_LAST;
842
843 for (size_t hostIndex = 0; hostIndex < hostsPool.size() && (responseCode >= 400 || curlResultCode > 0); hostIndex++) {
844 std::string fullUrl = getFullUrlForRetrieval(curlHandle, path, metadata, timestamp, hostIndex);
845 curl_easy_setopt(curlHandle, CURLOPT_URL, fullUrl.c_str());
846
847 // Per host: the gate token is per endpoint (see appendGateToken).
848 curl_slist* option_list = nullptr;
849 initCurlHTTPHeaderOptionsForRetrieve(curlHandle, option_list, timestamp, headers, etag, createdNotAfter, createdNotBefore, fullUrl);
850
851 curlResultCode = static_cast<CURLcode>(CURL_perform(curlHandle));
852
853 if (curlResultCode != CURLE_OK) {
854 LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(curlResultCode));
855 } else {
856 curlResultCode = curl_easy_getinfo(curlHandle, CURLINFO_RESPONSE_CODE, &responseCode);
857 if ((curlResultCode == CURLE_OK) && (responseCode < 300)) {
858 curl_slist_free_all(option_list);
859 curl_easy_cleanup(curlHandle);
860 return true;
861 } else {
862 if (curlResultCode != CURLE_OK) {
863 LOGP(alarm, "invalid URL {}", fullUrl);
864 } else {
865 LOGP(alarm, "not found under link {}", fullUrl);
866 }
867 }
868 }
869 curl_slist_free_all(option_list);
870 }
871
872 curl_easy_cleanup(curlHandle);
873 }
874 return false;
875}
876
877TObject* CcdbApi::retrieve(std::string const& path, std::map<std::string, std::string> const& metadata,
878 long timestamp) const
879{
880 struct MemoryStruct chunk{
881 (char*)malloc(1) /*memory*/, 0 /*size*/
882 };
883
884 TObject* result = nullptr;
885
886 bool res = receiveToMemory((void*)&chunk, path, metadata, timestamp);
887
888 if (res) {
889 std::lock_guard<std::mutex> guard(gIOMutex);
890 TMessage mess(kMESS_OBJECT);
891 mess.SetBuffer(chunk.memory, chunk.size, kFALSE);
892 mess.SetReadMode();
893 mess.Reset();
894 result = (TObject*)(mess.ReadObjectAny(mess.GetClass()));
895 if (result == nullptr) {
896 LOGP(info, "couldn't retrieve the object {}", path);
897 }
898 }
899
900 free(chunk.memory);
901
902 return result;
903}
904
905std::string CcdbApi::generateFileName(const std::string& inp)
906{
907 // generate file name for the CCDB object (for now augment the input string by the timestamp)
908 std::string str = inp;
909 str.erase(std::remove_if(str.begin(), str.end(), ::isspace), str.end());
910 str = std::regex_replace(str, std::regex("::"), "-");
912 return str;
913}
914
915TObject* CcdbApi::retrieveFromTFile(std::string const& path, std::map<std::string, std::string> const& metadata,
916 long timestamp, std::map<std::string, std::string>* headers, std::string const& etag,
917 const std::string& createdNotAfter, const std::string& createdNotBefore) const
918{
919 return (TObject*)retrieveFromTFile(typeid(TObject), path, metadata, timestamp, headers, etag, createdNotAfter, createdNotBefore);
920}
921
922bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir, std::map<std::string, std::string> const& metadata,
923 long timestamp, bool preservePath, std::string const& localFileName, std::string const& createdNotAfter, std::string const& createdNotBefore, std::map<std::string, std::string>* outHeaders) const
924{
925
926 // we setup the target path for this blob
927 std::string fulltargetdir = targetdir + (preservePath ? ('/' + path) : "");
928
929 try {
931 } catch (std::exception e) {
932 LOGP(error, "Could not create local snapshot cache directory {}, reason: {}", fulltargetdir, e.what());
933 return false;
934 }
935
937 std::map<std::string, std::string> headers;
938 // avoid creating snapshot via loadFileToMemory itself
939 loadFileToMemory(buff, path, metadata, timestamp, &headers, "", createdNotAfter, createdNotBefore, false);
940 if ((headers.count("Error") != 0) || (buff.empty())) {
941 LOGP(error, "Unable to find object {}/{}, Aborting", path, timestamp);
942 return false;
943 }
944 // determine local filename --> use user given one / default -- or if empty string determine from content
945 auto getFileName = [&headers]() {
946 auto& s = headers["Content-Disposition"];
947 if (s != "") {
948 std::regex re("(.*;)filename=\"(.*)\"");
949 std::cmatch m;
950 if (std::regex_match(s.c_str(), m, re)) {
951 return m[2].str();
952 }
953 }
954 std::string backupname("ccdb-blob.bin");
955 LOG(error) << "Cannot determine original filename from Content-Disposition ... falling back to " << backupname;
956 return backupname;
957 };
958 auto filename = localFileName.size() > 0 ? localFileName : getFileName();
959 std::string targetpath = fulltargetdir + "/" + filename;
960 {
961 std::ofstream objFile(targetpath, std::ios::out | std::ofstream::binary);
962 std::copy(buff.begin(), buff.end(), std::ostreambuf_iterator<char>(objFile));
963 if (!objFile.good()) {
964 LOGP(error, "Unable to open local file {}, Aborting", targetpath);
965 return false;
966 }
967 }
968 CCDBQuery querysummary(path, metadata, timestamp);
969
970 updateMetaInformationInLocalFile(targetpath.c_str(), &headers, &querysummary);
971 if (outHeaders) {
972 *outHeaders = std::move(headers);
973 }
974 return true;
975}
976
977void CcdbApi::snapshot(std::string const& ccdbrootpath, std::string const& localDir, long timestamp) const
978{
979 // query all subpaths to ccdbrootpath
980 const auto allfolders = getAllFolders(ccdbrootpath);
981 std::map<std::string, std::string> metadata;
982 for (auto& folder : allfolders) {
983 retrieveBlob(folder, localDir, metadata, timestamp);
984 }
985}
986
987void* CcdbApi::extractFromTFile(TFile& file, TClass const* cl, const char* what)
988{
989 if (!cl) {
990 return nullptr;
991 }
992#if defined(_LIBCPP_VERSION)
993 o2::framework::DataRefUtils::registerLibstdcxxStdArray(file, cl);
994#endif
995 auto object = file.GetObjectChecked(what, cl);
996 if (!object) {
997 // it could be that object was stored with previous convention
998 // where the classname was taken as key
999 std::string objectName(cl->GetName());
1000 o2::utils::Str::trim(objectName);
1001 object = file.GetObjectChecked(objectName.c_str(), cl);
1002 LOG(warn) << "Did not find object under expected name " << what;
1003 if (!object) {
1004 return nullptr;
1005 }
1006 LOG(warn) << "Found object under deprecated name " << cl->GetName();
1007 }
1008 auto result = object;
1009 // We need to handle some specific cases as ROOT ties them deeply
1010 // to the file they are contained in
1011 // IsTObject avoids InheritsFrom, which makes cling instantiate STL collection classes
1012 if (cl->IsTObject()) {
1013 // make a clone
1014 // detach from the file
1015 auto tree = dynamic_cast<TTree*>((TObject*)object);
1016 if (tree) {
1017 tree->LoadBaskets(0x1L << 32); // make tree memory based
1018 tree->SetDirectory(nullptr);
1019 result = tree;
1020 } else {
1021 auto h = dynamic_cast<TH1*>((TObject*)object);
1022 if (h) {
1023 h->SetDirectory(nullptr);
1024 result = h;
1025 }
1026 }
1027 }
1028 return result;
1029}
1030
1031void* CcdbApi::extractFromLocalFile(std::string const& filename, std::type_info const& tinfo, std::map<std::string, std::string>* headers) const
1032{
1033 if (!std::filesystem::exists(filename)) {
1034 LOG(error) << "Local snapshot " << filename << " not found \n";
1035 return nullptr;
1036 }
1037 std::lock_guard<std::mutex> guard(gIOMutex);
1038 auto tcl = tinfo2TClass(tinfo);
1039 TFile f(filename.c_str(), "READ");
1040 if (headers) {
1041 auto storedmeta = retrieveMetaInfo(f);
1042 if (storedmeta) {
1043 *headers = *storedmeta; // do a simple deep copy
1044 delete storedmeta;
1045 }
1046 if ((isSnapshotMode() || mPreferSnapshotCache) && headers->find("ETag") == headers->end()) { // generate dummy ETag to profit from the caching
1047 (*headers)["ETag"] = filename;
1048 }
1049 if (headers->find("fileSize") == headers->end()) {
1050 (*headers)["fileSize"] = fmt::format("{}", f.GetEND());
1051 }
1052 }
1053 return extractFromTFile(f, tcl);
1054}
1055
1056bool CcdbApi::initTGrid() const
1057{
1058 if (mNeedAlienToken && !gGrid) {
1059 static bool allowNoToken = getenv("ALICEO2_CCDB_NOTOKENCHECK") && atoi(getenv("ALICEO2_CCDB_NOTOKENCHECK"));
1060 if (!allowNoToken && !checkAlienToken()) {
1061 LOG(fatal) << "Alien Token Check failed - Please get an alien token before running with https CCDB endpoint, or alice-ccdb.cern.ch!";
1062 }
1063 TGrid::Connect("alien");
1064 static bool errorShown = false;
1065 if (!gGrid && errorShown == false) {
1066 if (allowNoToken) {
1067 LOG(error) << "TGrid::Connect returned nullptr. May be due to missing alien token";
1068 } else {
1069 LOG(fatal) << "TGrid::Connect returned nullptr. May be due to missing alien token";
1070 }
1071 errorShown = true;
1072 }
1073 }
1074 return gGrid != nullptr;
1075}
1076
1077void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo, std::map<std::string, std::string>* headers) const
1078{
1079 if ((url.find("alien:/", 0) != std::string::npos) && !initTGrid()) {
1080 return nullptr;
1081 }
1082 std::lock_guard<std::mutex> guard(gIOMutex);
1083 auto memfile = TMemFile::Open(url.c_str(), "OPEN");
1084 if (memfile) {
1085 auto cl = tinfo2TClass(tinfo);
1086 auto content = extractFromTFile(*memfile, cl);
1087 if (headers && headers->find("fileSize") == headers->end()) {
1088 (*headers)["fileSize"] = fmt::format("{}", memfile->GetEND());
1089 }
1090 delete memfile;
1091 return content;
1092 }
1093 return nullptr;
1094}
1095
1096void* CcdbApi::interpretAsTMemFileAndExtract(char* contentptr, size_t contentsize, std::type_info const& tinfo)
1097{
1098 void* result = nullptr;
1099 Int_t previousErrorLevel = gErrorIgnoreLevel;
1100 gErrorIgnoreLevel = kFatal;
1101 std::lock_guard<std::mutex> guard(gIOMutex);
1102 TMemFile memFile("name", contentptr, contentsize, "READ");
1103 gErrorIgnoreLevel = previousErrorLevel;
1104 if (!memFile.IsZombie()) {
1105 auto tcl = tinfo2TClass(tinfo);
1106 result = extractFromTFile(memFile, tcl);
1107 if (!result) {
1108 LOG(error) << o2::utils::Str::concat_string("Couldn't retrieve object corresponding to ", tcl->GetName(), " from TFile");
1109 }
1110 memFile.Close();
1111 }
1112 return result;
1113}
1114
1115// navigate sequence of URLs until TFile content is found; object is extracted and returned
1116void* CcdbApi::navigateURLsAndRetrieveContent(CurlHandle* curl_handle, std::string const& url, std::type_info const& tinfo, std::map<std::string, std::string>* headers) const
1117{
1118 // a global internal data structure that can be filled with HTTP header information
1119 // static --> to avoid frequent alloc/dealloc as optimization
1120 // not sure if thread_local takes away that benefit
1121 static thread_local std::multimap<std::string, std::string> headerData;
1122
1123 // let's see first of all if the url is something specific that curl cannot handle
1124 if ((url.find("alien:/", 0) != std::string::npos) || (url.find("file:/", 0) != std::string::npos)) {
1125 return downloadFilesystemContent(url, tinfo, headers);
1126 }
1127 // add other final cases here
1128 // example root://
1129
1130 // otherwise make an HTTP/CURL request
1131 // specify URL to get
1132 curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
1133
1134 MemoryStruct chunk{(char*)malloc(1), 0};
1135 initCurlOptionsForRetrieve(curl_handle, (void*)&chunk, WriteMemoryCallback, false);
1136
1137 curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback<decltype(headerData)>);
1138 headerData.clear();
1139 curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (void*)&headerData);
1140
1141 curlSetSSLOptions(curl_handle);
1142
1143 auto res = static_cast<CURLcode>(CURL_perform(curl_handle));
1144 long response_code = -1;
1145 void* content = nullptr;
1146 bool errorflag = false;
1147 if (res == CURLE_OK && curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code) == CURLE_OK) {
1148 if (headers) {
1149 for (auto& p : headerData) {
1150 (*headers)[p.first] = p.second;
1151 }
1152 }
1153 if (200 <= response_code && response_code < 300) {
1154 // good response and the content is directly provided and should have been dumped into "chunk"
1155 content = interpretAsTMemFileAndExtract(chunk.memory, chunk.size, tinfo);
1156 if (headers && headers->find("fileSize") == headers->end()) {
1157 (*headers)["fileSize"] = fmt::format("{}", chunk.size);
1158 }
1159 } else if (response_code == 304) {
1160 // this means the object exist but I am not serving
1161 // it since it's already in your possession
1162
1163 // there is nothing to be done here
1164 LOGP(debug, "Object exists but I am not serving it since it's already in your possession");
1165 }
1166 // this is a more general redirection
1167 else if (300 <= response_code && response_code < 400) {
1168 // we try content locations in order of appearance until one succeeds
1169 // 1st: The "Location" field
1170 // 2nd: Possible "Content-Location" fields - Location field
1171
1172 // some locations are relative to the main server so we need to fix/complement them
1173 auto complement_Location = [this](std::string const& loc) {
1174 if (loc[0] == '/') {
1175 // if it's just a path (noticed by trailing '/' we prepend the server url
1176 return getURL() + loc;
1177 }
1178 return loc;
1179 };
1180
1181 std::vector<std::string> locs;
1182 auto iter = headerData.find("Location");
1183 if (iter != headerData.end()) {
1184 locs.push_back(complement_Location(iter->second));
1185 }
1186 // add alternative locations (not yet included)
1187 auto iter2 = headerData.find("Content-Location");
1188 if (iter2 != headerData.end()) {
1189 auto range = headerData.equal_range("Content-Location");
1190 for (auto it = range.first; it != range.second; ++it) {
1191 if (std::find(locs.begin(), locs.end(), it->second) == locs.end()) {
1192 locs.push_back(complement_Location(it->second));
1193 }
1194 }
1195 }
1196 for (auto& l : locs) {
1197 if (l.size() > 0) {
1198 LOG(debug) << "Trying content location " << l;
1199 content = navigateURLsAndRetrieveContent(curl_handle, l, tinfo, headers);
1200 if (content /* or other success marker in future */) {
1201 break;
1202 }
1203 }
1204 }
1205 } else if (response_code == 404) {
1206 LOG(error) << "Requested resource does not exist: " << url;
1207 errorflag = true;
1208 } else {
1209 LOG(error) << "Error in fetching object " << url << ", curl response code:" << response_code;
1210 errorflag = true;
1211 }
1212 // cleanup
1213 if (chunk.memory != nullptr) {
1214 free(chunk.memory);
1215 }
1216 } else {
1217 LOGP(alarm, "Curl request to {} failed with result {}, response code: {}", url, int(res), response_code);
1218 errorflag = true;
1219 }
1220 // indicate that an error occurred ---> used by caching layers (such as CCDBManager)
1221 if (errorflag && headers) {
1222 (*headers)["Error"] = "An error occurred during retrieval";
1223 }
1224 return content;
1225}
1226
1227void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const& path,
1228 std::map<std::string, std::string> const& metadata, long timestamp,
1229 std::map<std::string, std::string>* headers, std::string const& etag,
1230 const std::string& createdNotAfter, const std::string& createdNotBefore) const
1231{
1232 if (!mSnapshotCachePath.empty()) {
1233 // protect this sensitive section by a multi-process named semaphore
1234 auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath, path);
1235 std::string logfile = mSnapshotCachePath + "/log";
1236 std::fstream out(logfile, ios_base::out | ios_base::app);
1237 if (out.is_open()) {
1238 out << "CCDB-access[" << getpid() << "] of " << mUniqueAgentID << " to " << path << " timestamp " << timestamp << "\n";
1239 }
1240 auto snapshotfile = getSnapshotFile(mSnapshotCachePath, path);
1241 bool snapshoting = false;
1242 if (!std::filesystem::exists(snapshotfile)) {
1243 snapshoting = true;
1244 out << "CCDB-access[" << getpid() << "] ... " << mUniqueAgentID << " downloading to snapshot " << snapshotfile << "\n";
1245 // if file not already here and valid --> snapshot it
1246 if (!retrieveBlob(path, mSnapshotCachePath, metadata, timestamp)) {
1247 out << "CCDB-access[" << getpid() << "] ... " << mUniqueAgentID << " failed to create directory for " << snapshotfile << "\n";
1248 }
1249 } else {
1250 out << "CCDB-access[" << getpid() << "] ... " << mUniqueAgentID << "serving from local snapshot " << snapshotfile << "\n";
1251 }
1252
1253 auto res = extractFromLocalFile(snapshotfile, tinfo, headers);
1254 if (!snapshoting) { // if snapshot was created at this call, the log was already done
1255 logReading(path, timestamp, headers, "retrieve from snapshot");
1256 }
1257 return res;
1258 }
1259
1260 // normal mode follows
1261
1262 CurlHandle* curl_handle = curl_easy_init();
1263 curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1264 std::string fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp); // todo check if function still works correctly in case mInSnapshotMode
1265 // if we are in snapshot mode we can simply open the file; extract the object and return
1266 if (mInSnapshotMode) {
1267 auto res = extractFromLocalFile(fullUrl, tinfo, headers);
1268 if (res) {
1269 logReading(path, timestamp, headers, "retrieve from snapshot");
1270 }
1271 return res;
1272 }
1273
1274 curl_slist* option_list = nullptr;
1275 initCurlHTTPHeaderOptionsForRetrieve(curl_handle, option_list, timestamp, headers, etag, createdNotAfter, createdNotBefore, fullUrl);
1276 auto content = navigateURLsAndRetrieveContent(curl_handle, fullUrl, tinfo, headers);
1277
1278 for (size_t hostIndex = 1; hostIndex < hostsPool.size() && !(content); hostIndex++) {
1279 fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp, hostIndex);
1280 // Per host: the gate token is per endpoint (see appendGateToken).
1281 curl_slist_free_all(option_list);
1282 option_list = nullptr;
1283 initCurlHTTPHeaderOptionsForRetrieve(curl_handle, option_list, timestamp, headers, etag, createdNotAfter, createdNotBefore, fullUrl);
1284 content = navigateURLsAndRetrieveContent(curl_handle, fullUrl, tinfo, headers);
1285 }
1286 if (content) {
1287 logReading(path, timestamp, headers, "retrieve");
1288 }
1289 curl_slist_free_all(option_list);
1290 curl_easy_cleanup(curl_handle);
1291 return content;
1292}
1293
1294size_t CurlWrite_CallbackFunc_StdString2(void* contents, size_t size, size_t nmemb, std::string* s)
1295{
1296 size_t newLength = size * nmemb;
1297 size_t oldLength = s->size();
1298 try {
1299 s->resize(oldLength + newLength);
1300 } catch (std::bad_alloc& e) {
1301 LOG(error) << "memory error when getting data from CCDB";
1302 return 0;
1303 }
1304
1305 std::copy((char*)contents, (char*)contents + newLength, s->begin() + oldLength);
1306 return size * nmemb;
1307}
1308
1309std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string const& returnFormat, long createdNotAfter, long createdNotBefore) const
1310{
1311 CurlHandle* curl;
1312 CURLcode res = CURL_LAST;
1313 std::string result;
1314
1315 curl = curl_easy_init();
1316 if (curl != nullptr) {
1317 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString2);
1318 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
1319 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1320
1321 curlSetSSLOptions(curl);
1322
1323 std::string fullUrl;
1324 // Perform the request, res will get the return code
1325 for (size_t hostIndex = 0; hostIndex < hostsPool.size() && res != CURLE_OK; hostIndex++) {
1326 fullUrl = getHostUrl(hostIndex);
1327 fullUrl += latestOnly ? "/latest/" : "/browse/";
1328 fullUrl += path;
1329 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
1330
1331 // Per host: the gate token is per endpoint (see appendGateToken).
1332 struct curl_slist* headers = nullptr;
1333 headers = curl_slist_append(headers, (std::string("Accept: ") + returnFormat).c_str());
1334 headers = curl_slist_append(headers, (std::string("Content-Type: ") + returnFormat).c_str());
1335 if (createdNotAfter >= 0) {
1336 headers = curl_slist_append(headers, ("If-Not-After: " + std::to_string(createdNotAfter)).c_str());
1337 }
1338 if (createdNotBefore >= 0) {
1339 headers = curl_slist_append(headers, ("If-Not-Before: " + std::to_string(createdNotBefore)).c_str());
1340 }
1341 headers = appendGateToken(headers, fullUrl);
1342 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
1343
1344 res = static_cast<CURLcode>(CURL_perform(curl));
1345 if (res != CURLE_OK) {
1346 LOGP(alarm, "CURL_perform() failed: {}", curl_easy_strerror(res));
1347 }
1348 curl_slist_free_all(headers);
1349 }
1350 curl_easy_cleanup(curl);
1351 }
1352
1353 return result;
1354}
1355
1356std::string CcdbApi::getTimestampString(long timestamp) const
1357{
1358 stringstream ss;
1359 ss << timestamp;
1360 return ss.str();
1361}
1362
1363void CcdbApi::deleteObject(std::string const& path, long timestamp) const
1364{
1365 CurlHandle* curl;
1366 CURLcode res;
1367 long timestampLocal = timestamp == -1 ? getCurrentTimestamp() : timestamp;
1368
1369 curl = curl_easy_init();
1370 if (curl != nullptr) {
1371 curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
1372 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1373 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1374 curlSetSSLOptions(curl);
1375
1376 for (size_t hostIndex = 0; hostIndex < hostsPool.size(); hostIndex++) {
1377 // Inside the loop: hoisted out, the stream accumulates and the second
1378 // host's URL is the first with the second appended.
1379 stringstream fullUrl;
1380 fullUrl << getHostUrl(hostIndex) << "/" << path << "/" << timestampLocal;
1381 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str());
1382
1383 // A DELETE is a write, so it needs the gate token as storing does -- per
1384 // host, since the token is per endpoint (see appendGateToken).
1385 struct curl_slist* list = appendGateToken(nullptr, fullUrl.str());
1386 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1387
1388 // Perform the request, res will get the return code
1389 res = static_cast<CURLcode>(CURL_perform(curl));
1390 if (res != CURLE_OK) {
1391 LOGP(alarm, "CURL_perform() failed: {}", curl_easy_strerror(res));
1392 }
1393 curl_slist_free_all(list);
1394 }
1395 // After the loop, not inside it: cleaning up per host left every later
1396 // iteration using a freed handle.
1397 curl_easy_cleanup(curl);
1398 }
1399}
1400
1401void CcdbApi::truncate(std::string const& path) const
1402{
1403 CurlHandle* curl;
1404 CURLcode res;
1405 for (size_t i = 0; i < hostsPool.size(); i++) {
1406 // Declared inside the loop: a stringstream hoisted out of it accumulates,
1407 // so the second host's URL would be the first one with the second appended
1408 // to it. Latent until now -- every caller used a single-host pool.
1409 stringstream fullUrl;
1410 std::string url = getHostUrl(i);
1411 fullUrl << url << "/truncate/" << path;
1412
1413 curl = curl_easy_init();
1414 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1415 if (curl != nullptr) {
1416 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str());
1417
1418 // Truncating is a write, so it needs the gate token exactly as storing
1419 // does. This was the one write path left without it, which a broker
1420 // answers 401 -- failing every CCDB suite in their teardown, since each
1421 // one truncates the path it just wrote.
1422 struct curl_slist* list = appendGateToken(nullptr, fullUrl.str());
1423 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1424 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1425 curlSetSSLOptions(curl);
1426
1427 // Perform the request, res will get the return code
1428 res = static_cast<CURLcode>(CURL_perform(curl));
1429 if (res != CURLE_OK) {
1430 LOGP(alarm, "CURL_perform() failed: {}", curl_easy_strerror(res));
1431 }
1432 curl_easy_cleanup(curl);
1433 curl_slist_free_all(list);
1434 }
1435 }
1436}
1437
1438size_t write_data(void*, size_t size, size_t nmemb, void*)
1439{
1440 return size * nmemb;
1441}
1442
1444{
1445 CurlHandle* curl;
1446 CURLcode res = CURL_LAST;
1447 bool result = false;
1448
1449 curl = curl_easy_init();
1450 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1451 if (curl) {
1452 // NOTE: mUrl, not getHostUrl(hostIndex), even though hostIndex is unused.
1453 // For a failover setup mUrl is the whole comma-separated list, which curl
1454 // rejects as malformed, so every multi-host instance reports itself
1455 // unreachable however healthy its hosts are -- and testCcdbApiMultipleUrls,
1456 // whose cases are gated on this, is skipped rather than run.
1457 //
1458 // Fixing it is a separate change: the suite then runs for the first time
1459 // and its storeAndRetrieve fails, so the multi-host store/retrieve path
1460 // needs looking at before this can be corrected. Callers outside the tests
1461 // are affected too -- HMPID/PedestalsCalculationSpec sets mWriteToDB from
1462 // this, and TPC workflows branch on it.
1463 for (size_t hostIndex = 0; hostIndex < hostsPool.size() && res != CURLE_OK; hostIndex++) {
1464 curl_easy_setopt(curl, CURLOPT_URL, mUrl.data());
1465 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
1466 curlSetSSLOptions(curl);
1467 res = static_cast<CURLcode>(CURL_perform(curl));
1468 result = (res == CURLE_OK);
1469 }
1470
1471 /* always cleanup */
1472 curl_easy_cleanup(curl);
1473 }
1474 return result;
1475}
1476
1477std::vector<std::string> CcdbApi::parseSubFolders(std::string const& reply) const
1478{
1479 // this needs some text filtering
1480 // go through reply line by line until we see "SubFolders:"
1481 std::stringstream ss(reply.c_str());
1482 std::string line;
1483 std::vector<std::string> folders;
1484
1485 size_t numberoflines = std::count(reply.begin(), reply.end(), '\n');
1486 bool inSubFolderSection = false;
1487
1488 for (size_t linenumber = 0; linenumber < numberoflines; ++linenumber) {
1489 std::getline(ss, line);
1490 if (inSubFolderSection && line.size() > 0) {
1491 // remove all white space
1492 folders.push_back(sanitizeObjectName(line));
1493 }
1494
1495 if (line.compare("Subfolders:") == 0) {
1496 inSubFolderSection = true;
1497 }
1498 }
1499 return folders;
1500}
1501
1502namespace
1503{
1504size_t header_callback(char* buffer, size_t size, size_t nitems, void* userdata)
1505{
1506 auto* headers = static_cast<std::vector<std::string>*>(userdata);
1507 auto header = std::string(buffer, size * nitems);
1508 headers->emplace_back(std::string(header.data()));
1509 return size * nitems;
1510}
1511} // namespace
1512
1513bool stdmap_to_jsonfile(std::map<std::string, std::string> const& meta, std::string const& filename)
1514{
1515
1516 // create directory structure if necessary
1517 auto p = std::filesystem::path(filename).parent_path();
1518 if (!std::filesystem::exists(p)) {
1519 std::filesystem::create_directories(p);
1520 }
1521
1522 rapidjson::StringBuffer buffer;
1523 rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
1524 writer.StartObject();
1525 for (const auto& pair : meta) {
1526 writer.Key(pair.first.c_str());
1527 writer.String(pair.second.c_str());
1528 }
1529 writer.EndObject();
1530
1531 // Write JSON to file
1532 std::ofstream file(filename);
1533 if (file.is_open()) {
1534 file << buffer.GetString();
1535 file.close();
1536 } else {
1537 return false;
1538 }
1539 return true;
1540}
1541
1542bool jsonfile_to_stdmap(std::map<std::string, std::string>& meta, std::string const& filename)
1543{
1544 // Read JSON from file
1545 std::ifstream file(filename);
1546 if (!file.is_open()) {
1547 std::cerr << "Failed to open file for reading." << std::endl;
1548 return false;
1549 }
1550
1551 std::string jsonStr((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
1552
1553 // Parse JSON
1554 rapidjson::Document document;
1555 document.Parse(jsonStr.c_str());
1556
1557 if (document.HasParseError()) {
1558 std::cerr << "Error parsing JSON" << std::endl;
1559 return false;
1560 }
1561
1562 // Convert JSON to std::map
1563 for (auto itr = document.MemberBegin(); itr != document.MemberEnd(); ++itr) {
1564 meta[itr->name.GetString()] = itr->value.GetString();
1565 }
1566 return true;
1567}
1568
1569std::map<std::string, std::string> CcdbApi::retrieveHeaders(std::string const& path, std::map<std::string, std::string> const& metadata, long timestamp) const
1570{
1571 // lambda that actually does the call to the CCDB server
1572 auto do_remote_header_call = [this, &path, &metadata, timestamp]() -> std::map<std::string, std::string> {
1573 CurlHandle* curl = curl_easy_init();
1574 CURLcode res = CURL_LAST;
1575 std::string fullUrl = getFullUrlForRetrieval(curl, path, metadata, timestamp);
1576 std::map<std::string, std::string> headers;
1577
1578 if (curl != nullptr) {
1579 struct curl_slist* list = nullptr;
1580 list = curl_slist_append(list, ("If-None-Match: " + std::to_string(timestamp)).c_str());
1581 list = appendGateToken(list, fullUrl);
1582
1583 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1584
1585 /* get us the resource without a body! */
1586 curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
1587 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1588 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_map_callback<>);
1589 curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);
1590 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1591
1592 curlSetSSLOptions(curl);
1593
1594 // Perform the request, res will get the return code
1595 long httpCode = 404;
1596 CURLcode getCodeRes = CURL_LAST;
1597 for (size_t hostIndex = 0; hostIndex < hostsPool.size() && (httpCode >= 400 || res > 0 || getCodeRes > 0); hostIndex++) {
1598 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
1599 res = static_cast<CURLcode>(CURL_perform(curl));
1600 if (res != CURLE_OK && res != CURLE_UNSUPPORTED_PROTOCOL) {
1601 // We take out the unsupported protocol error because we are only querying
1602 // header info which is returned in any case. Unsupported protocol error
1603 // occurs sometimes because of redirection to alien for blobs.
1604 LOG(error) << "CURL_perform() failed: " << curl_easy_strerror(res);
1605 }
1606 getCodeRes = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpCode);
1607 }
1608 if (httpCode == 404) {
1609 headers.clear();
1610 }
1611 curl_easy_cleanup(curl);
1612 }
1613 return headers;
1614 };
1615
1616 if (!mSnapshotCachePath.empty()) {
1617 // protect this sensitive section by a multi-process named semaphore
1618 auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath + std::string("_headers"), path);
1619
1620 std::string logfile = mSnapshotCachePath + "/log";
1621 std::fstream out(logfile, ios_base::out | ios_base::app);
1622 if (out.is_open()) {
1623 out << "CCDB-header-access[" << getpid() << "] of " << mUniqueAgentID << " to " << path << " timestamp " << timestamp << "\n";
1624 }
1625 auto snapshotfile = getSnapshotFile(mSnapshotCachePath, path + "/" + std::to_string(timestamp), "header.json");
1626 if (!std::filesystem::exists(snapshotfile)) {
1627 out << "CCDB-header-access[" << getpid() << "] ... " << mUniqueAgentID << " storing to snapshot " << snapshotfile << "\n";
1628
1629 // if file not already here and valid --> snapshot it
1630 auto meta = do_remote_header_call();
1631
1632 // cache the result
1633 if (!stdmap_to_jsonfile(meta, snapshotfile)) {
1634 LOG(warn) << "Failed to cache the header information to disc";
1635 }
1636 return meta;
1637 } else {
1638 out << "CCDB-header-access[" << getpid() << "] ... " << mUniqueAgentID << "serving from local snapshot " << snapshotfile << "\n";
1639 std::map<std::string, std::string> meta;
1640 if (!jsonfile_to_stdmap(meta, snapshotfile)) {
1641 LOG(warn) << "Failed to read cached information from disc";
1642 return do_remote_header_call();
1643 }
1644 return meta;
1645 }
1646 }
1647 return do_remote_header_call();
1648}
1649
1650bool CcdbApi::getCCDBEntryHeaders(std::string const& url, std::string const& etag, std::vector<std::string>& headers, const std::string& agentID)
1651{
1652 auto curl = curl_easy_init();
1653 headers.clear();
1654 if (!curl) {
1655 return true;
1656 }
1657
1658 struct curl_slist* list = nullptr;
1659 list = curl_slist_append(list, ("If-None-Match: " + etag).c_str());
1660 list = appendGateToken(list, url);
1661
1662 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1663
1664 curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
1665 /* get us the resource without a body! */
1666 curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
1667 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1668 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
1669 curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);
1670 if (!agentID.empty()) {
1671 curl_easy_setopt(curl, CURLOPT_USERAGENT, agentID.c_str());
1672 }
1673
1674 curlSetSSLOptions(curl);
1675
1676 /* Perform the request */
1677 curl_easy_perform(curl);
1678 long http_code = 404;
1679 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
1680 if (http_code == 304) {
1681 return false;
1682 }
1683 return true;
1684}
1685
1686void CcdbApi::parseCCDBHeaders(std::vector<std::string> const& headers, std::vector<std::string>& pfns, std::string& etag)
1687{
1688 static std::string etagHeader = "ETag: ";
1689 static std::string locationHeader = "Content-Location: ";
1690 // Trimmed: `headers` holds raw header lines, CRLF and all, and the etag goes
1691 // straight back out as an If-None-Match request header.
1692 for (auto h : headers) {
1693 if (h.find(etagHeader) == 0) {
1694 etag = trimHeaderValue(std::string_view(h).substr(etagHeader.size()));
1695 } else if (h.find(locationHeader) == 0) {
1696 pfns.emplace_back(trimHeaderValue(std::string_view(h).substr(locationHeader.size())));
1697 }
1698 }
1699}
1700
1702{
1703 auto object = file.GetObjectChecked(CCDBQUERY_ENTRY, TClass::GetClass(typeid(o2::ccdb::CCDBQuery)));
1704 if (object) {
1705 return static_cast<CCDBQuery*>(object);
1706 }
1707 return nullptr;
1708}
1709
1710std::map<std::string, std::string>* CcdbApi::retrieveMetaInfo(TFile& file)
1711{
1712 auto object = file.GetObjectChecked(CCDBMETA_ENTRY, TClass::GetClass(typeid(std::map<std::string, std::string>)));
1713 if (object) {
1714 return static_cast<std::map<std::string, std::string>*>(object);
1715 }
1716 return nullptr;
1717}
1718
1719namespace
1720{
1721void traverseAndFillFolders(CcdbApi const& api, std::string const& top, std::vector<std::string>& folders)
1722{
1723 // LOG(info) << "Querying " << top;
1724 auto reply = api.list(top);
1725 folders.emplace_back(top);
1726 // LOG(info) << reply;
1727 auto subfolders = api.parseSubFolders(reply);
1728 if (subfolders.size() > 0) {
1729 // LOG(info) << subfolders.size() << " folders in " << top;
1730 for (auto& sub : subfolders) {
1731 traverseAndFillFolders(api, sub, folders);
1732 }
1733 } else {
1734 // LOG(info) << "NO subfolders in " << top;
1735 }
1736}
1737} // namespace
1738
1739std::vector<std::string> CcdbApi::getAllFolders(std::string const& top) const
1740{
1741 std::vector<std::string> folders;
1742 traverseAndFillFolders(*this, top, folders);
1743 return folders;
1744}
1745
1746TClass* CcdbApi::tinfo2TClass(std::type_info const& tinfo)
1747{
1748 TClass* cl = TClass::GetClass(tinfo);
1749 if (!cl) {
1750 throw std::runtime_error(fmt::format("Could not retrieve ROOT dictionary for type {}, aborting", tinfo.name()));
1751 return nullptr;
1752 }
1753 return cl;
1754}
1755
1756int CcdbApi::updateMetadata(std::string const& path, std::map<std::string, std::string> const& metadata, long timestamp, std::string const& id, long newEOV)
1757{
1758 int ret = -1;
1759 CurlHandle* curl = curl_easy_init();
1760 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1761 if (curl != nullptr) {
1762 CURLcode res;
1763 for (size_t hostIndex = 0; hostIndex < hostsPool.size(); hostIndex++) {
1764 // Inside the loop: hoisted out, the stream accumulates and the second
1765 // host's URL is the first with the second appended.
1766 stringstream fullUrl;
1767 fullUrl << getHostUrl(hostIndex) << "/" << path << "/" << timestamp;
1768 if (newEOV > 0) {
1769 fullUrl << "/" << newEOV;
1770 }
1771 if (!id.empty()) {
1772 fullUrl << "/" << id;
1773 }
1774 fullUrl << "?";
1775
1776 for (auto& kv : metadata) {
1777 std::string mfirst = kv.first;
1778 std::string msecond = kv.second;
1779 // same trick for the metadata as for the object type
1780 char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
1781 char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
1782 fullUrl << std::string(mfirstEncoded) + "=" + std::string(msecondEncoded) + "&";
1783 curl_free(mfirstEncoded);
1784 curl_free(msecondEncoded);
1785 }
1786
1787 if (curl != nullptr) {
1788 LOG(debug) << "passing to curl: " << fullUrl.str();
1789 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str());
1790 curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT"); // make sure we use PUT
1791 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1792 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1793 // A PUT is a write, so it needs the gate token as storing does -- per
1794 // host, since the token is per endpoint (see appendGateToken).
1795 struct curl_slist* list = appendGateToken(nullptr, fullUrl.str());
1796 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1797 curlSetSSLOptions(curl);
1798
1799 // Perform the request, res will get the return code
1800 res = static_cast<CURLcode>(CURL_perform(curl));
1801 if (res != CURLE_OK) {
1802 LOGP(alarm, "CURL_perform() failed: {}, code: {}", curl_easy_strerror(res), int(res));
1803 ret = int(res);
1804 } else {
1805 ret = 0;
1806 }
1807 curl_slist_free_all(list);
1808 }
1809 }
1810 // After the loop, not inside it: cleaning up per host left every later
1811 // iteration using a freed handle.
1812 curl_easy_cleanup(curl);
1813 }
1814 return ret;
1815}
1816
1817void CcdbApi::initHostsPool(std::string hosts)
1818{
1819 hostsPool.clear();
1820 auto splitted = hosts | std::views::transform([](char c) { return (c == ';') ? ',' : c; }) | std::views::split(',');
1821 for (auto&& part : splitted) {
1822 hostsPool.emplace_back(part.begin(), part.end());
1823 }
1824}
1825
1826std::string CcdbApi::getHostUrl(int hostIndex) const
1827{
1828 return hostsPool.at(hostIndex);
1829}
1830
1831void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCounter) const
1832{
1833 auto data = new DownloaderRequestData(); // Deleted in transferFinished of CCDBDownloader.cxx
1834 data->hoPair.object = &requestContext.dest;
1835
1836 std::function<bool(std::string)> localContentCallback = [this, &requestContext](std::string url) {
1837 return this->loadLocalContentToMemory(requestContext.dest, url);
1838 };
1839
1840 auto writeCallback = [](void* contents, size_t size, size_t nmemb, void* chunkptr) {
1841 auto& ho = *static_cast<HeaderObjectPair_t*>(chunkptr);
1842 auto& chunk = *ho.object;
1843 size_t realsize = size * nmemb, sz = 0;
1844 ho.counter++;
1845 try {
1846 if (chunk.capacity() < chunk.size() + realsize) {
1847 // estimate headers size when converted to annotated text string
1848 const char hannot[] = "header";
1849 size_t hsize = getFlatHeaderSize(ho.header);
1850 auto cl = ho.header.find("Content-Length");
1851 if (cl != ho.header.end()) {
1852 size_t sizeFromHeader = std::stol(cl->second);
1853 sz = hsize + std::max(chunk.size() * (sizeFromHeader ? 1 : 2) + realsize, sizeFromHeader);
1854 } else {
1855 sz = hsize + std::max(chunk.size() * 2, chunk.size() + realsize);
1856 // LOGP(debug, "SIZE IS NOT IN HEADER, allocate {}", sz);
1857 }
1858 chunk.reserve(sz);
1859 }
1860 char* contC = (char*)contents;
1861 chunk.insert(chunk.end(), contC, contC + realsize);
1862 } catch (std::exception e) {
1863 // LOGP(alarm, "failed to reserve {} bytes in CURL write callback (realsize = {}): {}", sz, realsize, e.what());
1864 realsize = 0;
1865 }
1866 return realsize;
1867 };
1868
1869 CurlHandle* curl_handle = curl_easy_init();
1870 curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1871 std::string fullUrl = getFullUrlForRetrieval(curl_handle, requestContext.path, requestContext.metadata, requestContext.timestamp);
1872
1873 data->headers = &requestContext.headers;
1874 data->hosts = hostsPool;
1875 data->path = requestContext.path;
1876 data->timestamp = requestContext.timestamp;
1877 data->localContentCallback = localContentCallback;
1878 data->userAgent = mUniqueAgentID;
1879
1880 // One header list per host, built HERE because this is where the gate-token
1881 // table is visible -- the downloader only indexes them. A single shared list
1882 // sent the first host's token to every host it failed over to, which a broker
1883 // answers 401: the failover then retrieved nothing while looking like a
1884 // network failure (testCcdbApi multi_host_test).
1885 data->optionsLists.reserve(hostsPool.size());
1886 for (size_t hostIndex = 0; hostIndex < hostsPool.size(); hostIndex++) {
1887 curl_slist* hostOptions = nullptr;
1888 const std::string hostUrl = getFullUrlForRetrieval(curl_handle, requestContext.path, requestContext.metadata,
1889 requestContext.timestamp, hostIndex);
1890 initCurlHTTPHeaderOptionsForRetrieve(curl_handle, hostOptions, requestContext.timestamp, &requestContext.headers,
1891 requestContext.etag, requestContext.createdNotAfter, requestContext.createdNotBefore,
1892 hostUrl);
1893 data->optionsLists.push_back(hostOptions);
1894 }
1895 // initCurlHTTPHeaderOptionsForRetrieve sets CURLOPT_HTTPHEADER as a side
1896 // effect, so the handle currently points at the LAST host's list. Point it
1897 // back at host 0, which is the one this transfer starts with.
1898 if (!data->optionsLists.empty()) {
1899 curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, data->optionsLists.front());
1900 }
1901
1902 curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str());
1903 initCurlOptionsForRetrieve(curl_handle, (void*)(&data->hoPair), writeCallback, false);
1904 curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback<decltype(data->hoPair.header)>);
1905 curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (void*)&(data->hoPair.header));
1906 curl_easy_setopt(curl_handle, CURLOPT_PRIVATE, (void*)data);
1907 curlSetSSLOptions(curl_handle);
1908
1909 asynchPerform(curl_handle, requestCounter);
1910}
1911
1912std::string CcdbApi::determineSemaphoreName(std::string const& basedir, std::string const& ccdbpath)
1913{
1914 std::hash<std::string> hasher;
1915 std::string semhashedstring = "aliceccdb" + std::to_string(hasher(basedir + ccdbpath)).substr(0, 16);
1916 return semhashedstring;
1917}
1918
1919boost::interprocess::named_semaphore* CcdbApi::createNamedSemaphore(std::string const& path) const
1920{
1921 std::string semhashedstring = determineSemaphoreName(mSnapshotCachePath, path);
1922 // LOG(info) << "Creating named semaphore with name " << semhashedstring.c_str();
1923 try {
1924 return new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, semhashedstring.c_str(), 1);
1925 } catch (std::exception e) {
1926 LOG(warn) << "Exception occurred during CCDB (cache) semaphore setup; Continuing without";
1927 return nullptr;
1928 }
1929}
1930
1931void CcdbApi::releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, std::string const& path) const
1932{
1933 if (sem) {
1934 sem->post();
1935 if (sem->try_wait()) { // if nobody else is waiting remove the semaphore resource
1936 sem->post();
1937 boost::interprocess::named_semaphore::remove(determineSemaphoreName(mSnapshotCachePath, path).c_str());
1938 }
1939 }
1940}
1941
1942bool CcdbApi::removeSemaphore(std::string const& semaname, bool remove)
1943{
1944 // removes a given named semaphore from the system
1945 try {
1946 boost::interprocess::named_semaphore semaphore(boost::interprocess::open_only, semaname.c_str());
1947 std::cout << "Found CCDB semaphore: " << semaname << "\n";
1948 if (remove) {
1949 auto success = boost::interprocess::named_semaphore::remove(semaname.c_str());
1950 if (success) {
1951 std::cout << "Removed CCDB semaphore: " << semaname << "\n";
1952 }
1953 return success;
1954 }
1955 return true;
1956 } catch (std::exception const& e) {
1957 // no EXISTING under this name semaphore found
1958 // nothing to be done
1959 }
1960 return false;
1961}
1962
1963// helper function checking for leaking semaphores associated to CCDB cache files and removing them
1964// walks a local CCDB snapshot tree and checks
1965void CcdbApi::removeLeakingSemaphores(std::string const& snapshotdir, bool remove)
1966{
1967 namespace fs = std::filesystem;
1968 std::string fileName{"snapshot.root"};
1969 try {
1970 auto absolutesnapshotdir = fs::weakly_canonical(fs::absolute(snapshotdir));
1971 for (const auto& entry : fs::recursive_directory_iterator(absolutesnapshotdir)) {
1972 if (entry.is_directory()) {
1973 const fs::path& currentDir = fs::canonical(fs::absolute(entry.path()));
1974 fs::path filePath = currentDir / fileName;
1975 if (fs::exists(filePath) && fs::is_regular_file(filePath)) {
1976 std::cout << "Directory with file '" << fileName << "': " << currentDir << std::endl;
1977
1978 // we need to obtain the path relative to snapshotdir
1979 auto pathtokens = o2::utils::Str::tokenize(currentDir, '/', true);
1980 auto numtokens = pathtokens.size();
1981 if (numtokens < 3) {
1982 // cannot be a CCDB path
1983 continue;
1984 }
1985 // path are last 3 entries
1986 std::string path = pathtokens[numtokens - 3] + "/" + pathtokens[numtokens - 2] + "/" + pathtokens[numtokens - 1];
1987 auto semaname = o2::ccdb::CcdbApi::determineSemaphoreName(absolutesnapshotdir, path);
1988 removeSemaphore(semaname, remove);
1989 }
1990 }
1991 }
1992 } catch (std::exception const& e) {
1993 LOG(info) << "Semaphore search had exception " << e.what();
1994 }
1995}
1996
1997void CcdbApi::getFromSnapshot(bool createSnapshot, std::string const& path,
1998 long timestamp, std::map<std::string, std::string>& headers,
1999 std::string& snapshotpath, o2::pmr::vector<char>& dest, int& fromSnapshot, std::string const& etag) const
2000{
2001 if (createSnapshot) { // create named semaphore
2002 std::string logfile = mSnapshotCachePath + "/log";
2003 std::fstream logStream = std::fstream(logfile, ios_base::out | ios_base::app);
2004 if (logStream.is_open()) {
2005 logStream << "CCDB-access[" << getpid() << "] of " << mUniqueAgentID << " to " << path << " timestamp " << timestamp << " for load to memory\n";
2006 }
2007 }
2008 if (mInSnapshotMode) { // file must be there, otherwise a fatal will be produced;
2009 if (etag.empty()) {
2010 loadFileToMemory(dest, getSnapshotFile(mSnapshotTopPath, path), &headers);
2011 }
2012 fromSnapshot = 1;
2013 } else if (mPreferSnapshotCache && std::filesystem::exists(snapshotpath)) {
2014 // if file is available, use it, otherwise cache it below from the server. Do this only when etag is empty since otherwise the object was already fetched and cached
2015 if (etag.empty()) {
2016 loadFileToMemory(dest, snapshotpath, &headers);
2017 }
2018 fromSnapshot = 2;
2019 }
2020}
2021
2022void CcdbApi::saveSnapshot(RequestContext& requestContext) const
2023{
2024 // Consider saving snapshot
2025 if (!mSnapshotCachePath.empty() && !(mInSnapshotMode && mSnapshotTopPath == mSnapshotCachePath)) { // store in the snapshot only if the object was not read from the snapshot
2026 auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath, requestContext.path);
2027
2028 auto snapshotdir = getSnapshotDir(mSnapshotCachePath, requestContext.path);
2029 std::string snapshotpath = getSnapshotFile(mSnapshotCachePath, requestContext.path);
2031 std::fstream logStream;
2032 if (logStream.is_open()) {
2033 logStream << "CCDB-access[" << getpid() << "] ... " << mUniqueAgentID << " downloading to snapshot " << snapshotpath << " from memory\n";
2034 }
2035 { // dump image to a file
2036 LOGP(debug, "creating snapshot {} -> {}", requestContext.path, snapshotpath);
2037 CCDBQuery querysummary(requestContext.path, requestContext.metadata, requestContext.timestamp);
2038 {
2039 std::ofstream objFile(snapshotpath, std::ios::out | std::ofstream::binary);
2040 std::copy(requestContext.dest.begin(), requestContext.dest.end(), std::ostreambuf_iterator<char>(objFile));
2041 }
2042 // now open the same file as root file and store metadata
2043 updateMetaInformationInLocalFile(snapshotpath, &requestContext.headers, &querysummary);
2044 }
2045 }
2046}
2047
2048void CcdbApi::loadFileToMemory(std::vector<char>& dest, std::string const& path,
2049 std::map<std::string, std::string> const& metadata, long timestamp,
2050 std::map<std::string, std::string>* headers, std::string const& etag,
2051 const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot) const
2052{
2054 destP.reserve(dest.size());
2055 loadFileToMemory(destP, path, metadata, timestamp, headers, etag, createdNotAfter, createdNotBefore, considerSnapshot);
2056 dest.clear();
2057 dest.reserve(destP.size());
2058 for (const auto c : destP) {
2059 dest.push_back(c);
2060 }
2061}
2062
2064 std::map<std::string, std::string> const& metadata, long timestamp,
2065 std::map<std::string, std::string>* headers, std::string const& etag,
2066 const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot) const
2067{
2068 RequestContext requestContext(dest, metadata, *headers);
2069 requestContext.path = path;
2070 // std::map<std::string, std::string> metadataCopy = metadata; // Create a copy because metadata will be passed as a pointer so it cannot be constant. The const in definition is for backwards compatability.
2071 // requestContext.metadata = metadataCopy;
2072 requestContext.timestamp = timestamp;
2073 requestContext.etag = etag;
2074 requestContext.createdNotAfter = createdNotAfter;
2075 requestContext.createdNotBefore = createdNotBefore;
2076 requestContext.considerSnapshot = considerSnapshot;
2077 std::vector<RequestContext> contexts = {requestContext};
2078 vectoredLoadFileToMemory(contexts);
2079}
2080
2081void CcdbApi::appendFlatHeader(o2::pmr::vector<char>& dest, const std::map<std::string, std::string>& headers)
2082{
2083 size_t hsize = getFlatHeaderSize(headers), cnt = dest.size();
2084 dest.resize(cnt + hsize);
2085 auto addString = [&dest, &cnt](const std::string& s) {
2086 for (char c : s) {
2087 dest[cnt++] = c;
2088 }
2089 dest[cnt++] = 0;
2090 };
2091
2092 for (auto& h : headers) {
2093 addString(h.first);
2094 addString(h.second);
2095 }
2096 *reinterpret_cast<int*>(&dest[cnt]) = hsize; // store size
2097 std::memcpy(&dest[cnt + sizeof(int)], FlatHeaderAnnot, sizeof(FlatHeaderAnnot)); // annotate the flattened headers map
2098}
2099
2100void CcdbApi::navigateSourcesAndLoadFile(RequestContext& requestContext, int& fromSnapshot, size_t* requestCounter) const
2101{
2102 LOGP(debug, "loadFileToMemory {} ETag=[{}]", requestContext.path, requestContext.etag);
2103 bool createSnapshot = requestContext.considerSnapshot && !mSnapshotCachePath.empty(); // create snaphot if absent
2104
2105 std::string snapshotpath;
2106 if (mInSnapshotMode || std::filesystem::exists(snapshotpath = getSnapshotFile(mSnapshotCachePath, requestContext.path))) {
2107 auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath, requestContext.path);
2108 // if we are in snapshot mode we can simply open the file, unless the etag is non-empty:
2109 // this would mean that the object was is already fetched and in this mode we don't to validity checks!
2110 getFromSnapshot(createSnapshot, requestContext.path, requestContext.timestamp, requestContext.headers, snapshotpath, requestContext.dest, fromSnapshot, requestContext.etag);
2111 } else { // look on the server
2112 scheduleDownload(requestContext, requestCounter);
2113 }
2114}
2115
2116void CcdbApi::vectoredLoadFileToMemory(std::vector<RequestContext>& requestContexts) const
2117{
2118 std::vector<int> fromSnapshots(requestContexts.size());
2119 size_t requestCounter = 0;
2120
2121 // Get files from snapshots and schedule downloads
2122 for (int i = 0; i < requestContexts.size(); i++) {
2123 // navigateSourcesAndLoadFile either retrieves file from snapshot immediately, or schedules it to be downloaded when mDownloader->runLoop is ran at a later time
2124 auto& requestContext = requestContexts.at(i);
2125 navigateSourcesAndLoadFile(requestContext, fromSnapshots.at(i), &requestCounter);
2126 }
2127
2128 // Download the rest
2129 while (requestCounter > 0) {
2130 mDownloader->runLoop(0);
2131 }
2132
2133 // Save snapshots
2134 for (int i = 0; i < requestContexts.size(); i++) {
2135 auto& requestContext = requestContexts.at(i);
2136 if (!requestContext.dest.empty()) {
2137 logReading(requestContext.path, requestContext.timestamp, &requestContext.headers,
2138 fmt::format("{}{}", requestContext.considerSnapshot ? "load to memory" : "retrieve", fromSnapshots.at(i) ? " from snapshot" : ""));
2139 if (requestContext.considerSnapshot && fromSnapshots.at(i) != 2) {
2140 saveSnapshot(requestContext);
2141 }
2142 }
2143 }
2144}
2145
2147{
2148 if (url.find("alien:/", 0) != std::string::npos) {
2149 std::map<std::string, std::string> localHeaders;
2150 loadFileToMemory(dest, url, &localHeaders, false);
2151 auto it = localHeaders.find("Error");
2152 if (it != localHeaders.end() && it->second == "An error occurred during retrieval") {
2153 return false;
2154 } else {
2155 return true;
2156 }
2157 }
2158 if ((url.find("file:/", 0) != std::string::npos)) {
2159 std::string path = url.substr(7);
2160 if (std::filesystem::exists(path)) {
2161 std::map<std::string, std::string> localHeaders;
2162 loadFileToMemory(dest, url, &localHeaders, o2::utils::Str::endsWith(path, ".root"));
2163 auto it = localHeaders.find("Error");
2164 if (it != localHeaders.end() && it->second == "An error occurred during retrieval") {
2165 return false;
2166 } else {
2167 return true;
2168 }
2169 }
2170 }
2171 return false;
2172}
2173
2174void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, const std::string& path, std::map<std::string, std::string>* localHeaders, bool fetchLocalMetaData) const
2175{
2176 // Read file to memory as vector. For special case of the locally cached file retriev metadata stored directly in the file
2177 constexpr size_t MaxCopySize = 0x1L << 25;
2178 auto signalError = [&dest, localHeaders]() {
2179 dest.clear();
2180 dest.reserve(1);
2181 if (localHeaders) { // indicate that an error occurred ---> used by caching layers (such as CCDBManager)
2182 (*localHeaders)["Error"] = "An error occurred during retrieval";
2183 }
2184 };
2185 if (path.find("alien:/") == 0 && !initTGrid()) {
2186 signalError();
2187 return;
2188 }
2189 std::string fname(path);
2190 if (fname.find("?filetype=raw") == std::string::npos) {
2191 fname += "?filetype=raw";
2192 }
2193 std::unique_ptr<TFile> sfile{TFile::Open(fname.c_str())};
2194 if (!sfile || sfile->IsZombie()) {
2195 LOG(error) << "Failed to open file " << fname;
2196 signalError();
2197 return;
2198 }
2199 size_t totalread = 0, fsize = sfile->GetSize(), b00 = sfile->GetBytesRead();
2200 dest.resize(fsize);
2201 char* dptr = dest.data();
2202 sfile->Seek(0);
2203 long nread = 0;
2204 do {
2205 size_t b0 = sfile->GetBytesRead(), b1 = b0 - b00;
2206 size_t readsize = fsize - b1 > MaxCopySize ? MaxCopySize : fsize - b1;
2207 if (readsize == 0) {
2208 break;
2209 }
2210 sfile->Seek(totalread, TFile::kBeg);
2211 bool failed = sfile->ReadBuffer(dptr, (Int_t)readsize);
2212 nread = sfile->GetBytesRead() - b0;
2213 if (failed || nread < 0) {
2214 LOG(error) << "failed to copy file " << fname << " to memory buffer";
2215 signalError();
2216 return;
2217 }
2218 dptr += nread;
2219 totalread += nread;
2220 } while (nread == (long)MaxCopySize);
2221
2222 if (localHeaders && fetchLocalMetaData) {
2223 TMemFile memFile("name", const_cast<char*>(dest.data()), dest.size(), "READ");
2224 auto storedmeta = (std::map<std::string, std::string>*)extractFromTFile(memFile, TClass::GetClass("std::map<std::string, std::string>"), CCDBMETA_ENTRY);
2225 if (storedmeta) {
2226 *localHeaders = *storedmeta; // do a simple deep copy
2227 delete storedmeta;
2228 }
2229 if ((isSnapshotMode() || mPreferSnapshotCache) && localHeaders->find("ETag") == localHeaders->end()) { // generate dummy ETag to profit from the caching
2230 (*localHeaders)["ETag"] = path;
2231 }
2232 if (localHeaders->find("fileSize") == localHeaders->end()) {
2233 (*localHeaders)["fileSize"] = fmt::format("{}", memFile.GetEND());
2234 }
2235 }
2236 return;
2237}
2238
2239void CcdbApi::checkMetadataKeys(std::map<std::string, std::string> const& metadata) const
2240{
2241
2242 // function to check if any key contains invalid characters
2243 // if so, a fatal will be issued
2244
2245 const std::regex regexPatternSearch(R"([ :;.,\\/'?!\‍(\)\{\}\[\]@<>=+*#$&`|~^%])");
2246 bool isInvalid = false;
2247
2248 for (auto& el : metadata) {
2249 auto keyMd = el.first;
2250 auto tmp = keyMd;
2251 std::smatch searchRes;
2252 while (std::regex_search(keyMd, searchRes, regexPatternSearch)) {
2253 isInvalid = true;
2254 LOG(error) << "Invalid character found in metadata key '" << tmp << "\': '" << searchRes.str() << "\'";
2255 keyMd = searchRes.suffix();
2256 }
2257 }
2258 if (isInvalid) {
2259 LOG(fatal) << "Some metadata keys have invalid characters, please fix!";
2260 }
2261 return;
2262}
2263
2264void CcdbApi::logReading(const std::string& path, long ts, const std::map<std::string, std::string>* headers, const std::string& comment) const
2265{
2266 std::string upath{path};
2267 if (headers) {
2268 auto ent = headers->find("Valid-From");
2269 if (ent != headers->end()) {
2270 upath += "/" + ent->second;
2271 }
2272 ent = headers->find("ETag");
2273 if (ent != headers->end()) {
2274 upath += "/" + ent->second;
2275 }
2276 }
2277 upath.erase(remove(upath.begin(), upath.end(), '\"'), upath.end());
2278 LOGP(info, "ccdb reads {}{}{} for {} ({}, agent_id: {}), ", mUrl, mUrl.back() == '/' ? "" : "/", upath, ts < 0 ? getCurrentTimestamp() : ts, comment, mUniqueAgentID);
2279}
2280
2281void CcdbApi::asynchPerform(CurlHandle* handle, size_t* requestCounter) const
2282{
2283 mDownloader->asynchSchedule(handle, requestCounter);
2284}
2285
2286int CcdbApi::CURL_perform(CurlHandle* handle) const
2287{
2288 if (mIsCCDBDownloaderPreferred) {
2289 return mDownloader->perform(handle);
2290 }
2291 CURLcode result;
2292 for (int i = 1; i <= mCurlRetries && (result = curl_easy_perform(handle)) != CURLE_OK; i++) {
2293 usleep(mCurlDelayRetries * i);
2294 }
2295 return result;
2296}
2297
2302CCDBSemaphore::CCDBSemaphore(std::string const& snapshotpath, std::string const& path)
2303{
2304 LOG(debug) << "Entering semaphore barrier";
2305 mSemName = CcdbApi::determineSemaphoreName(snapshotpath, path);
2306 try {
2307 mSem = new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, mSemName.c_str(), 1);
2308 } catch (std::exception e) {
2309 LOG(warn) << "Exception occurred during CCDB (cache) semaphore setup; Continuing without";
2310 mSem = nullptr;
2311 }
2312 // automatically wait
2313 if (mSem) {
2314 gSemaRegistry.add(this);
2315 mSem->wait();
2316 }
2317}
2318
2320{
2321 LOG(debug) << "Ending semaphore barrier";
2322 if (mSem) {
2323 mSem->post();
2324 if (mSem->try_wait()) { // if nobody else is waiting remove the semaphore resource
2325 mSem->post();
2326 boost::interprocess::named_semaphore::remove(mSemName.c_str());
2327 }
2328 gSemaRegistry.remove(this);
2329 }
2330}
2331
2333{
2334 LOG(debug) << "Cleaning up semaphore registry with count " << mStore.size();
2335 for (auto& s : mStore) {
2336 delete s;
2337 mStore.erase(s);
2338 }
2339}
2340
2342{
2343 mStore.insert(ptr);
2344}
2345
2347{
2348 mStore.erase(ptr);
2349}
2350
2351} // namespace o2::ccdb
std::string createdNotBefore
std::string createdNotAfter
size_t maxSize
std::string etag
std::string url
std::ostringstream debug
std::vector< std::string > header
std::vector< long > entries
int32_t i
configStandalone Light OpenGL display display_light Apply smoothing to points Enable Z buffer Highlight links Highlight seeds Highlight sector tracks Highlight final tracks Show individual Show related Highlight grid Show only individual collision Color clusters belonging to track objects Set line Enable drawing TPC data Enable drawing TOF data Enable drawing magnetic field Set field line step count Invert colors configStandalone Heavy OpenGL display display_heavy Show tracks with ITS contribution Show tracks with TOF contribution Enable propagation of loopers Show clusters at nominal x of pad row for early transformed data Split CE tracks when they cross the central Mark clusters Mark adjacent clusters Hide rejected tracks Apply filter on tracks to be displayed Separate increase X h configStandalone Camera window renderer OpenGL display display_camera Orient the camera such that the y axis is always upwards Full Screen MultiSample Anti Downsampling Anti Enable Vertical Sync Target frame or if the backend cannot limit to the refresh Use OpenGL indirect draws to reduce number of draw calls Display h configStandalone Vulkan display display_vulkan Have uniform buffers in host accessible device memory h configStandalone Internal display display_internal Number of updates the Qt GUI may send ahead of the last one the display has before it waits for the display Opacity of the console background Number of console lines the history is shortened to this when it reaches twice the size Time in seconds an info message fades out after setInfoTime h OpenGL display display Font size Do not use Freetype for font Renderer for event Frontend to use for event ROOT macros used as track filter Console script file
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char e
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc bool
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h configStandalone Processing proc_rtctech Use a file lock to serialize access to the cache folder Print launch bounds used for RTC code as debugging for exit after Allow RTC with slave GPUReconstruction instances with optConstexpr and optSpecialcode Maximum number of RTC cache files per backend in the cache folder
#define failed(...)
Definition Utils.h:42
char error[CURL_ERROR_SIZE]
CURL * handle
std::string className
Definition Objects.cxx:33
uint8_t errorflag
Definition RawData.h:0
uint32_t res
Definition RawData.h:0
uint32_t c
Definition RawData.h:2
TBranch * ptr
StringRef key
Class for time synchronization of RawReader instances.
void setRequestTimeoutTime(int timeoutMS)
CURLcode perform(CURL *handle)
void asynchSchedule(CURL *handle, size_t *requestCounter)
void setKeepaliveTimeoutTime(int timeoutMS)
CCDBSemaphore(std::string const &cachepath, std::string const &path)
Definition CcdbApi.cxx:2302
static std::string generateFileName(const std::string &inp)
Definition CcdbApi.cxx:905
std::string list(std::string const &path="", bool latestOnly=false, std::string const &returnFormat="text/plain", long createdNotAfter=-1, long createdNotBefore=-1) const
Definition CcdbApi.cxx:1309
int storeAsTFile_impl(const void *obj1, std::type_info const &info, std::string const &path, std::map< std::string, std::string > const &metadata, long startValidityTimestamp=-1, long endValidityTimestamp=-1, std::vector< char >::size_type maxSize=0) const
Definition CcdbApi.cxx:413
static bool checkAlienToken()
Definition CcdbApi.cxx:203
void runDownloaderLoop(bool noWait)
Definition CcdbApi.cxx:345
void releaseNamedSemaphore(boost::interprocess::named_semaphore *sem, std::string const &path) const
Definition CcdbApi.cxx:1931
static std::map< std::string, std::string > * retrieveMetaInfo(TFile &)
Definition CcdbApi.cxx:1710
void scheduleDownload(RequestContext &requestContext, size_t *requestCounter) const
Definition CcdbApi.cxx:1831
TObject * retrieve(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp) const
Definition CcdbApi.cxx:877
void init(std::string const &hosts)
Definition CcdbApi.cxx:240
TObject * retrieveFromTFile(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp, std::map< std::string, std::string > *headers, std::string const &etag, const std::string &createdNotAfter, const std::string &createdNotBefore) const
Definition CcdbApi.cxx:915
std::string const & getURL() const
Definition CcdbApi.h:93
void getFromSnapshot(bool createSnapshot, std::string const &path, long timestamp, std::map< std::string, std::string > &headers, std::string &snapshotpath, o2::pmr::vector< char > &dest, int &fromSnapshot, std::string const &etag) const
Definition CcdbApi.cxx:1997
bool loadLocalContentToMemory(o2::pmr::vector< char > &dest, std::string &url) const
Definition CcdbApi.cxx:2146
static void removeLeakingSemaphores(std::string const &basedir, bool remove=false)
Definition CcdbApi.cxx:1965
void saveSnapshot(RequestContext &requestContext) const
Definition CcdbApi.cxx:2022
static std::unique_ptr< std::vector< char > > createObjectImage(const T *obj, CcdbObjectInfo *info=nullptr)
Definition CcdbApi.h:109
static void * extractFromTFile(TFile &file, TClass const *cl, const char *what=CCDBOBJECT_ENTRY)
Definition CcdbApi.cxx:987
int storeAsTFile(const TObject *rootObject, std::string const &path, std::map< std::string, std::string > const &metadata, long startValidityTimestamp=-1, long endValidityTimestamp=-1, std::vector< char >::size_type maxSize=0) const
Definition CcdbApi.cxx:553
void snapshot(std::string const &ccdbrootpath, std::string const &localDir, long timestamp) const
Definition CcdbApi.cxx:977
bool isSnapshotMode() const
Definition CcdbApi.h:99
bool isHostReachable() const
Definition CcdbApi.cxx:1443
void loadFileToMemory(std::vector< char > &dest, std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp, std::map< std::string, std::string > *headers, std::string const &etag, const std::string &createdNotAfter, const std::string &createdNotBefore, bool considerSnapshot=true) const
Definition CcdbApi.cxx:2048
static std::string determineSemaphoreName(std::string const &basedir, std::string const &objectpath)
Definition CcdbApi.cxx:1912
void deleteObject(std::string const &path, long timestamp=-1) const
Definition CcdbApi.cxx:1363
static void appendFlatHeader(o2::pmr::vector< char > &dest, const std::map< std::string, std::string > &headers)
Definition CcdbApi.cxx:2081
std::vector< std::string > getAllFolders(std::string const &top) const
Definition CcdbApi.cxx:1739
void vectoredLoadFileToMemory(std::vector< RequestContext > &requestContext) const
Definition CcdbApi.cxx:2116
static void curlSetSSLOptions(CurlHandle *curl)
Definition CcdbApi.cxx:699
boost::interprocess::named_semaphore * createNamedSemaphore(std::string const &path) const
Definition CcdbApi.cxx:1919
static bool removeSemaphore(std::string const &name, bool remove=false)
Definition CcdbApi.cxx:1942
std::map< std::string, std::string > retrieveHeaders(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp=-1) const
Definition CcdbApi.cxx:1569
CcdbApi()
Default constructor.
Definition CcdbApi.cxx:176
static bool getCCDBEntryHeaders(std::string const &url, std::string const &etag, std::vector< std::string > &headers, const std::string &agentID="")
Definition CcdbApi.cxx:1650
static CCDBQuery * retrieveQueryInfo(TFile &)
Definition CcdbApi.cxx:1701
static constexpr const char * CCDBQUERY_ENTRY
Definition CcdbApi.h:341
void truncate(std::string const &path) const
Definition CcdbApi.cxx:1401
int storeAsBinaryFile(const char *buffer, size_t size, const std::string &fileName, const std::string &objectType, const std::string &path, const std::map< std::string, std::string > &metadata, long startValidityTimestamp, long endValidityTimestamp, std::vector< char >::size_type maxSize=0) const
Definition CcdbApi.cxx:429
static void parseCCDBHeaders(std::vector< std::string > const &headers, std::vector< std::string > &pfns, std::string &etag)
Definition CcdbApi.cxx:1686
bool retrieveBlob(std::string const &path, std::string const &targetdir, std::map< std::string, std::string > const &metadata, long timestamp, bool preservePathStructure=true, std::string const &localFileName="snapshot.root", std::string const &createdNotAfter="", std::string const &createdNotBefore="", std::map< std::string, std::string > *headers=nullptr) const
Definition CcdbApi.cxx:922
int updateMetadata(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp, std::string const &id="", long newEOV=0)
Definition CcdbApi.cxx:1756
static constexpr const char * CCDBMETA_ENTRY
Definition CcdbApi.h:342
static constexpr const char * CCDBOBJECT_ENTRY
Definition CcdbApi.h:343
void navigateSourcesAndLoadFile(RequestContext &requestContext, int &fromSnapshot, size_t *requestCounter) const
Definition CcdbApi.cxx:2100
std::vector< std::string > parseSubFolders(std::string const &reply) const
Definition CcdbApi.cxx:1477
virtual ~CcdbApi()
Default destructor.
Definition CcdbApi.cxx:192
void add(CCDBSemaphore const *ptr)
Definition CcdbApi.cxx:2341
void remove(CCDBSemaphore const *ptr)
Definition CcdbApi.cxx:2346
const GLfloat * m
Definition glcorearb.h:4066
GLuint64EXT * result
Definition glcorearb.h:5662
GLint void * img
Definition glcorearb.h:550
GLuint buffer
Definition glcorearb.h:655
GLsizeiptr size
Definition glcorearb.h:659
GLuint GLuint end
Definition glcorearb.h:469
GLdouble GLdouble GLdouble GLdouble top
Definition glcorearb.h:4077
GLint first
Definition glcorearb.h:399
GLdouble f
Definition glcorearb.h:310
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLenum GLint * range
Definition glcorearb.h:1899
GLsizei const GLfloat * value
Definition glcorearb.h:819
GLsizei const GLchar *const * path
Definition glcorearb.h:3591
GLuint object
Definition glcorearb.h:4041
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
GLuint GLuint stream
Definition glcorearb.h:1806
GLbitfield GLuint64 timeout
Definition glcorearb.h:1573
GLuint id
Definition glcorearb.h:650
information complementary to a CCDB object (path, metadata, startTimeValidity, endTimeValidity etc)
bool stdmap_to_jsonfile(std::map< std::string, std::string > const &meta, std::string const &filename)
Definition CcdbApi.cxx:1513
size_t write_data(void *, size_t size, size_t nmemb, void *)
Definition CcdbApi.cxx:1438
long getCurrentTimestamp()
returns the timestamp in long corresponding to "now"
size_t(*)(void *, size_t, size_t, void *) CurlWriteCallback
Definition CcdbApi.cxx:725
std::string sanitizeObjectName(const std::string &objectName)
Definition CcdbApi.cxx:376
std::mutex gIOMutex
Definition CcdbApi.cxx:67
bool jsonfile_to_stdmap(std::map< std::string, std::string > &meta, std::string const &filename)
Definition CcdbApi.cxx:1542
long getFutureTimestamp(int secondsInFuture)
returns the timestamp in long corresponding to "now + secondsInFuture"
void CurlHandle
stands in for libcurl's typedef void CURL without including <curl/curl.h>
Definition CcdbApi.h:63
size_t CurlWrite_CallbackFunc_StdString2(void *contents, size_t size, size_t nmemb, std::string *s)
Definition CcdbApi.cxx:1294
std::string timestamp() noexcept
Definition Clock.h:84
Defining ITS Vertex explicitly as messageable.
Definition Cartesian.h:288
std::vector< T, fair::mq::pmr::polymorphic_allocator< T > > vector
void createDirectoriesIfAbsent(std::string const &path)
std::string to_string(gsl::span< T, Size > span)
Definition common.h:52
std::string filename()
void empty(int)
std::map< std::string, std::string > const & metadata
Definition CcdbApi.h:366
std::map< std::string, std::string > & headers
Definition CcdbApi.h:368
o2::pmr::vector< char > & dest
Definition CcdbApi.h:364
o2::pmr::vector< char > * object
static DeploymentMode deploymentMode()
static std::string getClassName(const T &obj)
get the class name of the object
static std::unique_ptr< FileImage > createFileImage(const TObject &obj, const std::string &fileName, const std::string &objName)
static void trim(std::string &s)
Definition StringUtils.h:70
static std::vector< std::string > tokenize(const std::string &src, char delim, bool trimToken=true, bool skipEmpty=true)
static std::string concat_string(Ts const &... ts)
static bool endsWith(const std::string &s, const std::string &ending)
void * realloc(void *old, size_t n)
void * malloc(size_t n)
void free(void *p)
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))
const std::string str
uint64_t const void const *restrict const msg
Definition x9.h:153