34#include <TStreamerInfo.h>
38#include <fairlogger/Logger.h>
44#include <boost/algorithm/string.hpp>
47#include <boost/interprocess/sync/named_semaphore.hpp>
53#include <TAlienUserAgent.h>
54#if defined(_LIBCPP_VERSION)
57#include <unordered_set>
58#include "rapidjson/document.h"
59#include "rapidjson/writer.h"
60#include "rapidjson/stringbuffer.h"
68unique_ptr<TJAlienCredentials> CcdbApi::mJAlienCredentials =
nullptr;
76std::string_view trimHeaderValue(std::string_view
value)
78 constexpr std::string_view whitespace =
" \t\r\n";
79 const auto first =
value.find_first_not_of(whitespace);
80 return first == std::string_view::npos
91const std::vector<std::pair<std::string, std::string>>& gateTokenTable()
93 static const auto table = []() {
94 std::vector<std::pair<std::string, std::string>>
entries;
95 const char* spec = getenv(
"ALICEO2_CCDB_AUTH_TOKENS");
96 std::string_view rest = spec ? spec :
"";
97 while (!rest.empty()) {
98 const auto sep = rest.find(
';');
99 const auto entry = trimHeaderValue(rest.substr(0, sep));
100 rest = (sep == std::string_view::npos) ? std::string_view{} : rest.substr(sep + 1);
101 const auto eq =
entry.find(
'=');
102 if (eq == std::string_view::npos) {
105 auto url = trimHeaderValue(
entry.substr(0, eq));
108 const auto token = trimHeaderValue(
entry.substr(eq + 1));
109 while (
url.size() > 1 &&
url.back() ==
'/') {
110 url.remove_suffix(1);
112 if (!
url.empty() && !token.empty()) {
113 entries.emplace_back(std::string(
url), std::string(
"Authorization: Bearer ").append(token));
117 [](
const auto&
a,
const auto&
b) { return a.first.size() > b.first.size(); });
131curl_slist* appendGateToken(curl_slist* list, std::string_view
url)
133 for (
const auto& [prefix,
header] : gateTokenTable()) {
134 if (
url.substr(0, prefix.size()) == prefix &&
135 (
url.size() == prefix.size() ||
url[prefix.size()] ==
'/')) {
136 return curl_slist_append(list,
header.c_str());
156 boost::interprocess::named_semaphore* mSem =
nullptr;
157 std::string mSemName{};
172 std::unordered_set<CCDBSemaphore const*> mStore;
182 mIsCCDBDownloaderPreferred = 0;
183 if (deploymentMode == DeploymentMode::OnlineDDS && deploymentMode == DeploymentMode::OnlineECS && deploymentMode == DeploymentMode::OnlineAUX && deploymentMode == DeploymentMode::FST) {
184 mIsCCDBDownloaderPreferred = 1;
186 if (getenv(
"ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI")) {
187 mIsCCDBDownloaderPreferred = atoi(getenv(
"ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI"));
194 curl_global_cleanup();
198void CcdbApi::setUniqueAgentID()
200 mUniqueAgentID = TAlienUserAgent::BasedOnEnvironment().ToString();
206 LOG(
debug) <<
"On macOS we simply rely on TGrid::Connect(\"alien\").";
209 if (getenv(
"ALICEO2_CCDB_NOTOKENCHECK") && atoi(getenv(
"ALICEO2_CCDB_NOTOKENCHECK"))) {
212 if (getenv(
"JALIEN_TOKEN_CERT")) {
215 auto returncode = system(
"LD_PRELOAD= alien-token-info &> /dev/null");
216 if (returncode == -1) {
219 return returncode == 0;
222void CcdbApi::curlInit()
225 curl_global_init(CURL_GLOBAL_DEFAULT);
226 CcdbApi::mJAlienCredentials = std::make_unique<TJAlienCredentials>();
227 CcdbApi::mJAlienCredentials->loadCredentials();
228 CcdbApi::mJAlienCredentials->selectPreferedCredentials();
231 if (getenv(
"ALICEO2_CCDB_SOCKET_TIMEOUT")) {
232 auto timeoutMS = atoi(getenv(
"ALICEO2_CCDB_SOCKET_TIMEOUT"));
233 if (timeoutMS >= 0) {
234 LOG(info) <<
"Setting socket timeout to " << timeoutMS <<
" milliseconds";
243 throw std::invalid_argument(
"Empty url passed CcdbApi, cannot initialize. Aborting.");
248 constexpr const char* SNAPSHOTPREFIX =
"file://";
251 if (host.substr(0, 7).compare(SNAPSHOTPREFIX) == 0) {
252 auto path = host.substr(7);
253 initInSnapshotMode(
path);
274 std::string snapshotReport{};
275 const char* cachedir = getenv(
"ALICEO2_CCDB_LOCALCACHE");
276 namespace fs = std::filesystem;
278 if (cachedir[0] == 0) {
279 mSnapshotCachePath = fs::weakly_canonical(fs::absolute(
"."));
281 mSnapshotCachePath = fs::weakly_canonical(fs::absolute(cachedir));
283 snapshotReport = fmt::format(
"(cache snapshots to dir={}", mSnapshotCachePath);
286 mPreferSnapshotCache =
true;
287 if (mSnapshotCachePath.empty()) {
288 LOGP(fatal,
"IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE is defined but the ALICEO2_CCDB_LOCALCACHE is not");
290 snapshotReport +=
", prefer if available";
292 if (!snapshotReport.empty()) {
293 snapshotReport +=
')';
296 mNeedAlienToken = (host.find(
"https://") != std::string::npos) || (host.find(
"alice-ccdb.cern.ch") != std::string::npos) || (host.find(
"ccdb-test.cern.ch") != std::string::npos);
298 if (getenv(
"ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD")) {
299 auto timeout = atoi(getenv(
"ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD"));
301 mCurlTimeoutDownload =
timeout;
308 mCurlTimeoutDownload = 15;
311 mCurlTimeoutDownload = 15;
313 mCurlTimeoutDownload = 5;
317 if (getenv(
"ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD")) {
318 auto timeout = atoi(getenv(
"ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD"));
327 mCurlTimeoutUpload = 3;
330 mCurlTimeoutUpload = 20;
332 mCurlTimeoutUpload = 20;
339 LOGP(
debug,
"Curl timeouts are set to: download={:2}, upload={:2} seconds", mCurlTimeoutDownload, mCurlTimeoutUpload);
341 LOGP(info,
"Init CcdApi with UserAgentID: {}, Host: {}{}, Curl timeouts: upload:{} download:{}", mUniqueAgentID, host,
342 mInSnapshotMode ?
"(snapshot readonly mode)" : snapshotReport.c_str(), mCurlTimeoutUpload, mCurlTimeoutDownload);
351void CcdbApi::updateMetaInformationInLocalFile(std::string
const&
filename, std::map<std::string, std::string>
const* headers,
CCDBQuery const* querysummary)
353 std::lock_guard<std::mutex> guard(
gIOMutex);
354 auto oldlevel = gErrorIgnoreLevel;
355 gErrorIgnoreLevel = 6001;
356 TFile snapshotfile(
filename.c_str(),
"UPDATE");
358 if (!snapshotfile.IsZombie()) {
360 snapshotfile.WriteObjectAny(querysummary, TClass::GetClass(
typeid(*querysummary)),
CCDBQUERY_ENTRY);
363 snapshotfile.WriteObjectAny(headers, TClass::GetClass(
typeid(*headers)),
CCDBMETA_ENTRY);
365 snapshotfile.Write();
366 snapshotfile.Close();
368 gErrorIgnoreLevel = oldlevel;
378 std::string tmpObjectName = objectName;
379 tmpObjectName.erase(std::remove_if(tmpObjectName.begin(), tmpObjectName.end(),
380 [](
auto const&
c) ->
bool { return (!std::isalnum(c) && c !=
'_' && c !=
'/' && c !=
'.'); }),
381 tmpObjectName.end());
382 return tmpObjectName;
389 std::lock_guard<std::mutex> guard(
gIOMutex);
393 info->setFileName(tmpFileName);
403 std::string
className = rootObject->GetName();
406 info->setFileName(tmpFileName);
407 info->setObjectType(
"TObject");
409 std::lock_guard<std::mutex> guard(
gIOMutex);
414 std::map<std::string, std::string>
const& metadata,
415 long startValidityTimestamp,
long endValidityTimestamp,
416 std::vector<char>::size_type
maxSize)
const
420 LOGP(
error,
"nullptr is provided for object {}/{}/{}",
path, startValidityTimestamp, endValidityTimestamp);
426 path, metadata, startValidityTimestamp, endValidityTimestamp,
maxSize);
430 const std::string&
path,
const std::map<std::string, std::string>& metadata,
431 long startValidityTimestamp,
long endValidityTimestamp, std::vector<char>::size_type
maxSize)
const
434 LOGP(alarm,
"Object will not be uploaded to {} since its size {} exceeds max allowed {}",
path,
size,
maxSize);
440 long sanitizedStartValidityTimestamp = startValidityTimestamp;
441 if (startValidityTimestamp == -1) {
442 LOGP(info,
"Start of Validity not set, current timestamp used.");
445 long sanitizedEndValidityTimestamp = endValidityTimestamp;
446 if (endValidityTimestamp == -1) {
447 LOGP(info,
"End of Validity not set, start of validity plus 1 day used.");
450 if (mInSnapshotMode) {
452 LOGP(alarm,
"Snapshot mode does not support headers-only upload");
455 auto pthLoc = getSnapshotDir(mSnapshotTopPath,
path);
457 auto flLoc = getSnapshotFile(mSnapshotTopPath,
path,
filename);
459 auto pent = flLoc.find_last_of(
'.');
460 if (pent == std::string::npos) {
463 flLoc.insert(pent, fmt::format(
"_{}_{}", startValidityTimestamp, endValidityTimestamp));
464 ofstream outf(flLoc.c_str(), ios::out | ios::binary);
468 throw std::runtime_error(fmt::format(
"Failed to write local CCDB file {}", flLoc));
470 std::map<std::string, std::string> metaheader(metadata);
472 metaheader[
"Valid-From"] =
std::to_string(startValidityTimestamp);
474 updateMetaInformationInLocalFile(flLoc.c_str(), &metaheader);
475 std::string metaStr{};
476 for (
const auto& mentry : metadata) {
477 metaStr += fmt::format(
"{}={};", mentry.first, mentry.second);
479 metaStr +=
"$USER_META;";
480 LOGP(info,
"Created local snapshot {}", flLoc);
481 LOGP(info, R
"(Upload with: o2-ccdb-upload --host "$ccdbhost" -p {} -f {} -k {} --starttimestamp {} --endtimestamp {} -m "{}")",
489 curl = curl_easy_init();
492 checkMetadataKeys(metadata);
494 if (curl !=
nullptr) {
495 auto mime = curl_mime_init(curl);
496 auto field = curl_mime_addpart(mime);
497 curl_mime_name(field,
"send");
499 curl_mime_filedata(field,
filename.c_str());
504 curl_mime_data(field,
"", 0);
509 curl_easy_setopt(curl, CURLOPT_MIMEPOST, mime);
510 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
511 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
512 curl_easy_setopt(curl, CURLOPT_TIMEOUT, mCurlTimeoutUpload);
514 CURLcode
res = CURL_LAST;
516 for (
size_t hostIndex = 0; hostIndex < hostsPool.size() &&
res > 0; hostIndex++) {
517 std::string fullUrl = getFullUrlForStorage(curl,
path, objectType, metadata, sanitizedStartValidityTimestamp, sanitizedEndValidityTimestamp, hostIndex);
518 LOG(debug3) <<
"Full URL Encoded: " << fullUrl;
520 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
523 struct curl_slist* headerlist = curl_slist_append(
nullptr,
"Expect:");
524 headerlist = appendGateToken(headerlist, fullUrl);
525 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist);
528 res =
static_cast<CURLcode
>(CURL_perform(curl));
530 if (
res != CURLE_OK) {
531 if (
res == CURLE_OPERATION_TIMEDOUT) {
532 LOGP(alarm,
"curl_easy_perform() timed out. Consider increasing the timeout using the env var `ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD` (seconds), current one is {}", mCurlTimeoutUpload);
534 LOGP(alarm,
"curl_easy_perform() failed: {}", curl_easy_strerror(
res));
538 curl_slist_free_all(headerlist);
542 curl_easy_cleanup(curl);
545 curl_mime_free(mime);
547 LOGP(alarm,
"curl initialization failure");
554 long startValidityTimestamp,
long endValidityTimestamp, std::vector<char>::size_type
maxSize)
const
558 LOGP(
error,
"nullptr is provided for object {}/{}/{}",
path, startValidityTimestamp, endValidityTimestamp);
566std::string CcdbApi::getFullUrlForStorage(
CurlHandle* curl,
const std::string&
path,
const std::string& objtype,
567 const std::map<std::string, std::string>& metadata,
568 long startValidityTimestamp,
long endValidityTimestamp,
int hostIndex)
const
571 std::string startValidityString = getTimestampString(startValidityTimestamp < 0 ?
getCurrentTimestamp() : startValidityTimestamp);
572 std::string endValidityString = getTimestampString(endValidityTimestamp < 0 ?
getFutureTimestamp(60 * 60 * 24 * 1) : endValidityTimestamp);
574 std::string
url = getHostUrl(hostIndex);
576 std::string fullUrl =
url +
"/" +
path +
"/" + startValidityString +
"/" + endValidityString +
"/";
579 char* objtypeEncoded = curl_easy_escape(curl, objtype.c_str(), objtype.size());
580 fullUrl +=
"ObjectType=" + std::string(objtypeEncoded) +
"/";
581 curl_free(objtypeEncoded);
583 for (
auto& kv : metadata) {
584 std::string mfirst = kv.first;
585 std::string msecond = kv.second;
587 char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
588 char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
589 fullUrl += std::string(mfirstEncoded) +
"=" + std::string(msecondEncoded) +
"/";
590 curl_free(mfirstEncoded);
591 curl_free(msecondEncoded);
597std::string CcdbApi::getFullUrlForRetrieval(
CurlHandle* curl,
const std::string&
path,
const std::map<std::string, std::string>& metadata,
long timestamp,
int hostIndex)
const
599 if (mInSnapshotMode) {
600 return getSnapshotFile(mSnapshotTopPath,
path);
606 std::string hostUrl = getHostUrl(hostIndex);
608 std::string fullUrl = hostUrl +
"/" +
path +
"/" + validityString +
"/";
610 for (
auto& kv : metadata) {
611 std::string mfirst = kv.first;
612 std::string msecond = kv.second;
614 char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
615 char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
616 fullUrl += std::string(mfirstEncoded) +
"=" + std::string(msecondEncoded) +
"/";
617 curl_free(mfirstEncoded);
618 curl_free(msecondEncoded);
640static size_t WriteMemoryCallback(
void* contents,
size_t size,
size_t nmemb,
void* userp)
642 size_t realsize =
size * nmemb;
645 mem->memory = (
char*)
realloc(mem->memory, mem->size + realsize + 1);
646 if (mem->memory ==
nullptr) {
647 printf(
"not enough memory (realloc returned NULL)\n");
651 memcpy(&(mem->memory[mem->size]), contents, realsize);
652 mem->size += realsize;
653 mem->memory[mem->size] = 0;
669static size_t WriteToFileCallback(
void*
ptr,
size_t size,
size_t nmemb, FILE*
stream)
682static CURLcode ssl_ctx_callback(
CurlHandle*,
void*,
void* parm)
684 std::string
msg((
const char*)parm);
687 if (
msg.length() > 0 &&
end == -1) {
689 }
else if (
end > 0) {
701 CredentialsKind cmk = mJAlienCredentials->getPreferedCredentials();
704 if (cmk == cNOT_FOUND) {
708 TJAlienCredentialsObject cmo = mJAlienCredentials->get(cmk);
710 char* CAPath = getenv(
"X509_CERT_DIR");
712 curl_easy_setopt(curl_handle, CURLOPT_CAPATH, CAPath);
714 curl_easy_setopt(curl_handle, CURLOPT_CAINFO,
nullptr);
715 curl_easy_setopt(curl_handle, CURLOPT_SSLCERT, cmo.certpath.c_str());
716 curl_easy_setopt(curl_handle, CURLOPT_SSLKEY, cmo.keypath.c_str());
719 curl_easy_setopt(curl_handle, CURLOPT_SSL_CTX_FUNCTION, ssl_ctx_callback);
720 curl_easy_setopt(curl_handle, CURLOPT_SSL_CTX_DATA, mJAlienCredentials->getMessages().c_str());
727void CcdbApi::initCurlOptionsForRetrieve(
CurlHandle* curlHandle,
void* chunk,
CurlWriteCallback writeCallback,
bool followRedirect)
const
729 curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
730 curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, chunk);
731 curl_easy_setopt(curlHandle, CURLOPT_FOLLOWLOCATION, followRedirect ? 1L : 0L);
736template <
typename MapType = std::map<std::
string, std::
string>>
737size_t header_map_callback(
char*
buffer,
size_t size,
size_t nitems,
void* userdata)
739 auto* headers =
static_cast<MapType*
>(userdata);
741 std::string::size_type
index =
header.find(
':', 0);
742 if (
index != std::string::npos) {
743 const auto key = boost::algorithm::trim_copy(
header.substr(0,
index));
745 LOGP(
debug,
"Adding #{} {} -> {}", headers->size(),
key,
value);
747 if (
key ==
"Content-Length") {
748 auto cl = headers->find(
"Content-Length");
749 if (cl != headers->end()) {
750 if (std::stol(cl->second) < stol(
value)) {
760 auto cl = headers->find(
"ETag");
761 if (cl != headers->end()) {
767 if (
key ==
"Content-Type") {
768 auto cl = headers->find(
"Content-Type");
769 if (cl != headers->end()) {
775 headers->insert(std::make_pair(
key,
value));
778 return size * nitems;
782void CcdbApi::initCurlHTTPHeaderOptionsForRetrieve(
CurlHandle* curlHandle, curl_slist*& option_list,
long timestamp, std::map<std::string, std::string>* headers, std::string
const&
etag,
787 option_list = curl_slist_append(option_list, (
"If-None-Match: " +
etag).c_str());
791 option_list = curl_slist_append(option_list, (
"If-Not-After: " +
createdNotAfter).c_str());
795 option_list = curl_slist_append(option_list, (
"If-Not-Before: " +
createdNotBefore).c_str());
798 if (headers !=
nullptr) {
799 option_list = curl_slist_append(option_list, (
"If-None-Match: " +
to_string(timestamp)).c_str());
800 curl_easy_setopt(curlHandle, CURLOPT_HEADERFUNCTION, header_map_callback<>);
801 curl_easy_setopt(curlHandle, CURLOPT_HEADERDATA, headers);
804 option_list = appendGateToken(option_list,
url);
808 curl_easy_setopt(curlHandle, CURLOPT_HTTPHEADER, option_list);
810 curl_easy_setopt(curlHandle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
813bool CcdbApi::receiveToFile(FILE* fileHandle, std::string
const&
path, std::map<std::string, std::string>
const& metadata,
814 long timestamp, std::map<std::string, std::string>* headers, std::string
const&
etag,
820bool CcdbApi::receiveToMemory(
void* chunk, std::string
const&
path, std::map<std::string, std::string>
const& metadata,
821 long timestamp, std::map<std::string, std::string>* headers, std::string
const&
etag,
827bool CcdbApi::receiveObject(
void* dataHolder, std::string
const&
path, std::map<std::string, std::string>
const& metadata,
828 long timestamp, std::map<std::string, std::string>* headers, std::string
const&
etag,
833 curlHandle = curl_easy_init();
834 curl_easy_setopt(curlHandle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
836 if (curlHandle !=
nullptr) {
839 initCurlOptionsForRetrieve(curlHandle, dataHolder, writeCallback, followRedirect);
840 long responseCode = 0;
841 CURLcode curlResultCode = CURL_LAST;
843 for (
size_t hostIndex = 0; hostIndex < hostsPool.size() && (responseCode >= 400 || curlResultCode > 0); hostIndex++) {
844 std::string fullUrl = getFullUrlForRetrieval(curlHandle,
path, metadata, timestamp, hostIndex);
845 curl_easy_setopt(curlHandle, CURLOPT_URL, fullUrl.c_str());
848 curl_slist* option_list =
nullptr;
851 curlResultCode =
static_cast<CURLcode
>(CURL_perform(curlHandle));
853 if (curlResultCode != CURLE_OK) {
854 LOGP(alarm,
"curl_easy_perform() failed: {}", curl_easy_strerror(curlResultCode));
856 curlResultCode = curl_easy_getinfo(curlHandle, CURLINFO_RESPONSE_CODE, &responseCode);
857 if ((curlResultCode == CURLE_OK) && (responseCode < 300)) {
858 curl_slist_free_all(option_list);
859 curl_easy_cleanup(curlHandle);
862 if (curlResultCode != CURLE_OK) {
863 LOGP(alarm,
"invalid URL {}", fullUrl);
865 LOGP(alarm,
"not found under link {}", fullUrl);
869 curl_slist_free_all(option_list);
872 curl_easy_cleanup(curlHandle);
878 long timestamp)
const
886 bool res = receiveToMemory((
void*)&chunk,
path, metadata, timestamp);
889 std::lock_guard<std::mutex> guard(
gIOMutex);
891 mess.SetBuffer(chunk.
memory, chunk.
size, kFALSE);
896 LOGP(info,
"couldn't retrieve the object {}",
path);
908 std::string
str = inp;
909 str.erase(std::remove_if(
str.begin(),
str.end(), ::isspace),
str.end());
910 str = std::regex_replace(
str, std::regex(
"::"),
"-");
916 long timestamp, std::map<std::string, std::string>* headers, std::string
const&
etag,
923 long timestamp,
bool preservePath, std::string
const& localFileName, std::string
const&
createdNotAfter, std::string
const&
createdNotBefore, std::map<std::string, std::string>* outHeaders)
const
927 std::string fulltargetdir = targetdir + (preservePath ? (
'/' +
path) :
"");
931 }
catch (std::exception
e) {
932 LOGP(
error,
"Could not create local snapshot cache directory {}, reason: {}", fulltargetdir,
e.what());
937 std::map<std::string, std::string> headers;
940 if ((headers.count(
"Error") != 0) || (buff.empty())) {
941 LOGP(
error,
"Unable to find object {}/{}, Aborting",
path, timestamp);
945 auto getFileName = [&headers]() {
946 auto&
s = headers[
"Content-Disposition"];
948 std::regex re(
"(.*;)filename=\"(.*)\"");
950 if (std::regex_match(
s.c_str(),
m, re)) {
954 std::string backupname(
"ccdb-blob.bin");
955 LOG(
error) <<
"Cannot determine original filename from Content-Disposition ... falling back to " << backupname;
958 auto filename = localFileName.size() > 0 ? localFileName : getFileName();
959 std::string targetpath = fulltargetdir +
"/" +
filename;
961 std::ofstream objFile(targetpath, std::ios::out | std::ofstream::binary);
962 std::copy(buff.begin(), buff.end(), std::ostreambuf_iterator<char>(objFile));
963 if (!objFile.good()) {
964 LOGP(
error,
"Unable to open local file {}, Aborting", targetpath);
970 updateMetaInformationInLocalFile(targetpath.c_str(), &headers, &querysummary);
972 *outHeaders = std::move(headers);
977void CcdbApi::snapshot(std::string
const& ccdbrootpath, std::string
const& localDir,
long timestamp)
const
981 std::map<std::string, std::string> metadata;
982 for (
auto&
folder : allfolders) {
992#if defined(_LIBCPP_VERSION)
993 o2::framework::DataRefUtils::registerLibstdcxxStdArray(
file, cl);
995 auto object =
file.GetObjectChecked(what, cl);
999 std::string objectName(cl->GetName());
1001 object =
file.GetObjectChecked(objectName.c_str(), cl);
1002 LOG(warn) <<
"Did not find object under expected name " << what;
1006 LOG(warn) <<
"Found object under deprecated name " << cl->GetName();
1012 if (cl->IsTObject()) {
1015 auto tree =
dynamic_cast<TTree*
>((
TObject*)
object);
1017 tree->LoadBaskets(0x1L << 32);
1018 tree->SetDirectory(
nullptr);
1021 auto h =
dynamic_cast<TH1*
>((
TObject*)
object);
1023 h->SetDirectory(
nullptr);
1031void* CcdbApi::extractFromLocalFile(std::string
const&
filename, std::type_info
const& tinfo, std::map<std::string, std::string>* headers)
const
1033 if (!std::filesystem::exists(
filename)) {
1037 std::lock_guard<std::mutex> guard(
gIOMutex);
1038 auto tcl = tinfo2TClass(tinfo);
1043 *headers = *storedmeta;
1046 if ((
isSnapshotMode() || mPreferSnapshotCache) && headers->find(
"ETag") == headers->end()) {
1049 if (headers->find(
"fileSize") == headers->end()) {
1050 (*headers)[
"fileSize"] = fmt::format(
"{}",
f.GetEND());
1056bool CcdbApi::initTGrid()
const
1058 if (mNeedAlienToken && !gGrid) {
1059 static bool allowNoToken = getenv(
"ALICEO2_CCDB_NOTOKENCHECK") && atoi(getenv(
"ALICEO2_CCDB_NOTOKENCHECK"));
1061 LOG(fatal) <<
"Alien Token Check failed - Please get an alien token before running with https CCDB endpoint, or alice-ccdb.cern.ch!";
1063 TGrid::Connect(
"alien");
1064 static bool errorShown =
false;
1065 if (!gGrid && errorShown ==
false) {
1067 LOG(
error) <<
"TGrid::Connect returned nullptr. May be due to missing alien token";
1069 LOG(fatal) <<
"TGrid::Connect returned nullptr. May be due to missing alien token";
1074 return gGrid !=
nullptr;
1077void* CcdbApi::downloadFilesystemContent(std::string
const&
url, std::type_info
const& tinfo, std::map<std::string, std::string>* headers)
const
1079 if ((
url.find(
"alien:/", 0) != std::string::npos) && !initTGrid()) {
1082 std::lock_guard<std::mutex> guard(
gIOMutex);
1083 auto memfile = TMemFile::Open(
url.c_str(),
"OPEN");
1085 auto cl = tinfo2TClass(tinfo);
1087 if (headers && headers->find(
"fileSize") == headers->end()) {
1088 (*headers)[
"fileSize"] = fmt::format(
"{}", memfile->GetEND());
1096void* CcdbApi::interpretAsTMemFileAndExtract(
char* contentptr,
size_t contentsize, std::type_info
const& tinfo)
1099 Int_t previousErrorLevel = gErrorIgnoreLevel;
1100 gErrorIgnoreLevel = kFatal;
1101 std::lock_guard<std::mutex> guard(
gIOMutex);
1102 TMemFile memFile(
"name", contentptr, contentsize,
"READ");
1103 gErrorIgnoreLevel = previousErrorLevel;
1104 if (!memFile.IsZombie()) {
1105 auto tcl = tinfo2TClass(tinfo);
1116void* CcdbApi::navigateURLsAndRetrieveContent(
CurlHandle* curl_handle, std::string
const&
url, std::type_info
const& tinfo, std::map<std::string, std::string>* headers)
const
1121 static thread_local std::multimap<std::string, std::string> headerData;
1124 if ((
url.find(
"alien:/", 0) != std::string::npos) || (
url.find(
"file:/", 0) != std::string::npos)) {
1125 return downloadFilesystemContent(
url, tinfo, headers);
1132 curl_easy_setopt(curl_handle, CURLOPT_URL,
url.c_str());
1134 MemoryStruct chunk{(
char*)
malloc(1), 0};
1135 initCurlOptionsForRetrieve(curl_handle, (
void*)&chunk, WriteMemoryCallback,
false);
1137 curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback<
decltype(headerData)>);
1139 curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (
void*)&headerData);
1143 auto res =
static_cast<CURLcode
>(CURL_perform(curl_handle));
1144 long response_code = -1;
1145 void* content =
nullptr;
1147 if (
res == CURLE_OK && curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code) == CURLE_OK) {
1149 for (
auto& p : headerData) {
1150 (*headers)[
p.first] =
p.second;
1153 if (200 <= response_code && response_code < 300) {
1155 content = interpretAsTMemFileAndExtract(chunk.memory, chunk.size, tinfo);
1156 if (headers && headers->find(
"fileSize") == headers->end()) {
1157 (*headers)[
"fileSize"] = fmt::format(
"{}", chunk.size);
1159 }
else if (response_code == 304) {
1164 LOGP(
debug,
"Object exists but I am not serving it since it's already in your possession");
1167 else if (300 <= response_code && response_code < 400) {
1173 auto complement_Location = [
this](std::string
const& loc) {
1174 if (loc[0] ==
'/') {
1181 std::vector<std::string> locs;
1182 auto iter = headerData.find(
"Location");
1183 if (iter != headerData.end()) {
1184 locs.push_back(complement_Location(iter->second));
1187 auto iter2 = headerData.find(
"Content-Location");
1188 if (iter2 != headerData.end()) {
1189 auto range = headerData.equal_range(
"Content-Location");
1190 for (
auto it =
range.first; it !=
range.second; ++it) {
1191 if (std::find(locs.begin(), locs.end(), it->second) == locs.end()) {
1192 locs.push_back(complement_Location(it->second));
1196 for (
auto& l : locs) {
1198 LOG(
debug) <<
"Trying content location " << l;
1199 content = navigateURLsAndRetrieveContent(curl_handle, l, tinfo, headers);
1205 }
else if (response_code == 404) {
1206 LOG(
error) <<
"Requested resource does not exist: " <<
url;
1209 LOG(
error) <<
"Error in fetching object " <<
url <<
", curl response code:" << response_code;
1213 if (chunk.memory !=
nullptr) {
1217 LOGP(alarm,
"Curl request to {} failed with result {}, response code: {}",
url,
int(
res), response_code);
1222 (*headers)[
"Error"] =
"An error occurred during retrieval";
1228 std::map<std::string, std::string>
const& metadata,
long timestamp,
1229 std::map<std::string, std::string>* headers, std::string
const&
etag,
1232 if (!mSnapshotCachePath.empty()) {
1234 auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath,
path);
1235 std::string logfile = mSnapshotCachePath +
"/log";
1236 std::fstream out(logfile, ios_base::out | ios_base::app);
1237 if (out.is_open()) {
1238 out <<
"CCDB-access[" << getpid() <<
"] of " << mUniqueAgentID <<
" to " <<
path <<
" timestamp " << timestamp <<
"\n";
1240 auto snapshotfile = getSnapshotFile(mSnapshotCachePath,
path);
1241 bool snapshoting =
false;
1242 if (!std::filesystem::exists(snapshotfile)) {
1244 out <<
"CCDB-access[" << getpid() <<
"] ... " << mUniqueAgentID <<
" downloading to snapshot " << snapshotfile <<
"\n";
1247 out <<
"CCDB-access[" << getpid() <<
"] ... " << mUniqueAgentID <<
" failed to create directory for " << snapshotfile <<
"\n";
1250 out <<
"CCDB-access[" << getpid() <<
"] ... " << mUniqueAgentID <<
"serving from local snapshot " << snapshotfile <<
"\n";
1253 auto res = extractFromLocalFile(snapshotfile, tinfo, headers);
1255 logReading(
path, timestamp, headers,
"retrieve from snapshot");
1263 curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1264 std::string fullUrl = getFullUrlForRetrieval(curl_handle,
path, metadata, timestamp);
1266 if (mInSnapshotMode) {
1267 auto res = extractFromLocalFile(fullUrl, tinfo, headers);
1269 logReading(
path, timestamp, headers,
"retrieve from snapshot");
1274 curl_slist* option_list =
nullptr;
1276 auto content = navigateURLsAndRetrieveContent(curl_handle, fullUrl, tinfo, headers);
1278 for (
size_t hostIndex = 1; hostIndex < hostsPool.size() && !(content); hostIndex++) {
1279 fullUrl = getFullUrlForRetrieval(curl_handle,
path, metadata, timestamp, hostIndex);
1281 curl_slist_free_all(option_list);
1282 option_list =
nullptr;
1284 content = navigateURLsAndRetrieveContent(curl_handle, fullUrl, tinfo, headers);
1287 logReading(
path, timestamp, headers,
"retrieve");
1289 curl_slist_free_all(option_list);
1290 curl_easy_cleanup(curl_handle);
1296 size_t newLength =
size * nmemb;
1297 size_t oldLength =
s->size();
1299 s->resize(oldLength + newLength);
1300 }
catch (std::bad_alloc&
e) {
1301 LOG(
error) <<
"memory error when getting data from CCDB";
1305 std::copy((
char*)contents, (
char*)contents + newLength,
s->begin() + oldLength);
1306 return size * nmemb;
1312 CURLcode
res = CURL_LAST;
1315 curl = curl_easy_init();
1316 if (curl !=
nullptr) {
1318 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &
result);
1319 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1323 std::string fullUrl;
1325 for (
size_t hostIndex = 0; hostIndex < hostsPool.size() &&
res != CURLE_OK; hostIndex++) {
1326 fullUrl = getHostUrl(hostIndex);
1327 fullUrl += latestOnly ?
"/latest/" :
"/browse/";
1329 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
1332 struct curl_slist* headers =
nullptr;
1333 headers = curl_slist_append(headers, (std::string(
"Accept: ") + returnFormat).c_str());
1334 headers = curl_slist_append(headers, (std::string(
"Content-Type: ") + returnFormat).c_str());
1341 headers = appendGateToken(headers, fullUrl);
1342 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
1344 res =
static_cast<CURLcode
>(CURL_perform(curl));
1345 if (
res != CURLE_OK) {
1346 LOGP(alarm,
"CURL_perform() failed: {}", curl_easy_strerror(
res));
1348 curl_slist_free_all(headers);
1350 curl_easy_cleanup(curl);
1356std::string CcdbApi::getTimestampString(
long timestamp)
const
1369 curl = curl_easy_init();
1370 if (curl !=
nullptr) {
1371 curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST,
"DELETE");
1372 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1373 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1376 for (
size_t hostIndex = 0; hostIndex < hostsPool.size(); hostIndex++) {
1379 stringstream fullUrl;
1380 fullUrl << getHostUrl(hostIndex) <<
"/" <<
path <<
"/" << timestampLocal;
1381 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str());
1385 struct curl_slist*
list = appendGateToken(
nullptr, fullUrl.str());
1386 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1389 res =
static_cast<CURLcode
>(CURL_perform(curl));
1390 if (
res != CURLE_OK) {
1391 LOGP(alarm,
"CURL_perform() failed: {}", curl_easy_strerror(
res));
1393 curl_slist_free_all(list);
1397 curl_easy_cleanup(curl);
1405 for (
size_t i = 0;
i < hostsPool.size();
i++) {
1409 stringstream fullUrl;
1410 std::string
url = getHostUrl(
i);
1411 fullUrl <<
url <<
"/truncate/" <<
path;
1413 curl = curl_easy_init();
1414 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1415 if (curl !=
nullptr) {
1416 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str());
1422 struct curl_slist*
list = appendGateToken(
nullptr, fullUrl.str());
1423 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1424 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1428 res =
static_cast<CURLcode
>(CURL_perform(curl));
1429 if (
res != CURLE_OK) {
1430 LOGP(alarm,
"CURL_perform() failed: {}", curl_easy_strerror(
res));
1432 curl_easy_cleanup(curl);
1433 curl_slist_free_all(list);
1440 return size * nmemb;
1446 CURLcode
res = CURL_LAST;
1449 curl = curl_easy_init();
1450 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1463 for (
size_t hostIndex = 0; hostIndex < hostsPool.size() &&
res != CURLE_OK; hostIndex++) {
1464 curl_easy_setopt(curl, CURLOPT_URL, mUrl.data());
1465 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
write_data);
1467 res =
static_cast<CURLcode
>(CURL_perform(curl));
1472 curl_easy_cleanup(curl);
1481 std::stringstream ss(reply.c_str());
1483 std::vector<std::string> folders;
1485 size_t numberoflines = std::count(reply.begin(), reply.end(),
'\n');
1486 bool inSubFolderSection =
false;
1488 for (
size_t linenumber = 0; linenumber < numberoflines; ++linenumber) {
1489 std::getline(ss, line);
1490 if (inSubFolderSection && line.size() > 0) {
1495 if (line.compare(
"Subfolders:") == 0) {
1496 inSubFolderSection =
true;
1504size_t header_callback(
char*
buffer,
size_t size,
size_t nitems,
void* userdata)
1506 auto* headers =
static_cast<std::vector<std::string>*
>(userdata);
1508 headers->emplace_back(std::string(
header.data()));
1509 return size * nitems;
1517 auto p = std::filesystem::path(
filename).parent_path();
1518 if (!std::filesystem::exists(p)) {
1519 std::filesystem::create_directories(p);
1522 rapidjson::StringBuffer
buffer;
1523 rapidjson::Writer<rapidjson::StringBuffer> writer(
buffer);
1524 writer.StartObject();
1525 for (
const auto& pair : meta) {
1526 writer.Key(pair.first.c_str());
1527 writer.String(pair.second.c_str());
1533 if (
file.is_open()) {
1546 if (!
file.is_open()) {
1547 std::cerr <<
"Failed to open file for reading." << std::endl;
1551 std::string jsonStr((std::istreambuf_iterator<char>(
file)), std::istreambuf_iterator<char>());
1554 rapidjson::Document document;
1555 document.Parse(jsonStr.c_str());
1557 if (document.HasParseError()) {
1558 std::cerr <<
"Error parsing JSON" << std::endl;
1563 for (
auto itr = document.MemberBegin(); itr != document.MemberEnd(); ++itr) {
1564 meta[itr->name.GetString()] = itr->value.GetString();
1569std::map<std::string, std::string>
CcdbApi::retrieveHeaders(std::string
const&
path, std::map<std::string, std::string>
const& metadata,
long timestamp)
const
1572 auto do_remote_header_call = [
this, &
path, &metadata, timestamp]() -> std::map<std::string, std::string> {
1574 CURLcode
res = CURL_LAST;
1575 std::string fullUrl = getFullUrlForRetrieval(curl,
path, metadata, timestamp);
1576 std::map<std::string, std::string> headers;
1578 if (curl !=
nullptr) {
1579 struct curl_slist*
list =
nullptr;
1580 list = curl_slist_append(list, (
"If-None-Match: " +
std::to_string(timestamp)).c_str());
1581 list = appendGateToken(list, fullUrl);
1583 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1586 curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
1587 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1588 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_map_callback<>);
1589 curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);
1590 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1595 long httpCode = 404;
1596 CURLcode getCodeRes = CURL_LAST;
1597 for (
size_t hostIndex = 0; hostIndex < hostsPool.size() && (httpCode >= 400 ||
res > 0 || getCodeRes > 0); hostIndex++) {
1598 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
1599 res =
static_cast<CURLcode
>(CURL_perform(curl));
1600 if (
res != CURLE_OK &&
res != CURLE_UNSUPPORTED_PROTOCOL) {
1604 LOG(
error) <<
"CURL_perform() failed: " << curl_easy_strerror(
res);
1606 getCodeRes = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpCode);
1608 if (httpCode == 404) {
1611 curl_easy_cleanup(curl);
1616 if (!mSnapshotCachePath.empty()) {
1618 auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath + std::string(
"_headers"),
path);
1620 std::string logfile = mSnapshotCachePath +
"/log";
1621 std::fstream out(logfile, ios_base::out | ios_base::app);
1622 if (out.is_open()) {
1623 out <<
"CCDB-header-access[" << getpid() <<
"] of " << mUniqueAgentID <<
" to " <<
path <<
" timestamp " << timestamp <<
"\n";
1625 auto snapshotfile = getSnapshotFile(mSnapshotCachePath,
path +
"/" +
std::to_string(timestamp),
"header.json");
1626 if (!std::filesystem::exists(snapshotfile)) {
1627 out <<
"CCDB-header-access[" << getpid() <<
"] ... " << mUniqueAgentID <<
" storing to snapshot " << snapshotfile <<
"\n";
1630 auto meta = do_remote_header_call();
1634 LOG(warn) <<
"Failed to cache the header information to disc";
1638 out <<
"CCDB-header-access[" << getpid() <<
"] ... " << mUniqueAgentID <<
"serving from local snapshot " << snapshotfile <<
"\n";
1639 std::map<std::string, std::string> meta;
1641 LOG(warn) <<
"Failed to read cached information from disc";
1642 return do_remote_header_call();
1647 return do_remote_header_call();
1652 auto curl = curl_easy_init();
1658 struct curl_slist*
list =
nullptr;
1659 list = curl_slist_append(list, (
"If-None-Match: " +
etag).c_str());
1660 list = appendGateToken(list,
url);
1662 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1664 curl_easy_setopt(curl, CURLOPT_URL,
url.c_str());
1666 curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
1667 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1668 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
1669 curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);
1670 if (!agentID.empty()) {
1671 curl_easy_setopt(curl, CURLOPT_USERAGENT, agentID.c_str());
1677 curl_easy_perform(curl);
1678 long http_code = 404;
1679 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
1680 if (http_code == 304) {
1688 static std::string etagHeader =
"ETag: ";
1689 static std::string locationHeader =
"Content-Location: ";
1692 for (
auto h : headers) {
1693 if (
h.find(etagHeader) == 0) {
1694 etag = trimHeaderValue(std::string_view(
h).substr(etagHeader.size()));
1695 }
else if (
h.find(locationHeader) == 0) {
1696 pfns.emplace_back(trimHeaderValue(std::string_view(
h).substr(locationHeader.size())));
1712 auto object =
file.GetObjectChecked(
CCDBMETA_ENTRY, TClass::GetClass(
typeid(std::map<std::string, std::string>)));
1714 return static_cast<std::map<std::string, std::string>*
>(
object);
1721void traverseAndFillFolders(
CcdbApi const& api, std::string
const&
top, std::vector<std::string>& folders)
1725 folders.emplace_back(
top);
1728 if (subfolders.size() > 0) {
1730 for (
auto& sub : subfolders) {
1731 traverseAndFillFolders(api, sub, folders);
1741 std::vector<std::string> folders;
1742 traverseAndFillFolders(*
this,
top, folders);
1746TClass* CcdbApi::tinfo2TClass(std::type_info
const& tinfo)
1748 TClass* cl = TClass::GetClass(tinfo);
1750 throw std::runtime_error(fmt::format(
"Could not retrieve ROOT dictionary for type {}, aborting", tinfo.name()));
1756int CcdbApi::updateMetadata(std::string
const&
path, std::map<std::string, std::string>
const& metadata,
long timestamp, std::string
const&
id,
long newEOV)
1760 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1761 if (curl !=
nullptr) {
1763 for (
size_t hostIndex = 0; hostIndex < hostsPool.size(); hostIndex++) {
1766 stringstream fullUrl;
1767 fullUrl << getHostUrl(hostIndex) <<
"/" <<
path <<
"/" << timestamp;
1769 fullUrl <<
"/" << newEOV;
1772 fullUrl <<
"/" <<
id;
1776 for (
auto& kv : metadata) {
1777 std::string mfirst = kv.first;
1778 std::string msecond = kv.second;
1780 char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
1781 char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
1782 fullUrl << std::string(mfirstEncoded) +
"=" + std::string(msecondEncoded) +
"&";
1783 curl_free(mfirstEncoded);
1784 curl_free(msecondEncoded);
1787 if (curl !=
nullptr) {
1788 LOG(
debug) <<
"passing to curl: " << fullUrl.str();
1789 curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str());
1790 curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST,
"PUT");
1791 curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1792 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
1795 struct curl_slist*
list = appendGateToken(
nullptr, fullUrl.str());
1796 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
1800 res =
static_cast<CURLcode
>(CURL_perform(curl));
1801 if (
res != CURLE_OK) {
1802 LOGP(alarm,
"CURL_perform() failed: {}, code: {}", curl_easy_strerror(
res),
int(
res));
1807 curl_slist_free_all(list);
1812 curl_easy_cleanup(curl);
1817void CcdbApi::initHostsPool(std::string hosts)
1820 auto splitted = hosts | std::views::transform([](
char c) {
return (
c ==
';') ?
',' :
c; }) | std::views::split(
',');
1821 for (
auto&& part : splitted) {
1822 hostsPool.emplace_back(part.begin(), part.end());
1826std::string CcdbApi::getHostUrl(
int hostIndex)
const
1828 return hostsPool.at(hostIndex);
1834 data->hoPair.object = &requestContext.
dest;
1836 std::function<
bool(std::string)> localContentCallback = [
this, &requestContext](std::string
url) {
1840 auto writeCallback = [](
void* contents,
size_t size,
size_t nmemb,
void* chunkptr) {
1842 auto& chunk = *ho.
object;
1843 size_t realsize =
size * nmemb, sz = 0;
1846 if (chunk.capacity() < chunk.size() + realsize) {
1848 const char hannot[] =
"header";
1849 size_t hsize = getFlatHeaderSize(ho.header);
1850 auto cl = ho.header.find(
"Content-Length");
1851 if (cl != ho.header.end()) {
1852 size_t sizeFromHeader = std::stol(cl->second);
1853 sz = hsize + std::max(chunk.size() * (sizeFromHeader ? 1 : 2) + realsize, sizeFromHeader);
1855 sz = hsize + std::max(chunk.size() * 2, chunk.size() + realsize);
1860 char* contC = (
char*)contents;
1861 chunk.insert(chunk.end(), contC, contC + realsize);
1862 }
catch (std::exception
e) {
1870 curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
1871 std::string fullUrl = getFullUrlForRetrieval(curl_handle, requestContext.
path, requestContext.
metadata, requestContext.
timestamp);
1874 data->hosts = hostsPool;
1877 data->localContentCallback = localContentCallback;
1878 data->userAgent = mUniqueAgentID;
1885 data->optionsLists.reserve(hostsPool.size());
1886 for (
size_t hostIndex = 0; hostIndex < hostsPool.size(); hostIndex++) {
1887 curl_slist* hostOptions =
nullptr;
1888 const std::string hostUrl = getFullUrlForRetrieval(curl_handle, requestContext.
path, requestContext.
metadata,
1890 initCurlHTTPHeaderOptionsForRetrieve(curl_handle, hostOptions, requestContext.
timestamp, &requestContext.
headers,
1893 data->optionsLists.push_back(hostOptions);
1898 if (!
data->optionsLists.empty()) {
1899 curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER,
data->optionsLists.front());
1902 curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str());
1903 initCurlOptionsForRetrieve(curl_handle, (
void*)(&
data->hoPair), writeCallback,
false);
1904 curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback<
decltype(
data->hoPair.header)>);
1905 curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (
void*)&(
data->hoPair.header));
1906 curl_easy_setopt(curl_handle, CURLOPT_PRIVATE, (
void*)
data);
1909 asynchPerform(curl_handle, requestCounter);
1914 std::hash<std::string> hasher;
1915 std::string semhashedstring =
"aliceccdb" +
std::to_string(hasher(basedir + ccdbpath)).substr(0, 16);
1916 return semhashedstring;
1924 return new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, semhashedstring.c_str(), 1);
1925 }
catch (std::exception
e) {
1926 LOG(warn) <<
"Exception occurred during CCDB (cache) semaphore setup; Continuing without";
1935 if (sem->try_wait()) {
1946 boost::interprocess::named_semaphore semaphore(boost::interprocess::open_only, semaname.c_str());
1947 std::cout <<
"Found CCDB semaphore: " << semaname <<
"\n";
1949 auto success = boost::interprocess::named_semaphore::remove(semaname.c_str());
1951 std::cout <<
"Removed CCDB semaphore: " << semaname <<
"\n";
1956 }
catch (std::exception
const&
e) {
1967 namespace fs = std::filesystem;
1968 std::string fileName{
"snapshot.root"};
1970 auto absolutesnapshotdir = fs::weakly_canonical(fs::absolute(snapshotdir));
1971 for (
const auto&
entry : fs::recursive_directory_iterator(absolutesnapshotdir)) {
1972 if (
entry.is_directory()) {
1973 const fs::path& currentDir = fs::canonical(fs::absolute(
entry.path()));
1974 fs::path filePath = currentDir / fileName;
1975 if (fs::exists(filePath) && fs::is_regular_file(filePath)) {
1976 std::cout <<
"Directory with file '" << fileName <<
"': " << currentDir << std::endl;
1980 auto numtokens = pathtokens.size();
1981 if (numtokens < 3) {
1986 std::string
path = pathtokens[numtokens - 3] +
"/" + pathtokens[numtokens - 2] +
"/" + pathtokens[numtokens - 1];
1992 }
catch (std::exception
const&
e) {
1993 LOG(info) <<
"Semaphore search had exception " <<
e.what();
1998 long timestamp, std::map<std::string, std::string>& headers,
2001 if (createSnapshot) {
2002 std::string logfile = mSnapshotCachePath +
"/log";
2003 std::fstream logStream = std::fstream(logfile, ios_base::out | ios_base::app);
2004 if (logStream.is_open()) {
2005 logStream <<
"CCDB-access[" << getpid() <<
"] of " << mUniqueAgentID <<
" to " <<
path <<
" timestamp " << timestamp <<
" for load to memory\n";
2008 if (mInSnapshotMode) {
2013 }
else if (mPreferSnapshotCache && std::filesystem::exists(snapshotpath)) {
2025 if (!mSnapshotCachePath.empty() && !(mInSnapshotMode && mSnapshotTopPath == mSnapshotCachePath)) {
2026 auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath, requestContext.
path);
2028 auto snapshotdir = getSnapshotDir(mSnapshotCachePath, requestContext.
path);
2029 std::string snapshotpath = getSnapshotFile(mSnapshotCachePath, requestContext.
path);
2031 std::fstream logStream;
2032 if (logStream.is_open()) {
2033 logStream <<
"CCDB-access[" << getpid() <<
"] ... " << mUniqueAgentID <<
" downloading to snapshot " << snapshotpath <<
" from memory\n";
2036 LOGP(
debug,
"creating snapshot {} -> {}", requestContext.
path, snapshotpath);
2039 std::ofstream objFile(snapshotpath, std::ios::out | std::ofstream::binary);
2040 std::copy(requestContext.
dest.begin(), requestContext.
dest.end(), std::ostreambuf_iterator<char>(objFile));
2043 updateMetaInformationInLocalFile(snapshotpath, &requestContext.
headers, &querysummary);
2049 std::map<std::string, std::string>
const& metadata,
long timestamp,
2050 std::map<std::string, std::string>* headers, std::string
const&
etag,
2054 destP.reserve(dest.size());
2057 dest.reserve(destP.size());
2058 for (
const auto c : destP) {
2064 std::map<std::string, std::string>
const& metadata,
long timestamp,
2065 std::map<std::string, std::string>* headers, std::string
const&
etag,
2077 std::vector<RequestContext> contexts = {requestContext};
2083 size_t hsize = getFlatHeaderSize(headers), cnt = dest.size();
2084 dest.resize(cnt + hsize);
2085 auto addString = [&dest, &cnt](
const std::string&
s) {
2092 for (
auto&
h : headers) {
2094 addString(
h.second);
2096 *
reinterpret_cast<int*
>(&dest[cnt]) = hsize;
2097 std::memcpy(&dest[cnt +
sizeof(
int)], FlatHeaderAnnot,
sizeof(FlatHeaderAnnot));
2102 LOGP(
debug,
"loadFileToMemory {} ETag=[{}]", requestContext.
path, requestContext.
etag);
2103 bool createSnapshot = requestContext.
considerSnapshot && !mSnapshotCachePath.empty();
2105 std::string snapshotpath;
2106 if (mInSnapshotMode || std::filesystem::exists(snapshotpath = getSnapshotFile(mSnapshotCachePath, requestContext.
path))) {
2107 auto semaphore_barrier = std::make_unique<CCDBSemaphore>(mSnapshotCachePath, requestContext.
path);
2118 std::vector<int> fromSnapshots(requestContexts.size());
2119 size_t requestCounter = 0;
2122 for (
int i = 0;
i < requestContexts.size();
i++) {
2124 auto& requestContext = requestContexts.at(
i);
2129 while (requestCounter > 0) {
2134 for (
int i = 0;
i < requestContexts.size();
i++) {
2135 auto& requestContext = requestContexts.at(
i);
2136 if (!requestContext.dest.empty()) {
2137 logReading(requestContext.path, requestContext.timestamp, &requestContext.headers,
2138 fmt::format(
"{}{}", requestContext.considerSnapshot ?
"load to memory" :
"retrieve", fromSnapshots.at(
i) ?
" from snapshot" :
""));
2139 if (requestContext.considerSnapshot && fromSnapshots.at(
i) != 2) {
2148 if (
url.find(
"alien:/", 0) != std::string::npos) {
2149 std::map<std::string, std::string> localHeaders;
2151 auto it = localHeaders.find(
"Error");
2152 if (it != localHeaders.end() && it->second ==
"An error occurred during retrieval") {
2158 if ((
url.find(
"file:/", 0) != std::string::npos)) {
2159 std::string
path =
url.substr(7);
2160 if (std::filesystem::exists(
path)) {
2161 std::map<std::string, std::string> localHeaders;
2163 auto it = localHeaders.find(
"Error");
2164 if (it != localHeaders.end() && it->second ==
"An error occurred during retrieval") {
2177 constexpr size_t MaxCopySize = 0x1L << 25;
2178 auto signalError = [&dest, localHeaders]() {
2182 (*localHeaders)[
"Error"] =
"An error occurred during retrieval";
2185 if (
path.find(
"alien:/") == 0 && !initTGrid()) {
2189 std::string fname(
path);
2190 if (fname.find(
"?filetype=raw") == std::string::npos) {
2191 fname +=
"?filetype=raw";
2193 std::unique_ptr<TFile> sfile{TFile::Open(fname.c_str())};
2194 if (!sfile || sfile->IsZombie()) {
2195 LOG(
error) <<
"Failed to open file " << fname;
2199 size_t totalread = 0, fsize = sfile->GetSize(), b00 = sfile->GetBytesRead();
2201 char* dptr = dest.data();
2205 size_t b0 = sfile->GetBytesRead(), b1 = b0 - b00;
2206 size_t readsize = fsize - b1 > MaxCopySize ? MaxCopySize : fsize - b1;
2207 if (readsize == 0) {
2210 sfile->Seek(totalread, TFile::kBeg);
2211 bool failed = sfile->ReadBuffer(dptr, (Int_t)readsize);
2212 nread = sfile->GetBytesRead() - b0;
2213 if (
failed || nread < 0) {
2214 LOG(
error) <<
"failed to copy file " << fname <<
" to memory buffer";
2220 }
while (nread == (
long)MaxCopySize);
2222 if (localHeaders && fetchLocalMetaData) {
2223 TMemFile memFile(
"name",
const_cast<char*
>(dest.data()), dest.size(),
"READ");
2224 auto storedmeta = (std::map<std::string, std::string>*)
extractFromTFile(memFile, TClass::GetClass(
"std::map<std::string, std::string>"),
CCDBMETA_ENTRY);
2226 *localHeaders = *storedmeta;
2229 if ((
isSnapshotMode() || mPreferSnapshotCache) && localHeaders->find(
"ETag") == localHeaders->end()) {
2230 (*localHeaders)[
"ETag"] =
path;
2232 if (localHeaders->find(
"fileSize") == localHeaders->end()) {
2233 (*localHeaders)[
"fileSize"] = fmt::format(
"{}", memFile.GetEND());
2239void CcdbApi::checkMetadataKeys(std::map<std::string, std::string>
const& metadata)
const
2245 const std::regex regexPatternSearch(R
"([ :;.,\\/'?!\(\)\{\}\[\]@<>=+*#$&`|~^%])");
2246 bool isInvalid =
false;
2248 for (
auto& el : metadata) {
2249 auto keyMd = el.first;
2251 std::smatch searchRes;
2252 while (std::regex_search(keyMd, searchRes, regexPatternSearch)) {
2254 LOG(
error) <<
"Invalid character found in metadata key '" << tmp <<
"\': '" << searchRes.str() <<
"\'";
2255 keyMd = searchRes.suffix();
2259 LOG(fatal) <<
"Some metadata keys have invalid characters, please fix!";
2264void CcdbApi::logReading(
const std::string&
path,
long ts,
const std::map<std::string, std::string>* headers,
const std::string& comment)
const
2266 std::string upath{
path};
2268 auto ent = headers->find(
"Valid-From");
2269 if (ent != headers->end()) {
2270 upath +=
"/" + ent->second;
2272 ent = headers->find(
"ETag");
2273 if (ent != headers->end()) {
2274 upath +=
"/" + ent->second;
2277 upath.erase(remove(upath.begin(), upath.end(),
'\"'), upath.end());
2278 LOGP(info,
"ccdb reads {}{}{} for {} ({}, agent_id: {}), ", mUrl, mUrl.back() ==
'/' ?
"" :
"/", upath, ts < 0 ?
getCurrentTimestamp() : ts, comment, mUniqueAgentID);
2281void CcdbApi::asynchPerform(
CurlHandle*
handle,
size_t* requestCounter)
const
2288 if (mIsCCDBDownloaderPreferred) {
2292 for (
int i = 1;
i <= mCurlRetries && (
result = curl_easy_perform(
handle)) != CURLE_OK;
i++) {
2293 usleep(mCurlDelayRetries *
i);
2304 LOG(
debug) <<
"Entering semaphore barrier";
2307 mSem =
new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, mSemName.c_str(), 1);
2308 }
catch (std::exception
e) {
2309 LOG(warn) <<
"Exception occurred during CCDB (cache) semaphore setup; Continuing without";
2314 gSemaRegistry.
add(
this);
2321 LOG(
debug) <<
"Ending semaphore barrier";
2324 if (mSem->try_wait()) {
2326 boost::interprocess::named_semaphore::remove(mSemName.c_str());
2328 gSemaRegistry.
remove(
this);
2334 LOG(
debug) <<
"Cleaning up semaphore registry with count " << mStore.size();
2335 for (
auto&
s : mStore) {
std::string createdNotBefore
std::string createdNotAfter
std::vector< std::string > header
std::vector< long > entries
configStandalone Light OpenGL display display_light Apply smoothing to points Enable Z buffer Highlight links Highlight seeds Highlight sector tracks Highlight final tracks Show individual Show related Highlight grid Show only individual collision Color clusters belonging to track objects Set line Enable drawing TPC data Enable drawing TOF data Enable drawing magnetic field Set field line step count Invert colors configStandalone Heavy OpenGL display display_heavy Show tracks with ITS contribution Show tracks with TOF contribution Enable propagation of loopers Show clusters at nominal x of pad row for early transformed data Split CE tracks when they cross the central Mark clusters Mark adjacent clusters Hide rejected tracks Apply filter on tracks to be displayed Separate increase X h configStandalone Camera window renderer OpenGL display display_camera Orient the camera such that the y axis is always upwards Full Screen MultiSample Anti Downsampling Anti Enable Vertical Sync Target frame or if the backend cannot limit to the refresh Use OpenGL indirect draws to reduce number of draw calls Display h configStandalone Vulkan display display_vulkan Have uniform buffers in host accessible device memory h configStandalone Internal display display_internal Number of updates the Qt GUI may send ahead of the last one the display has before it waits for the display Opacity of the console background Number of console lines the history is shortened to this when it reaches twice the size Time in seconds an info message fades out after setInfoTime h OpenGL display display Font size Do not use Freetype for font Renderer for event Frontend to use for event ROOT macros used as track filter Console script file
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char e
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc bool
configStandalone Reconstruction rec_tpc multiplier for the combined cluster track error during track following area in cm for the search of for z only used if decrease removal by when above this local occupancy rowx Shift of TPC revert cluster flag edge bit distance to edge exceeds this ZOffset for secondary multiple of radius max P of electron K BB bands crossing max P of electron p BB bands crossing max P of electron d BB bands crossing max P of electron t BB bands crossing min accepted dEdxMaxTPC of the track Additive extra cluster error for Y2 if edge flag set Additive extra cluster error for Y2 if single set Additive extra cluster error for Y2 if or single set Additive extra cluster error for Z2 if or single set Extra errors to add to track when crossing depending on addErrorsCECrossing Inner portion of y range in sector that is not used in searching for extrapolated track candidates factor2YS for track merging factor2K for track merging Margin in cm of Y position when rejecting edge clusters based on uncorrected track Y Max fraction of shared clusters for tracklet Except pad from cluster finding if total number of charges in a fragment is above this Except pad from cluster finding if number of consecutive charges in a fragment is above this Threshold where a timebin is considered disabling the noisy pad check for that pad Treat signal above this minimum as saturated Smoothing factor for the exponential Highly Ionising Particle tail filter Number of timebins to use for the averaging Add additional custom track errors when crossing relative as fraction of Do N fit passes in final fit of< 0:reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking(prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit(1:attachment, 2:propagation, 4:loop following, 8:mirroring)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression(0=no, 1=strategy A, 2=strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits(1=truncate charge/width LSB, 2=differences, 4=track-model)") AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", 0, "Sort order of TPC compression(0=time, 1=pad, 2=Z-time-pad, 3=Z-pad-time, 4=no sorting(use incoming order))") AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF Merge tracks accross the central electrode Enable PID response Add sub threshold clusters in TPC dEdx computation OR mask of TPC flags that will reject the cluster in alternative dEdx Reject edge clusters based on uncorrected track Y during track fit Increase decrease removal by when on this number of pads from the edge h configStandalone Reconstruction rec_trd Max chi2 for TRD tracklets to be matched to a track Chi2 cut for strict matching mode Number of sigmas for ITS TPC track time error estimate Addition to search road around track prolongation along Y in cm Optimal tracklet rphi resolution in maximum chi2 per attached tracklet for TRD tracks take into account the full likelihood of the slope including exponential tails Tracks with less attached tracklets are discarded after the tracking Post trigger Pile up integration time in BCs fits all iterations but the last one in projections Use material lookup table for TPC refit Apply workarounds to avoid cyclic merge should not be needed tpc dyn configStandalone Processing proc_rtc Replace constant variables static by constexpr expressions Compile RTC in deterministic with NO_FAST_MATH flags and GPUCA_DETERMINISTIC_MODE define Use RTC to optimize GPU code h configStandalone Processing proc_rtctech Use a file lock to serialize access to the cache folder Print launch bounds used for RTC code as debugging for exit after Allow RTC with slave GPUReconstruction instances with optConstexpr and optSpecialcode Maximum number of RTC cache files per backend in the cache folder
char error[CURL_ERROR_SIZE]
Class for time synchronization of RawReader instances.
void setRequestTimeoutTime(int timeoutMS)
CURLcode perform(CURL *handle)
void asynchSchedule(CURL *handle, size_t *requestCounter)
void setKeepaliveTimeoutTime(int timeoutMS)
void runLoop(bool noWait)
CCDBSemaphore(std::string const &cachepath, std::string const &path)
static std::string generateFileName(const std::string &inp)
std::string list(std::string const &path="", bool latestOnly=false, std::string const &returnFormat="text/plain", long createdNotAfter=-1, long createdNotBefore=-1) const
int storeAsTFile_impl(const void *obj1, std::type_info const &info, std::string const &path, std::map< std::string, std::string > const &metadata, long startValidityTimestamp=-1, long endValidityTimestamp=-1, std::vector< char >::size_type maxSize=0) const
static bool checkAlienToken()
void runDownloaderLoop(bool noWait)
void releaseNamedSemaphore(boost::interprocess::named_semaphore *sem, std::string const &path) const
static std::map< std::string, std::string > * retrieveMetaInfo(TFile &)
void scheduleDownload(RequestContext &requestContext, size_t *requestCounter) const
TObject * retrieve(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp) const
void init(std::string const &hosts)
TObject * retrieveFromTFile(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp, std::map< std::string, std::string > *headers, std::string const &etag, const std::string &createdNotAfter, const std::string &createdNotBefore) const
std::string const & getURL() const
void getFromSnapshot(bool createSnapshot, std::string const &path, long timestamp, std::map< std::string, std::string > &headers, std::string &snapshotpath, o2::pmr::vector< char > &dest, int &fromSnapshot, std::string const &etag) const
bool loadLocalContentToMemory(o2::pmr::vector< char > &dest, std::string &url) const
static void removeLeakingSemaphores(std::string const &basedir, bool remove=false)
void saveSnapshot(RequestContext &requestContext) const
static std::unique_ptr< std::vector< char > > createObjectImage(const T *obj, CcdbObjectInfo *info=nullptr)
static void * extractFromTFile(TFile &file, TClass const *cl, const char *what=CCDBOBJECT_ENTRY)
int storeAsTFile(const TObject *rootObject, std::string const &path, std::map< std::string, std::string > const &metadata, long startValidityTimestamp=-1, long endValidityTimestamp=-1, std::vector< char >::size_type maxSize=0) const
void snapshot(std::string const &ccdbrootpath, std::string const &localDir, long timestamp) const
bool isSnapshotMode() const
bool isHostReachable() const
void loadFileToMemory(std::vector< char > &dest, std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp, std::map< std::string, std::string > *headers, std::string const &etag, const std::string &createdNotAfter, const std::string &createdNotBefore, bool considerSnapshot=true) const
static std::string determineSemaphoreName(std::string const &basedir, std::string const &objectpath)
void deleteObject(std::string const &path, long timestamp=-1) const
static void appendFlatHeader(o2::pmr::vector< char > &dest, const std::map< std::string, std::string > &headers)
std::vector< std::string > getAllFolders(std::string const &top) const
void vectoredLoadFileToMemory(std::vector< RequestContext > &requestContext) const
static void curlSetSSLOptions(CurlHandle *curl)
boost::interprocess::named_semaphore * createNamedSemaphore(std::string const &path) const
static bool removeSemaphore(std::string const &name, bool remove=false)
std::map< std::string, std::string > retrieveHeaders(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp=-1) const
CcdbApi()
Default constructor.
static bool getCCDBEntryHeaders(std::string const &url, std::string const &etag, std::vector< std::string > &headers, const std::string &agentID="")
static CCDBQuery * retrieveQueryInfo(TFile &)
static constexpr const char * CCDBQUERY_ENTRY
void truncate(std::string const &path) const
int storeAsBinaryFile(const char *buffer, size_t size, const std::string &fileName, const std::string &objectType, const std::string &path, const std::map< std::string, std::string > &metadata, long startValidityTimestamp, long endValidityTimestamp, std::vector< char >::size_type maxSize=0) const
static void parseCCDBHeaders(std::vector< std::string > const &headers, std::vector< std::string > &pfns, std::string &etag)
bool retrieveBlob(std::string const &path, std::string const &targetdir, std::map< std::string, std::string > const &metadata, long timestamp, bool preservePathStructure=true, std::string const &localFileName="snapshot.root", std::string const &createdNotAfter="", std::string const &createdNotBefore="", std::map< std::string, std::string > *headers=nullptr) const
int updateMetadata(std::string const &path, std::map< std::string, std::string > const &metadata, long timestamp, std::string const &id="", long newEOV=0)
static constexpr const char * CCDBMETA_ENTRY
static constexpr const char * CCDBOBJECT_ENTRY
void navigateSourcesAndLoadFile(RequestContext &requestContext, int &fromSnapshot, size_t *requestCounter) const
std::vector< std::string > parseSubFolders(std::string const &reply) const
virtual ~CcdbApi()
Default destructor.
void add(CCDBSemaphore const *ptr)
void remove(CCDBSemaphore const *ptr)
SemaphoreRegistry()=default
GLdouble GLdouble GLdouble GLdouble top
GLboolean GLboolean GLboolean b
GLsizei const GLfloat * value
GLsizei const GLchar *const * path
GLboolean GLboolean GLboolean GLboolean a
GLbitfield GLuint64 timeout
information complementary to a CCDB object (path, metadata, startTimeValidity, endTimeValidity etc)
bool stdmap_to_jsonfile(std::map< std::string, std::string > const &meta, std::string const &filename)
size_t write_data(void *, size_t size, size_t nmemb, void *)
long getCurrentTimestamp()
returns the timestamp in long corresponding to "now"
size_t(*)(void *, size_t, size_t, void *) CurlWriteCallback
std::string sanitizeObjectName(const std::string &objectName)
bool jsonfile_to_stdmap(std::map< std::string, std::string > &meta, std::string const &filename)
long getFutureTimestamp(int secondsInFuture)
returns the timestamp in long corresponding to "now + secondsInFuture"
void CurlHandle
stands in for libcurl's typedef void CURL without including <curl/curl.h>
size_t CurlWrite_CallbackFunc_StdString2(void *contents, size_t size, size_t nmemb, std::string *s)
std::string timestamp() noexcept
Defining ITS Vertex explicitly as messageable.
std::vector< T, fair::mq::pmr::polymorphic_allocator< T > > vector
void createDirectoriesIfAbsent(std::string const &path)
std::string to_string(gsl::span< T, Size > span)
std::map< std::string, std::string > const & metadata
std::string createdNotAfter
std::map< std::string, std::string > & headers
o2::pmr::vector< char > & dest
std::string createdNotBefore
static DeploymentMode deploymentMode()
static std::string getClassName(const T &obj)
get the class name of the object
static std::unique_ptr< FileImage > createFileImage(const TObject &obj, const std::string &fileName, const std::string &objName)
static void trim(std::string &s)
static std::vector< std::string > tokenize(const std::string &src, char delim, bool trimToken=true, bool skipEmpty=true)
static std::string concat_string(Ts const &... ts)
static bool endsWith(const std::string &s, const std::string &ending)
void * realloc(void *old, size_t n)
LOG(info)<< "Compressed in "<< sw.CpuTime()<< " s"
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))
uint64_t const void const *restrict const msg