Project
Loading...
Searching...
No Matches
HttpByteSource.cxx
Go to the documentation of this file.
1// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#include "HttpByteSource.h"
13#include "AODNative/Errors.h"
14#include <curl/curl.h>
15#include <cstdlib>
16#include <cstring>
17#include <vector>
18
19namespace o2::aod::native
20{
21
22namespace
23{
24// Larger ranges are split, so that the pieces go out in parallel.
25constexpr size_t kMaxRequest = 16 * 1024 * 1024;
26
27void globalInit()
28{
29 static std::once_flag once;
30 std::call_once(once, [] { curl_global_init(CURL_GLOBAL_DEFAULT); });
31}
32
34struct Transfer {
36 size_t length;
38 size_t received = 0;
39 bool overflow = false;
40 CURL* handle = nullptr;
41 char error[CURL_ERROR_SIZE] = {};
42};
43
44size_t writeInto(char* data, size_t size, size_t n, void* userdata)
45{
46 auto* t = (Transfer*)userdata;
47 auto bytes = size * n;
48 // More than asked for: the server ignored the range. Abort rather than
49 // downloading the whole file.
50 if (t->received + bytes > t->length) {
51 t->overflow = true;
52 return 0;
53 }
54 std::memcpy(t->dst + t->received, data, bytes);
55 t->received += bytes;
56 return bytes;
57}
58
59size_t discard(char*, size_t size, size_t n, void*) { return size * n; }
60} // namespace
61
64 CURLSH* share = nullptr;
65 std::mutex locks[CURL_LOCK_DATA_LAST];
66
68 {
69 share = curl_share_init();
70 curl_share_setopt(share, CURLSHOPT_LOCKFUNC, +[](CURL*, curl_lock_data data, curl_lock_access, void* self) { ((Shared*)self)->locks[data].lock(); });
71 curl_share_setopt(share, CURLSHOPT_UNLOCKFUNC, +[](CURL*, curl_lock_data data, void* self) { ((Shared*)self)->locks[data].unlock(); });
72 curl_share_setopt(share, CURLSHOPT_USERDATA, this);
73 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
74 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
75 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
76 }
77 ~Shared() { curl_share_cleanup(share); }
78
79 CURL* handle(std::string const& url, Options const& options) const
80 {
81 auto* h = curl_easy_init();
82 curl_easy_setopt(h, CURLOPT_URL, url.c_str());
83 curl_easy_setopt(h, CURLOPT_SHARE, share);
84 curl_easy_setopt(h, CURLOPT_FOLLOWLOCATION, 1L);
85 curl_easy_setopt(h, CURLOPT_MAXREDIRS, 10L);
86 curl_easy_setopt(h, CURLOPT_NOSIGNAL, 1L);
87 curl_easy_setopt(h, CURLOPT_USERAGENT, "o2-aod-native");
88 curl_easy_setopt(h, CURLOPT_CONNECTTIMEOUT, 30L);
89 // Give up on a transfer slower than 1 kB/s for a minute.
90 curl_easy_setopt(h, CURLOPT_LOW_SPEED_LIMIT, 1024L);
91 curl_easy_setopt(h, CURLOPT_LOW_SPEED_TIME, 60L);
92 if (auto* dir = std::getenv("X509_CERT_DIR")) {
93 curl_easy_setopt(h, CURLOPT_CAPATH, dir);
94 }
95 long version = CURL_HTTP_VERSION_2TLS;
96 if (options.httpVersion == Options::HttpVersion::Http1_1) {
97 version = CURL_HTTP_VERSION_1_1;
98 } else if (options.httpVersion == Options::HttpVersion::Http2) {
99 version = CURL_HTTP_VERSION_2_0;
100 }
101 curl_easy_setopt(h, CURLOPT_HTTP_VERSION, version);
102 return h;
103 }
104};
105
106HttpByteSource::HttpByteSource(std::string url, Options const& options)
107 : mUrl(std::move(url)), mName(mUrl.substr(0, mUrl.find('?'))), mOptions(options)
108{
109 globalInit();
110 mShared = std::make_unique<Shared>();
111 auto* h = mShared->handle(mUrl, mOptions);
112 char error[CURL_ERROR_SIZE] = {};
113 curl_easy_setopt(h, CURLOPT_ERRORBUFFER, error);
114 curl_easy_setopt(h, CURLOPT_NOBODY, 1L);
115 curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, discard);
116 auto code = curl_easy_perform(h);
117 long status = 0;
118 curl_off_t length = -1;
119 curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &status);
120 curl_easy_getinfo(h, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length);
121 curl_easy_cleanup(h);
122 if (code != CURLE_OK) {
123 throw IoError(mName, error[0] ? error : curl_easy_strerror(code));
124 }
125 if (status != 200) {
126 throw IoError(mName, "HEAD returned HTTP " + std::to_string(status));
127 }
128 if (length < 0) {
129 throw IoError(mName, "the server does not tell the size of the file");
130 }
131 mSize = length;
132}
133
135
136void HttpByteSource::checkRange(uint64_t off, size_t len) const
137{
138 if (off > mSize || len > mSize - off) {
139 throw CorruptData(mName, off, "read of " + std::to_string(len) + " bytes past the end of a " + std::to_string(mSize) + " byte file");
140 }
141}
142
143void HttpByteSource::readInto(uint64_t off, size_t len, void* dst)
144{
145 ReadRequest request{off, len, dst};
146 readvInto({&request, 1});
147}
148
149void HttpByteSource::readvInto(std::span<ReadRequest> requests)
150{
151 std::vector<Transfer> transfers;
152 size_t total = 0;
153 for (auto const& r : requests) {
154 checkRange(r.offset, r.length);
155 total += r.length;
156 for (size_t done = 0; done < r.length; done += kMaxRequest) {
157 transfers.push_back({r.offset + done, std::min(kMaxRequest, r.length - done), (uint8_t*)r.dst + done});
158 }
159 }
160 if (transfers.empty()) {
161 return;
162 }
163
164 auto* multi = curl_multi_init();
165 curl_multi_setopt(multi, CURLMOPT_PIPELINING, (long)CURLPIPE_MULTIPLEX);
166 curl_multi_setopt(multi, CURLMOPT_MAX_HOST_CONNECTIONS, (long)std::max(1, mOptions.httpConnections));
167 std::vector<std::string> ranges;
168 ranges.reserve(transfers.size());
169 for (auto& t : transfers) {
170 t.handle = mShared->handle(mUrl, mOptions);
171 ranges.push_back(std::to_string(t.offset) + "-" + std::to_string(t.offset + t.length - 1));
172 curl_easy_setopt(t.handle, CURLOPT_RANGE, ranges.back().c_str());
173 curl_easy_setopt(t.handle, CURLOPT_WRITEFUNCTION, writeInto);
174 curl_easy_setopt(t.handle, CURLOPT_WRITEDATA, &t);
175 curl_easy_setopt(t.handle, CURLOPT_ERRORBUFFER, t.error);
176 curl_easy_setopt(t.handle, CURLOPT_PRIVATE, &t);
177 // Wait for a multiplexed connection rather than opening one per request.
178 curl_easy_setopt(t.handle, CURLOPT_PIPEWAIT, 1L);
179 curl_multi_add_handle(multi, t.handle);
180 }
181
182 std::string error;
183 int running = 0;
184 do {
185 auto mc = curl_multi_perform(multi, &running);
186 if (mc != CURLM_OK) {
187 error = curl_multi_strerror(mc);
188 break;
189 }
190 int queued = 0;
191 while (auto* msg = curl_multi_info_read(multi, &queued)) {
192 if (msg->msg != CURLMSG_DONE) {
193 continue;
194 }
195 Transfer* t = nullptr;
196 curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char**)&t);
197 long status = 0;
198 curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &status);
199 if (!error.empty()) {
200 continue;
201 }
202 if (t->overflow) {
203 error = "the server ignored the byte range (HTTP " + std::to_string(status) + ")";
204 } else if (msg->data.result != CURLE_OK) {
205 error = t->error[0] ? t->error : curl_easy_strerror(msg->data.result);
206 } else if (status != 206 && !(status == 200 && t->offset == 0 && t->length == mSize)) {
207 error = "range request returned HTTP " + std::to_string(status);
208 } else if (t->received != t->length) {
209 error = "short read: " + std::to_string(t->received) + " of " + std::to_string(t->length) + " bytes";
210 }
211 }
212 if (running > 0) {
213 curl_multi_poll(multi, nullptr, 0, 1000, nullptr);
214 }
215 } while (running > 0 && error.empty());
216
217 for (auto& t : transfers) {
218 curl_multi_remove_handle(multi, t.handle);
219 curl_easy_cleanup(t.handle);
220 }
221 curl_multi_cleanup(multi);
222 if (!error.empty()) {
223 throw IoError(mName, error);
224 }
225 mReadCalls.fetch_add(transfers.size(), std::memory_order_relaxed);
226 mBytesRead.fetch_add(total, std::memory_order_relaxed);
227}
228
229} // namespace o2::aod::native
std::string url
bool done
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
bool overflow
char error[CURL_ERROR_SIZE]
size_t received
CURL * handle
uint32_t version
Definition RawData.h:8
size_t bytes
Class for time synchronization of RawReader instances.
void readInto(uint64_t off, size_t len, void *dst) override
Throws IoError, or CorruptData when the range is past the end of the file.
HttpByteSource(std::string url, Options const &options)
std::string url() const override
Without the query, which may hold a token.
void readvInto(std::span< ReadRequest > requests) override
GLdouble n
Definition glcorearb.h:1982
GLsizeiptr size
Definition glcorearb.h:659
GLenum GLenum dst
Definition glcorearb.h:1767
GLintptr offset
Definition glcorearb.h:660
GLuint GLsizei GLsizei * length
Definition glcorearb.h:790
GLenum GLenum GLsizei len
Definition glcorearb.h:4232
std::string to_string(gsl::span< T, Size > span)
Definition common.h:52
What all the requests on a source share: connections, TLS sessions and DNS.
std::mutex locks[CURL_LOCK_DATA_LAST]
CURL * handle(std::string const &url, Options const &options) const
Failure of the underlying transport.
Definition Errors.h:69
uint64_t const void const *restrict const msg
Definition x9.h:153