Project
Loading...
Searching...
No Matches
test_Failover.cxx
Go to the documentation of this file.
1// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#include "FailoverByteSource.h"
13#include "AODNative/Errors.h"
14#include <catch_amalgamated.hpp>
15#include <atomic>
16#include <map>
17#include <thread>
18
19using namespace o2::aod::native;
20
21namespace
22{
23// A replica whose byte i is i % 251. Its behaviour is shared between the
24// instances opened for the same URL, so that a reopen sees the same state.
25struct Behaviour {
26 size_t size = 1000;
27 bool opens = true;
28 std::atomic<int> failingReads = 0; // the next reads which fail with IoError
29 std::atomic<int> opened = 0;
30 std::atomic<int> reads = 0;
31};
32
33struct FakeSource : ByteSource {
34 FakeSource(std::string u, Behaviour& b) : name(std::move(u)), behaviour(b) {}
35 uint64_t size() const override { return behaviour.size; }
36 void readInto(uint64_t off, size_t len, void* dst) override
37 {
38 ReadRequest r{off, len, dst};
39 readvInto({&r, 1});
40 }
41 void readvInto(std::span<ReadRequest> requests) override
42 {
43 ++behaviour.reads;
44 int failing = behaviour.failingReads.load();
45 while (failing > 0 && !behaviour.failingReads.compare_exchange_weak(failing, failing - 1)) {
46 }
47 if (failing > 0) {
48 throw IoError(name, "Operation expired");
49 }
50 for (auto& r : requests) {
51 for (size_t i = 0; i < r.length; ++i) {
52 ((uint8_t*)r.dst)[i] = (r.offset + i) % 251;
53 }
54 bytes += r.length;
55 }
56 ++calls;
57 }
58 bool isRemote() const override { return true; }
59 uint64_t bytesRead() const override { return bytes; }
60 uint64_t readCalls() const override { return calls; }
61 std::string url() const override { return name; }
62 std::string name;
63 Behaviour& behaviour;
64 std::atomic<uint64_t> bytes = 0;
65 std::atomic<uint64_t> calls = 0;
66};
67
68struct Setup {
69 std::map<std::string, Behaviour> replicas;
70 std::vector<std::string> log;
71 Options options;
72 Setup() { options.retryDelay = 0; }
73
74 std::unique_ptr<FailoverByteSource> open()
75 {
76 std::vector<Replica> list;
77 for (auto& [url, _] : replicas) {
78 list.push_back({url, "SE-" + url});
79 }
80 auto opener = [this](std::string const& url, Options const&) -> std::unique_ptr<ByteSource> {
81 auto& b = replicas.at(url);
82 ++b.opened;
83 if (!b.opens) {
84 throw IoError(url, "no such file");
85 }
86 return std::make_unique<FakeSource>(url, b);
87 };
88 return std::make_unique<FailoverByteSource>(list, options, opener, [this](std::string const& m) { log.push_back(m); });
89 }
90};
91
92bool readsCorrectly(ByteSource& s, uint64_t off = 100, size_t len = 50)
93{
94 std::vector<uint8_t> buf(len);
95 s.readInto(off, len, buf.data());
96 for (size_t i = 0; i < len; ++i) {
97 if (buf[i] != (off + i) % 251) {
98 return false;
99 }
100 }
101 return true;
102}
103} // namespace
104
105TEST_CASE("FailoverByteSource reads the first replica")
106{
107 Setup s;
108 s.replicas["a"];
109 s.replicas["b"];
110 auto f = s.open();
111 REQUIRE(readsCorrectly(*f));
112 CHECK(f->replicaLabel() == "SE-a");
113 CHECK(s.replicas["b"].opened == 0);
114 CHECK(f->bytesRead() == 50);
115 CHECK(f->readCalls() == 1);
116 CHECK(s.log.empty());
117}
118
119TEST_CASE("FailoverByteSource skips replicas which do not open")
120{
121 Setup s;
122 s.replicas["a"].opens = false;
123 s.replicas["b"];
124 auto f = s.open();
125 CHECK(f->replicaLabel() == "SE-b");
126 REQUIRE(readsCorrectly(*f));
127 CHECK(s.log.size() == 1);
128}
129
130TEST_CASE("FailoverByteSource throws when no replica opens")
131{
132 Setup s;
133 s.replicas["a"].opens = false;
134 s.replicas["b"].opens = false;
135 CHECK_THROWS_AS(s.open(), IoError);
136}
137
138TEST_CASE("FailoverByteSource retries a transient read error on the same replica")
139{
140 Setup s;
141 s.replicas["a"].failingReads = 1;
142 s.replicas["b"];
143 auto f = s.open();
144 REQUIRE(readsCorrectly(*f));
145 CHECK(f->replicaLabel() == "SE-a");
146 CHECK(s.replicas["a"].opened == 2); // reopened for the retry
147 CHECK(s.replicas["b"].opened == 0);
148 REQUIRE(s.log.size() == 1);
149 CHECK(s.log[0].find("retry 1 of 2") != std::string::npos);
150}
151
152TEST_CASE("FailoverByteSource moves to the next replica after the retries")
153{
154 Setup s;
155 s.replicas["a"].failingReads = 100;
156 s.replicas["b"];
157 auto f = s.open();
158 REQUIRE(readsCorrectly(*f));
159 CHECK(f->replicaLabel() == "SE-b");
160 CHECK(s.replicas["a"].reads == 3); // first attempt + 2 retries
161 CHECK(s.log.back().find("Moving to replica on SE-b") != std::string::npos);
162 // Later reads stay on the new replica.
163 REQUIRE(readsCorrectly(*f, 900, 100));
164 CHECK(s.replicas["a"].reads == 3);
165}
166
167TEST_CASE("FailoverByteSource does not use a replica of a different size")
168{
169 Setup s;
170 s.replicas["a"].failingReads = 100;
171 s.replicas["b"].size = 999;
172 s.replicas["c"];
173 auto f = s.open();
174 REQUIRE(readsCorrectly(*f));
175 CHECK(f->replicaLabel() == "SE-c");
176}
177
178TEST_CASE("FailoverByteSource throws when every replica fails")
179{
180 Setup s;
181 s.replicas["a"].failingReads = 100;
182 s.replicas["b"].failingReads = 100;
183 auto f = s.open();
184 std::vector<uint8_t> buf(10);
185 try {
186 f->readInto(0, 10, buf.data());
187 FAIL("no exception");
188 } catch (IoError const& e) {
189 CHECK(std::string(e.what()).find("no replica left") != std::string::npos);
190 }
191 CHECK(s.replicas["a"].reads == 3);
192 CHECK(s.replicas["b"].reads == 3);
193}
194
195TEST_CASE("FailoverByteSource counts the bytes of retired replicas")
196{
197 Setup s;
198 s.replicas["a"];
199 s.replicas["b"];
200 auto f = s.open();
201 REQUIRE(readsCorrectly(*f, 0, 100));
202 s.replicas["a"].failingReads = 100;
203 REQUIRE(readsCorrectly(*f, 0, 30));
204 CHECK(f->bytesRead() == 130);
205 CHECK(f->readCalls() == 2);
206}
207
208TEST_CASE("FailoverByteSource with concurrent readers")
209{
210 Setup s;
211 s.replicas["a"].failingReads = 100;
212 s.replicas["b"];
213 auto f = s.open();
214 std::vector<std::thread> threads;
215 std::atomic<int> good = 0;
216 for (int t = 0; t < 4; ++t) {
217 threads.emplace_back([&, t] {
218 for (int i = 0; i < 20; ++i) {
219 good += readsCorrectly(*f, 10 * t + i, 40);
220 }
221 });
222 }
223 for (auto& t : threads) {
224 t.join();
225 }
226 CHECK(good == 80);
227 CHECK(f->replicaLabel() == "SE-b");
228}
std::string url
int32_t i
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error s
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char e
size_t bytes
virtual std::string url() const =0
virtual uint64_t bytesRead() const =0
virtual uint64_t size() const =0
virtual bool isRemote() const
Whether each read is a network round trip, worth hiding behind decoding.
Definition ByteSource.h:92
virtual void readInto(uint64_t off, size_t len, void *dst)=0
Throws IoError, or CorruptData when the range is past the end of the file.
virtual uint64_t readCalls() const =0
virtual void readvInto(std::span< ReadRequest > requests)=0
#define CHECK
const GLfloat * m
Definition glcorearb.h:4066
GLsizeiptr size
Definition glcorearb.h:659
GLuint const GLchar * name
Definition glcorearb.h:781
GLdouble f
Definition glcorearb.h:310
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLenum GLenum dst
Definition glcorearb.h:1767
GLboolean r
Definition glcorearb.h:1233
GLenum GLenum GLsizei len
Definition glcorearb.h:4232
GLenum GLuint GLenum GLsizei const GLchar * buf
Definition glcorearb.h:2514
bool list(IEventListener &reporter, Config const &config)
Failure of the underlying transport.
Definition Errors.h:69
TEST_CASE("FailoverByteSource reads the first replica")