Project
Loading...
Searching...
No Matches
test_ReadPlan.cxx
Go to the documentation of this file.
1// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
12#include "AODNative/ReadPlan.h"
13#include <catch_amalgamated.hpp>
14#include <cstring>
15#include <thread>
16
17using namespace o2::aod::native;
18
19namespace
20{
21// File contents where byte i is i % 251, recording every read.
22struct RecordingSource : ByteSource {
23 explicit RecordingSource(size_t n) : bytes(n)
24 {
25 for (size_t i = 0; i < n; ++i) {
26 bytes[i] = i % 251;
27 }
28 }
29 uint64_t size() const override { return bytes.size(); }
30 void readInto(uint64_t off, size_t len, void* dst) override
31 {
32 reads.emplace_back(off, len);
33 std::memcpy(dst, bytes.data() + off, len);
34 }
35 void readvInto(std::span<ReadRequest> requests) override
36 {
37 for (auto& r : requests) {
38 readInto(r.offset, r.length, r.dst);
39 }
40 }
41 uint64_t bytesRead() const override { return 0; }
42 uint64_t readCalls() const override { return reads.size(); }
43 std::string url() const override { return "memory"; }
44
45 std::vector<uint8_t> bytes;
46 std::vector<std::pair<uint64_t, size_t>> reads;
47};
48
49bool holds(std::span<const uint8_t> span, uint64_t offset)
50{
51 for (size_t i = 0; i < span.size(); ++i) {
52 if (span[i] != (offset + i) % 251) {
53 return false;
54 }
55 }
56 return true;
57}
58
59Options withGap(size_t gap)
60{
61 Options options;
62 options.coalesceGap = gap;
63 return options;
64}
65} // namespace
66
67TEST_CASE("ranges closer than the gap are fetched in one read, on first use")
68{
69 auto source = std::make_shared<RecordingSource>(1 << 20);
70 ReadPlan plan{source, withGap(100)};
71 plan.add(5000, 200);
72 plan.add(1000, 100);
73 plan.add(1150, 50); // 50 bytes after the first: merged
74 plan.add(1300, 10); // 100 bytes after the second: merged
75 REQUIRE(source->reads.empty());
76
77 auto bytes = plan.find(1150, 50);
78 REQUIRE(holds(bytes, 1150));
79 REQUIRE(source->reads == std::vector<std::pair<uint64_t, size_t>>{{1000, 310}, {5000, 200}});
80 REQUIRE(plan.bytesHeld() == 510);
81
82 REQUIRE(holds(plan.find(5100, 100), 5100));
83 REQUIRE(holds(plan.find(1000, 310), 1000));
84 REQUIRE(source->reads.size() == 2);
85}
86
87TEST_CASE("ranges which were never registered are not found")
88{
89 auto source = std::make_shared<RecordingSource>(1 << 20);
90 ReadPlan plan{source, withGap(0)};
91 plan.add(1000, 100);
92 REQUIRE(plan.find(900, 50).empty());
93 REQUIRE(plan.find(1050, 100).empty()); // runs past the fetched range
94 REQUIRE(plan.find(5000, 1).empty());
95 REQUIRE(source->reads.size() == 1);
96}
97
98TEST_CASE("ranges added after a fetch are fetched on their first use")
99{
100 auto source = std::make_shared<RecordingSource>(1 << 20);
101 ReadPlan plan{source, withGap(1000)};
102 plan.add(1000, 100);
103 REQUIRE(holds(plan.find(1000, 100), 1000));
104 auto first = plan.find(1000, 100);
105
106 // Within the gap of the fetched chunk on both sides, but not merged across it.
107 plan.add(500, 100);
108 plan.add(1500, 100);
109 plan.add(1000, 100); // already in memory: not read again
110 REQUIRE(holds(plan.find(1500, 100), 1500));
111 REQUIRE(holds(plan.find(500, 100), 500));
112 REQUIRE(source->reads == std::vector<std::pair<uint64_t, size_t>>{{1000, 100}, {500, 100}, {1500, 100}});
113 // Spans handed out before stay valid.
114 REQUIRE(plan.find(1000, 100).data() == first.data());
115 REQUIRE(holds(first, 1000));
116}
117
118TEST_CASE("ranges past the end of the file are left to the reader")
119{
120 auto source = std::make_shared<RecordingSource>(1000);
121 ReadPlan plan{source, withGap(0)};
122 plan.add(900, 200);
123 plan.add(2000, 1);
124 REQUIRE(plan.find(900, 100).empty());
125 REQUIRE(source->reads.empty());
126}
127
128TEST_CASE("a planned source serves planned ranges from memory and the rest from the source")
129{
130 auto source = std::make_shared<RecordingSource>(1 << 20);
131 auto plan = std::make_shared<ReadPlan>(source, withGap(0));
132 plan->add(1000, 100);
133 PlannedByteSource planned{source, plan};
134
135 REQUIRE(holds(planned.tryMap(1010, 20), 1010));
136 std::vector<uint8_t> out(30);
137 planned.readInto(1020, 30, out.data());
138 REQUIRE(holds(out, 1020));
139 REQUIRE(planned.tryMap(3000, 10).empty());
140 planned.readInto(3000, 30, out.data());
141 REQUIRE(holds(out, 3000));
142 REQUIRE(source->reads == std::vector<std::pair<uint64_t, size_t>>{{1000, 100}, {3000, 30}});
143 REQUIRE(planned.readCalls() == 2);
144}
145
146TEST_CASE("concurrent lookups fetch the pending ranges once")
147{
148 auto source = std::make_shared<RecordingSource>(1 << 20);
149 ReadPlan plan{source, withGap(0)};
150 for (int i = 0; i < 64; ++i) {
151 plan.add(i * 10000, 1000);
152 }
153 std::vector<std::thread> threads;
154 std::atomic<int> good = 0;
155 for (int t = 0; t < 8; ++t) {
156 threads.emplace_back([&, t] {
157 for (int i = t; i < 64; i += 8) {
158 good += holds(plan.find(i * 10000 + 10, 900), i * 10000 + 10);
159 }
160 });
161 }
162 for (auto& t : threads) {
163 t.join();
164 }
165 REQUIRE(good == 64);
166 REQUIRE(source->reads.size() == 64);
167}
168
169TEST_CASE("buffers of finished plans are reused by the next one")
170{
171 auto source = std::make_shared<RecordingSource>(1 << 20);
172 auto pool = std::make_shared<ReadBufferPool>();
173 uint8_t const* first = nullptr;
174 {
175 ReadPlan plan{source, withGap(0), pool};
176 plan.add(1000, 1000);
177 first = plan.find(1000, 1000).data();
178 }
179 ReadPlan next{source, withGap(0), pool};
180 next.add(50000, 900); // fits in the first buffer
181 auto bytes = next.find(50000, 900);
182 REQUIRE(bytes.data() == first);
183 REQUIRE(holds(bytes, 50000));
184
185 auto big = pool->get(1 << 16);
186 REQUIRE(big.capacity >= 1 << 16);
187}
188
189TEST_CASE("a pool with a budget drops the largest free buffers beyond it")
190{
191 ReadBufferPool pool{1000};
192 auto small = pool.get(400);
193 auto medium = pool.get(500);
194 auto large = pool.get(2000);
195 auto* smallData = small.data.get();
196 auto* mediumData = medium.data.get();
197 pool.put(std::move(small));
198 pool.put(std::move(medium));
199 pool.put(std::move(large)); // over the budget: the largest goes
200 REQUIRE(pool.get(300).data.get() == smallData);
201 REQUIRE(pool.get(480).data.get() == mediumData);
202}
203
204TEST_CASE("bytesOf counts the baskets a plan reads, not the embedded ones")
205{
207 tree.branches.resize(2);
208 tree.branches[0].baskets = {BasketRef{100, 300, 0, 10, false, {}}, BasketRef{400, 200, 10, 5, false, {}}};
209 tree.branches[1].baskets = {BasketRef{600, 50, 0, 10, false, {}}, BasketRef{0, 0, 10, 5, true, {1, 2, 3}}};
210 REQUIRE(ReadPlan::bytesOf(tree) == 550);
211}
int32_t i
Timeframe settings Average distance in cm of events merged into Randomize distance around average distance of merged events Simulate correct bunch interactions instead of placing only the average number of events A value[n] sets TFlen for[n] collisions in Number of bunches per trainsort Number of bunch trains Instantaneous interaction rate Do not simulate border Total number of collisions to be placed in the interior of all time Do not select random but walk over array of events in stride steps h Event generator settings h configStandalone Use CPU for GPU Force usage of the specified GPU device no CPU fallback Continue processing after an error First event to Number of events to Number of iterations to Number of iterations to Number of initial iterations excluded from const char Directory with events to Events directory is and not inside events Show standalone event Run event generator maximum time bin of continuous for triggered for automatic continuous for automatic continuous triggered uint64_t
size_t bytes
virtual std::string url() const =0
virtual uint64_t bytesRead() const =0
virtual uint64_t size() const =0
virtual void readInto(uint64_t off, size_t len, void *dst)=0
Throws IoError, or CorruptData when the range is past the end of the file.
virtual uint64_t readCalls() const =0
virtual void readvInto(std::span< ReadRequest > requests)=0
Buffer get(size_t size)
The smallest free buffer of at least size bytes, or a new one.
Definition ReadPlan.cxx:21
static uint64_t bytesOf(TreeInfo const &tree)
The bytes add(tree) registers.
Definition ReadPlan.cxx:106
GLdouble n
Definition glcorearb.h:1982
GLint first
Definition glcorearb.h:399
GLsizei GLsizei GLchar * source
Definition glcorearb.h:798
GLenum GLenum dst
Definition glcorearb.h:1767
GLintptr offset
Definition glcorearb.h:660
GLenum GLenum GLsizei len
Definition glcorearb.h:4232
std::unique_ptr< uint8_t[], Unmap > data
Definition ReadPlan.h:38
std::vector< BranchInfo > branches
Definition Objects.h:52
TEST_CASE("ranges closer than the gap are fetched in one read, on first use")
std::unique_ptr< TTree > tree((TTree *) flIn.Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()))