Project
Loading...
Searching...
No Matches
GPUReconstruction.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11
14
15#include <cstring>
16#include <cstdio>
17#include <iostream>
18#include <mutex>
19#include <string>
20#include <map>
21#include <queue>
22#include <mutex>
23#include <condition_variable>
24#include <array>
25
26#include "GPUReconstruction.h"
29#include "GPUReconstructionIO.h"
30#include "GPUROOTDumpCore.h"
31#include "GPUConfigDump.h"
32#include "GPUChainTracking.h"
33#include "GPUConstantMem.h"
34#include "GPUCommonHelpers.h"
35#include "GPUSettings.h"
36
37#include "GPUMemoryResource.h"
38#include "GPUChain.h"
40
41#include "GPULogging.h"
42#include "utils/strtag.h"
43
44#ifdef GPUCA_O2_LIB
46#endif
47
49
50namespace o2::gpu
51{
52namespace // anonymous
53{
54struct GPUReconstructionPipelineQueue {
55 uint32_t op = 0; // For now, 0 = process, 1 = terminate
56 GPUChain* chain = nullptr;
57 std::mutex m;
58 std::condition_variable c;
59 bool done = false;
60 int32_t retVal = 0;
61};
62} // namespace
63
65 std::queue<GPUReconstructionPipelineQueue*> queue;
66 std::mutex mutex;
67 std::condition_variable cond;
68 bool terminate = false;
69};
70} // namespace o2::gpu
71
72using namespace o2::gpu;
73
74constexpr const char* const GPUReconstruction::GEOMETRY_TYPE_NAMES[];
75constexpr const char* const GPUReconstruction::IOTYPENAMES[];
77
78static ptrdiff_t ptrDiff(void* a, void* b) { return (char*)a - (char*)b; }
79
80GPUReconstruction::GPUReconstruction(const GPUSettingsDeviceBackend& cfg) : mHostConstantMem(new GPUConstantMem), mGRPSettings(new GPUSettingsGRP), mDeviceBackendSettings(new GPUSettingsDeviceBackend(cfg)), mProcessingSettings(new GPUSettingsProcessing)
81{
82 if (cfg.master) {
84 throw std::invalid_argument("device type of master and slave GPUReconstruction does not match");
85 }
86 if (cfg.master->mMaster) {
87 throw std::invalid_argument("Cannot be slave to a slave");
88 }
89 mMaster = cfg.master;
90 cfg.master->mSlaves.emplace_back(this);
91 }
94 for (uint32_t i = 0; i < NSECTORS; i++) {
95 processors()->tpcTrackers[i].SetSector(i); // TODO: Move to a better place
97#ifdef GPUCA_HAS_ONNX
98 processors()->tpcNNClusterer[i].mISector = i;
99#endif
100 }
101#ifndef GPUCA_NO_ROOT
102 mROOTDump = GPUROOTDumpCore::getAndCreate();
103#endif
104}
105
107{
108 if (mInitialized) {
109 GPUError("GPU Reconstruction not properly deinitialized!");
110 }
111}
112
113void GPUReconstruction::GetITSTraits(std::unique_ptr<o2::its::TrackerTraits>* trackerTraits, std::unique_ptr<o2::its::VertexerTraits>* vertexerTraits, std::unique_ptr<o2::its::TimeFrame>* timeFrame)
114{
115 if (trackerTraits) {
116 trackerTraits->reset(new o2::its::TrackerTraits);
117 }
118 if (vertexerTraits) {
119 vertexerTraits->reset(new o2::its::VertexerTraits);
120 }
121 if (timeFrame) {
122 timeFrame->reset(new o2::its::TimeFrame);
123 }
124}
125
127{
128 return std::max<int32_t>(0, tbb::this_task_arena::current_thread_index());
129}
130
132{
133 if (mMaster) {
134 throw std::runtime_error("Must not call init on slave!");
135 }
136 int32_t retVal = InitPhaseBeforeDevice();
137 if (retVal) {
138 return retVal;
139 }
140 for (uint32_t i = 0; i < mSlaves.size(); i++) {
141 retVal = mSlaves[i]->InitPhaseBeforeDevice();
142 if (retVal) {
143 GPUError("Error initialization slave (before deviceinit)");
144 return retVal;
145 }
146 mNStreams = std::max(mNStreams, mSlaves[i]->mNStreams);
149 }
150 if (InitDevice()) {
151 return 1;
152 }
153 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
156 } else {
158 }
160 return 1;
161 }
162 for (uint32_t i = 0; i < mSlaves.size(); i++) {
163 mSlaves[i]->mDeviceMemoryBase = mDeviceMemoryPermanent;
164 mSlaves[i]->mHostMemoryBase = mHostMemoryPermanent;
165 mSlaves[i]->mDeviceMemorySize = mDeviceMemorySize - ptrDiff(mSlaves[i]->mDeviceMemoryBase, mDeviceMemoryBase);
166 mSlaves[i]->mHostMemorySize = mHostMemorySize - ptrDiff(mSlaves[i]->mHostMemoryBase, mHostMemoryBase);
167 mSlaves[i]->mHostMemoryPoolEnd = mHostMemoryPoolEnd;
168 mSlaves[i]->mDeviceMemoryPoolEnd = mDeviceMemoryPoolEnd;
169 if (mSlaves[i]->InitDevice()) {
170 GPUError("Error initialization slave (deviceinit)");
171 return 1;
172 }
174 GPUError("Error initialization slave (permanent memory)");
175 return 1;
176 }
177 mDeviceMemoryPermanent = mSlaves[i]->mDeviceMemoryPermanent;
178 mHostMemoryPermanent = mSlaves[i]->mHostMemoryPermanent;
179 }
181 if (retVal) {
182 return retVal;
183 }
185 for (uint32_t i = 0; i < mSlaves.size(); i++) {
186 mSlaves[i]->mDeviceMemoryPermanent = mDeviceMemoryPermanent;
187 mSlaves[i]->mHostMemoryPermanent = mHostMemoryPermanent;
188 retVal = mSlaves[i]->InitPhaseAfterDevice();
189 if (retVal) {
190 GPUError("Error initialization slave (after device init)");
191 return retVal;
192 }
193 mSlaves[i]->ClearAllocatedMemory();
194 }
195 return 0;
196}
197
198namespace o2::gpu::internal
199{
200static uint32_t getDefaultNThreads()
201{
202 const char* tbbEnv = getenv("TBB_NUM_THREADS");
203 uint32_t tbbNum = tbbEnv ? atoi(tbbEnv) : 0;
204 if (tbbNum) {
205 return tbbNum;
206 }
207 const char* ompEnv = getenv("OMP_NUM_THREADS");
208 uint32_t ompNum = ompEnv ? atoi(ompEnv) : 0;
209 if (ompNum) {
210 return tbbNum;
211 }
212 return tbb::info::default_concurrency();
213}
214} // namespace o2::gpu::internal
215
217{
218 if (GetProcessingSettings().printSettings) {
219 if (mSlaves.size() || mMaster) {
220 printf("\nConfig Dump %s\n", mMaster ? "Slave" : "Master");
221 }
222 const GPUChainTracking* chTrk;
223 for (uint32_t i = 0; i < mChains.size(); i++) {
224 if ((chTrk = dynamic_cast<GPUChainTracking*>(mChains[i].get()))) {
225 break;
226 }
227 }
228 GPUConfigDump::dumpConfig(&param().rec, mProcessingSettings.get(), chTrk ? chTrk->GetQAConfig() : nullptr, chTrk ? chTrk->GetEventDisplayConfig() : nullptr, mDeviceBackendSettings.get(), &mRecoSteps);
229 }
232 if (!IsGPU()) {
233 mRecoSteps.stepsGPUMask.set((uint8_t)0);
234 }
235
236 if (GetProcessingSettings().forceMemoryPoolSize >= 1024 || GetProcessingSettings().forceHostMemoryPoolSize >= 1024) {
238 }
239 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_AUTO) {
241 }
242 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
243 mProcessingSettings->forceMemoryPoolSize = mProcessingSettings->forceHostMemoryPoolSize = 0;
244 }
245 if (GetProcessingSettings().debugLevel >= 4) {
246 mProcessingSettings->keepAllMemory = true;
247 }
248 if (GetProcessingSettings().debugLevel >= 5 && GetProcessingSettings().allocDebugLevel < 2) {
249 mProcessingSettings->allocDebugLevel = 2;
250 }
252 mProcessingSettings->keepDisplayMemory = true;
253 }
254 if (GetProcessingSettings().debugLevel < 6) {
255 mProcessingSettings->debugMask = 0;
256 }
257 if (GetProcessingSettings().debugLevel < 1) {
258 mProcessingSettings->deviceTimers = false;
259 }
260 if (GetProcessingSettings().debugLevel > 0) {
261 mProcessingSettings->recoTaskTiming = true;
262 }
263 if (GetProcessingSettings().deterministicGPUReconstruction == -1) {
264 mProcessingSettings->deterministicGPUReconstruction = GetProcessingSettings().debugLevel >= 6;
265 }
266 if (GetProcessingSettings().deterministicGPUReconstruction) {
267#ifndef GPUCA_DETERMINISTIC_MODE
268 GPUError("WARNING, deterministicGPUReconstruction needs GPUCA_DETERMINISTIC_MODE for being fully deterministic, without only most indeterminism by concurrency is removed, but floating point effects remain!");
269#endif
270 mProcessingSettings->overrideClusterizerFragmentLen = TPC_MAX_FRAGMENT_LEN_GPU;
271 param().rec.tpc.nWaysOuter = true;
272 if (param().rec.tpc.looperInterpolationInExtraPass == -1) {
273 param().rec.tpc.looperInterpolationInExtraPass = 0;
274 }
275 if (GetProcessingSettings().createO2Output > 1) {
276 mProcessingSettings->createO2Output = 1;
277 }
278 mProcessingSettings->rtc.deterministic = 1;
279 } else {
280#ifdef GPUCA_DETERMINISTIC_MODE
281 GPUError("WARNING, compiled with GPUCA_DETERMINISTIC_MODE but deterministicGPUReconstruction not set, only compile-time determinism and deterministic math enforced, not fully deterministic!");
282#endif
283 }
284 if (GetProcessingSettings().deterministicGPUReconstruction && GetProcessingSettings().debugLevel >= 6) {
285 mProcessingSettings->nTPCClustererLanes = 1;
286 }
287 if (GetProcessingSettings().createO2Output > 1 && GetProcessingSettings().runQA && GetProcessingSettings().qcRunFraction == 100.f) {
288 mProcessingSettings->createO2Output = 1;
289 }
290 if (!GetProcessingSettings().createO2Output || !IsGPU()) {
291 mProcessingSettings->clearO2OutputFromGPU = false;
292 }
294 mProcessingSettings->mergerSortTracks = false;
295 }
296
297 if (GetProcessingSettings().debugLevel > 3 || !IsGPU() || GetProcessingSettings().deterministicGPUReconstruction) {
298 mProcessingSettings->delayedOutput = false;
299 }
300
301 if (!GetProcessingSettings().rtc.enable) {
302 mProcessingSettings->rtc.optConstexpr = false;
303 }
304
305 mMemoryScalers->factor = GetProcessingSettings().memoryScalingFactor;
306 mMemoryScalers->conservative = GetProcessingSettings().conservativeMemoryEstimate;
307 mMemoryScalers->returnMaxVal = GetProcessingSettings().forceMaxMemScalers != 0;
308 if (GetProcessingSettings().forceMaxMemScalers > 1) {
309 mMemoryScalers->rescaleMaxMem(GetProcessingSettings().forceMaxMemScalers);
310 }
311
312 if (GetProcessingSettings().nHostThreads != -1 && GetProcessingSettings().ompThreads != -1) {
313 GPUFatal("Must not use both nHostThreads and ompThreads at the same time!");
314 } else if (GetProcessingSettings().ompThreads != -1) {
315 mProcessingSettings->nHostThreads = GetProcessingSettings().ompThreads;
316 GPUWarning("You are using the deprecated ompThreads option, please switch to nHostThreads!");
317 }
318
319 if (GetProcessingSettings().nHostThreads <= 0) {
320 mProcessingSettings->nHostThreads = internal::getDefaultNThreads();
321 } else {
322 mProcessingSettings->autoAdjustHostThreads = false;
323 }
324 mMaxHostThreads = GetProcessingSettings().nHostThreads;
325 if (mMaster == nullptr) {
326 mThreading = std::make_shared<GPUReconstructionThreading>();
327 mThreading->control = std::make_unique<tbb::global_control>(tbb::global_control::max_allowed_parallelism, mMaxHostThreads);
328 mThreading->allThreads = std::make_unique<tbb::task_arena>(mMaxHostThreads);
329 mThreading->activeThreads = std::make_unique<tbb::task_arena>(mMaxHostThreads);
330 } else {
332 }
334 if (IsGPU()) {
335 mNStreams = std::max<int32_t>(GetProcessingSettings().nStreams, 3);
336 }
337
338 if (GetProcessingSettings().nTPCClustererLanes == -1) {
339 mProcessingSettings->nTPCClustererLanes = (GetRecoStepsGPU() & RecoStep::TPCClusterFinding) ? 3 : std::max<int32_t>(1, std::min<int32_t>(GPUCA_NSECTORS, GetProcessingSettings().inKernelParallel ? (mMaxHostThreads >= 4 ? std::min<int32_t>(mMaxHostThreads / 2, mMaxHostThreads >= 32 ? GPUCA_NSECTORS : 4) : 1) : mMaxHostThreads));
340 }
341 if (GetProcessingSettings().overrideClusterizerFragmentLen == -1) {
342 mProcessingSettings->overrideClusterizerFragmentLen = ((GetRecoStepsGPU() & RecoStep::TPCClusterFinding) || (mMaxHostThreads / GetProcessingSettings().nTPCClustererLanes >= 3)) ? TPC_MAX_FRAGMENT_LEN_GPU : TPC_MAX_FRAGMENT_LEN_HOST;
343 }
344 if (GetProcessingSettings().nTPCClustererLanes > GPUCA_NSECTORS) {
345 GPUError("Invalid value for nTPCClustererLanes: %d", GetProcessingSettings().nTPCClustererLanes);
346 mProcessingSettings->nTPCClustererLanes = GPUCA_NSECTORS;
347 }
348
349 if (GetProcessingSettings().doublePipeline && (mChains.size() != 1 || mChains[0]->SupportsDoublePipeline() == false || !IsGPU() || GetProcessingSettings().memoryAllocationStrategy != GPUMemoryResource::ALLOCATION_GLOBAL)) {
350 GPUError("Must use double pipeline mode only with exactly one chain that must support it");
351 return 1;
352 }
353
354 if (mMaster == nullptr && GetProcessingSettings().doublePipeline) {
356 }
357
359 for (uint32_t i = 0; i < mChains.size(); i++) {
360 if (mChains[i]->EarlyConfigure()) {
361 return 1;
362 }
363 mChains[i]->RegisterPermanentMemoryAndProcessors();
364 size_t memPrimary, memPageLocked;
365 mChains[i]->MemorySize(memPrimary, memPageLocked);
366 if (!IsGPU() || mOutputControl.useInternal()) {
367 memPageLocked = memPrimary;
368 }
369 mDeviceMemorySize += memPrimary;
370 mHostMemorySize += memPageLocked;
371 }
372 if (GetProcessingSettings().forceMemoryPoolSize && GetProcessingSettings().forceMemoryPoolSize <= 2 && CanQueryMaxMemory()) {
373 mDeviceMemorySize = GetProcessingSettings().forceMemoryPoolSize;
374 } else if (GetProcessingSettings().forceMemoryPoolSize > 2) {
375 mDeviceMemorySize = GetProcessingSettings().forceMemoryPoolSize;
376 if (!IsGPU() || mOutputControl.useInternal()) {
378 }
379 }
380 if (GetProcessingSettings().forceHostMemoryPoolSize) {
381 mHostMemorySize = GetProcessingSettings().forceHostMemoryPoolSize;
382 }
383
384 for (uint32_t i = 0; i < mProcessors.size(); i++) {
385 (mProcessors[i].proc->*(mProcessors[i].RegisterMemoryAllocation))();
386 }
387
388 return 0;
389}
390
392{
393 if (IsGPU()) {
394 for (uint32_t i = 0; i < mChains.size(); i++) {
395 mChains[i]->RegisterGPUProcessors();
396 }
397 }
399 return 0;
400}
401
403{
404 if (GetProcessingSettings().forceMaxMemScalers <= 1 && GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
406 }
407 for (uint32_t i = 0; i < mChains.size(); i++) {
408 if (mChains[i]->Init()) {
409 return 1;
410 }
411 }
412 for (uint32_t i = 0; i < mProcessors.size(); i++) {
413 (mProcessors[i].proc->*(mProcessors[i].InitializeProcessor))();
414 }
415
416 WriteConstantParams(); // Initialize with initial values, can optionally be updated later
417
418 mInitialized = true;
419 return 0;
420}
421
423{
424 if (IsGPU()) {
425 const auto threadContext = GetThreadContext();
426 WriteToConstantMemory(ptrDiff(&processors()->param, processors()), &param(), sizeof(param()), -1);
427 }
428}
429
431{
432 for (uint32_t i = 0; i < mChains.size(); i++) {
433 mChains[i]->Finalize();
434 }
435 return 0;
436}
437
439{
440 if (!mInitialized) {
441 return 1;
442 }
443 for (uint32_t i = 0; i < mSlaves.size(); i++) {
444 if (mSlaves[i]->Exit()) {
445 GPUError("Error exiting slave");
446 }
447 }
448
449 mChains.clear(); // Make sure we destroy a possible ITS GPU tracker before we call the destructors
450 mHostConstantMem.reset(); // Reset these explicitly before the destruction of other members unloads the library
451 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
452 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
453 if (mMemoryResources[i].mReuse >= 0) {
454 continue;
455 }
456 operator delete(mMemoryResources[i].mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
457 mMemoryResources[i].mPtr = mMemoryResources[i].mPtrDevice = nullptr;
458 }
459 }
460 mMemoryResources.clear();
461 if (mInitialized) {
462 ExitDevice();
463 }
464 mInitialized = false;
465 return 0;
466}
467
470
472{
473 for (auto it = mMemoryReuse1to1.begin(); it != mMemoryReuse1to1.end(); it++) {
474 auto& re = it->second;
475 if (proc == nullptr || re.proc == proc) {
476 GPUMemoryResource& resMain = mMemoryResources[re.res[0]];
477 resMain.mOverrideSize = 0;
478 for (uint32_t i = 0; i < re.res.size(); i++) {
480 resMain.mOverrideSize = std::max<size_t>(resMain.mOverrideSize, ptrDiff(res.SetPointers((void*)1), (char*)1));
481 }
482 }
483 }
484}
485
487{
489 if ((type & GPUMemoryResource::MEMORY_SCRATCH) && !GetProcessingSettings().keepDisplayMemory) { // keepAllMemory --> keepDisplayMemory
491 } else {
493 }
494 }
496 type &= ~GPUMemoryResource::MEMORY_GPU;
497 }
498 mMemoryResources.emplace_back(proc, setPtr, (GPUMemoryResource::MemoryType)type, name);
499 if (mMemoryResources.size() >= 32768) {
500 throw std::bad_alloc();
501 }
502 uint16_t retVal = mMemoryResources.size() - 1;
503 if (re.type != GPUMemoryReuse::NONE && !GetProcessingSettings().disableMemoryReuse) {
504 const auto& it = mMemoryReuse1to1.find(re.id);
505 if (it == mMemoryReuse1to1.end()) {
506 mMemoryReuse1to1[re.id] = {proc, retVal};
507 } else {
508 mMemoryResources[retVal].mReuse = it->second.res[0];
509 it->second.res.emplace_back(retVal);
510 }
511 }
512 return retVal;
513}
514
516{
517 if (GetProcessingSettings().debugLevel >= 5) {
518 GPUInfo("Allocating memory %p", (void*)proc);
519 }
520 size_t total = 0;
521 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
522 if (proc == nullptr ? !mMemoryResources[i].mProcessor->mAllocateAndInitializeLate : mMemoryResources[i].mProcessor == proc) {
524 total += AllocateRegisteredMemory(i);
525 } else if (resetCustom && (mMemoryResources[i].mPtr || mMemoryResources[i].mPtrDevice)) {
527 }
528 }
529 }
530 if (GetProcessingSettings().debugLevel >= 5) {
531 GPUInfo("Allocating memory done");
532 }
533 return total;
534}
535
537{
538 if (GetProcessingSettings().debugLevel >= 5) {
539 GPUInfo("Allocating Permanent Memory");
540 }
542 GPUError("Must not allocate permanent memory while volatile chunks are allocated");
543 throw std::bad_alloc();
544 }
545 int32_t total = 0;
546 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
547 if ((mMemoryResources[i].mType & GPUMemoryResource::MEMORY_PERMANENT) && mMemoryResources[i].mPtr == nullptr) {
548 total += AllocateRegisteredMemory(i);
549 }
550 }
553 if (GetProcessingSettings().debugLevel >= 5) {
554 GPUInfo("Permanent Memory Done");
555 }
556 return total;
557}
558
559size_t GPUReconstruction::AllocateRegisteredMemoryHelper(GPUMemoryResource* res, void*& ptr, void*& memorypool, void* memorybase, size_t memorysize, void* (GPUMemoryResource::*setPtr)(void*), void*& memorypoolend, const char* device)
560{
561 if (res->mReuse >= 0) {
562 ptr = (&ptr == &res->mPtrDevice) ? mMemoryResources[res->mReuse].mPtrDevice : mMemoryResources[res->mReuse].mPtr;
563 if (ptr == nullptr) {
564 GPUError("Invalid reuse ptr (%s)", res->mName);
565 throw std::bad_alloc();
566 }
567 size_t retVal = ptrDiff((res->*setPtr)(ptr), ptr);
568 if (retVal > mMemoryResources[res->mReuse].mSize) {
569 GPUError("Insufficient reuse memory %lu < %lu (%s) (%s)", mMemoryResources[res->mReuse].mSize, retVal, res->mName, device);
570 throw std::bad_alloc();
571 }
572 if (GetProcessingSettings().allocDebugLevel >= 2) {
573 std::cout << "Reused (" << device << ") " << res->mName << ": " << retVal << "\n";
574 }
575 return retVal;
576 }
577 if (memorypool == nullptr) {
578 GPUError("Cannot allocate memory from uninitialized pool");
579 throw std::bad_alloc();
580 }
581 size_t retVal;
582 if ((res->mType & GPUMemoryResource::MEMORY_STACK) && memorypoolend) {
583 retVal = ptrDiff((res->*setPtr)((char*)1), (char*)(1));
584 memorypoolend = (void*)((char*)memorypoolend - GPUProcessor::getAlignmentMod<GPUCA_MEMALIGN>(memorypoolend));
585 if (retVal < res->mOverrideSize) {
586 retVal = res->mOverrideSize;
587 }
588 retVal += GPUProcessor::getAlignment<GPUCA_MEMALIGN>(retVal);
589 memorypoolend = (char*)memorypoolend - retVal;
590 ptr = memorypoolend;
591 retVal = std::max<size_t>(ptrDiff((res->*setPtr)(ptr), ptr), res->mOverrideSize);
592 } else {
593 ptr = memorypool;
594 memorypool = (char*)((res->*setPtr)(ptr));
595 retVal = ptrDiff(memorypool, ptr);
596 if (retVal < res->mOverrideSize) {
597 retVal = res->mOverrideSize;
598 memorypool = (char*)ptr + res->mOverrideSize;
599 }
600 memorypool = (void*)((char*)memorypool + GPUProcessor::getAlignment<GPUCA_MEMALIGN>(memorypool));
601 }
602 if (memorypoolend ? (memorypool > memorypoolend) : ((size_t)ptrDiff(memorypool, memorybase) > memorysize)) {
603 std::cerr << "Memory pool size exceeded (" << device << ") (" << res->mName << ": " << (memorypoolend ? (memorysize + ptrDiff(memorypool, memorypoolend)) : ptrDiff(memorypool, memorybase)) << " > " << memorysize << "\n";
604 throw std::bad_alloc();
605 }
606 if (GetProcessingSettings().allocDebugLevel >= 2) {
607 std::cout << "Allocated (" << device << ") " << res->mName << ": " << retVal << " - available: " << (memorypoolend ? ptrDiff(memorypoolend, memorypool) : (memorysize - ptrDiff(memorypool, memorybase))) << "\n";
608 }
609 return retVal;
610}
611
613{
614 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL && (control == nullptr || control->useInternal())) {
615 if (!(res->mType & GPUMemoryResource::MEMORY_EXTERNAL)) {
616 if (res->mPtrDevice && res->mReuse < 0) {
617 operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
618 }
619 res->mSize = std::max((size_t)res->SetPointers((void*)1) - 1, res->mOverrideSize);
620 if (res->mReuse >= 0) {
621 if (res->mSize > mMemoryResources[res->mReuse].mSize) {
622 GPUError("Invalid reuse, insufficient size: %ld < %ld", (int64_t)mMemoryResources[res->mReuse].mSize, (int64_t)res->mSize);
623 throw std::bad_alloc();
624 }
625 res->mPtrDevice = mMemoryResources[res->mReuse].mPtrDevice;
626 } else {
627 res->mPtrDevice = operator new(res->mSize + GPUCA_BUFFER_ALIGNMENT, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
628 }
629 res->mPtr = GPUProcessor::alignPointer<GPUCA_BUFFER_ALIGNMENT>(res->mPtrDevice);
630 res->SetPointers(res->mPtr);
631 if (GetProcessingSettings().allocDebugLevel >= 2) {
632 std::cout << (res->mReuse >= 0 ? "Reused " : "Allocated ") << res->mName << ": " << res->mSize << "\n";
633 }
636 }
637 if ((size_t)res->mPtr % GPUCA_BUFFER_ALIGNMENT) {
638 GPUError("Got buffer with insufficient alignment");
639 throw std::bad_alloc();
640 }
641 }
642 } else {
643 if (res->mPtr != nullptr) {
644 GPUError("Double allocation! (%s)", res->mName);
645 throw std::bad_alloc();
646 }
647 if (IsGPU() && res->mOverrideSize < GPUCA_BUFFER_ALIGNMENT) {
648 res->mOverrideSize = GPUCA_BUFFER_ALIGNMENT;
649 }
650 if ((!IsGPU() || (res->mType & GPUMemoryResource::MEMORY_HOST) || GetProcessingSettings().keepDisplayMemory) && !(res->mType & GPUMemoryResource::MEMORY_EXTERNAL)) { // keepAllMemory --> keepDisplayMemory
651 if (control && control->useExternal()) {
652 if (control->allocator) {
653 res->mSize = std::max((size_t)res->SetPointers((void*)1) - 1, res->mOverrideSize);
654 res->mPtr = control->allocator(CAMath::nextMultipleOf<GPUCA_BUFFER_ALIGNMENT>(res->mSize));
655 res->mSize = std::max<size_t>(ptrDiff(res->SetPointers(res->mPtr), res->mPtr), res->mOverrideSize);
656 if (GetProcessingSettings().allocDebugLevel >= 2) {
657 std::cout << "Allocated (from callback) " << res->mName << ": " << res->mSize << "\n";
658 }
659 } else {
660 void* dummy = nullptr;
661 res->mSize = AllocateRegisteredMemoryHelper(res, res->mPtr, control->ptrCurrent, control->ptrBase, control->size, &GPUMemoryResource::SetPointers, dummy, "host");
662 }
663 } else {
665 }
666 if ((size_t)res->mPtr % GPUCA_BUFFER_ALIGNMENT) {
667 GPUError("Got buffer with insufficient alignment");
668 throw std::bad_alloc();
669 }
670 }
671 if (IsGPU() && (res->mType & GPUMemoryResource::MEMORY_GPU)) {
672 if (res->mProcessor->mLinkedProcessor == nullptr) {
673 GPUError("Device Processor not set (%s)", res->mName);
674 throw std::bad_alloc();
675 }
677 GPUError("Must not allocate non-stacked device memory while volatile chunks are allocated");
678 throw std::bad_alloc();
679 }
681
683 res->mSize = size;
684 } else if (size != res->mSize) {
685 GPUError("Inconsistent device memory allocation (%s: device %lu vs %lu)", res->mName, size, res->mSize);
686 throw std::bad_alloc();
687 }
688 if ((size_t)res->mPtrDevice % GPUCA_BUFFER_ALIGNMENT) {
689 GPUError("Got buffer with insufficient alignment");
690 throw std::bad_alloc();
691 }
692 }
694 }
695}
696
701
703{
705 if ((res->mType & GPUMemoryResource::MEMORY_PERMANENT) && res->mPtr != nullptr) {
707 } else {
709 }
710 return res->mReuse >= 0 ? 0 : res->mSize;
711}
712
714{
715 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL) {
716 char* retVal = new (std::align_val_t(GPUCA_BUFFER_ALIGNMENT)) char[size];
719 } else {
721 }
722 return retVal;
723 }
724
726 throw std::runtime_error("Requested invalid memory typo for direct allocation");
727 }
729 GPUError("Must not allocate direct memory while volatile chunks are allocated");
730 throw std::bad_alloc();
731 }
732
735 char* retVal;
737 poolend = (char*)poolend - size;
738 poolend = (char*)poolend - GPUProcessor::getAlignmentMod<GPUCA_MEMALIGN>(poolend);
739 retVal = (char*)poolend;
740 } else {
742 }
743 if (pool > poolend) {
744 GPUError("Insufficient unmanaged memory: missing %ld bytes", ptrDiff(pool, poolend));
745 throw std::bad_alloc();
746 }
748 if (GetProcessingSettings().allocDebugLevel >= 2) {
749 std::cout << "Allocated (unmanaged " << (type == GPUMemoryResource::MEMORY_GPU ? "gpu" : "host") << "): " << size << " - available: " << ptrDiff(poolend, pool) << "\n";
750 }
751 return retVal;
752}
753
755{
756 if (mVolatileMemoryStart == nullptr) {
758 }
759 if (size == 0) {
760 return nullptr; // Future GPU memory allocation is volatile
761 }
762 char* retVal;
765 GPUError("Insufficient volatile device memory: missing %ld", ptrDiff(mDeviceMemoryPool, mDeviceMemoryPoolEnd));
766 throw std::bad_alloc();
767 }
769 if (GetProcessingSettings().allocDebugLevel >= 2) {
770 std::cout << "Allocated (volatile GPU): " << size << " - available: " << ptrDiff(mDeviceMemoryPoolEnd, mDeviceMemoryPool) << "\n";
771 }
772 return retVal;
773}
774
776{
777 if (device) {
779 }
780 char* retVal = new (std::align_val_t(GPUCA_BUFFER_ALIGNMENT)) char[size];
781 mVolatileChunks.emplace_back(retVal, alignedDeleter());
782 return retVal;
783}
784
790
792{
796 mVolatileMemoryStart = nullptr;
797 }
798 if (GetProcessingSettings().allocDebugLevel >= 2) {
799 std::cout << "Freed (volatile GPU) - available: " << ptrDiff(mDeviceMemoryPoolEnd, mDeviceMemoryPool) << "\n";
800 }
801}
802
808
810{
811 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
812 if (proc == nullptr || mMemoryResources[i].mProcessor == proc) {
814 }
815 }
816}
817
819{
822 void* basePtr = res->mReuse >= 0 ? mMemoryResources[res->mReuse].mPtr : res->mPtr;
823 size_t size = ptrDiff(res->SetPointers(basePtr), basePtr);
824 if (basePtr && size > std::max(res->mSize, res->mOverrideSize)) {
825 std::cerr << "Updated pointers exceed available memory size: " << size << " > " << std::max(res->mSize, res->mOverrideSize) << " - host - " << res->mName << "\n";
826 throw std::bad_alloc();
827 }
828 }
829 if (IsGPU() && (res->mType & GPUMemoryResource::MEMORY_GPU)) {
830 void* basePtr = res->mReuse >= 0 ? mMemoryResources[res->mReuse].mPtrDevice : res->mPtrDevice;
831 size_t size = ptrDiff(res->SetDevicePointers(basePtr), basePtr);
832 if (basePtr && size > std::max(res->mSize, res->mOverrideSize)) {
833 std::cerr << "Updated pointers exceed available memory size: " << size << " > " << std::max(res->mSize, res->mOverrideSize) << " - GPU - " << res->mName << "\n";
834 throw std::bad_alloc();
835 }
836 }
837}
838
839void GPUReconstruction::FreeRegisteredMemory(GPUProcessor* proc, bool freeCustom, bool freePermanent)
840{
841 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
842 if ((proc == nullptr || mMemoryResources[i].mProcessor == proc) && (freeCustom || !(mMemoryResources[i].mType & GPUMemoryResource::MEMORY_CUSTOM)) && (freePermanent || !(mMemoryResources[i].mType & GPUMemoryResource::MEMORY_PERMANENT))) {
844 }
845 }
846}
847
852
854{
855 if (GetProcessingSettings().allocDebugLevel >= 2 && (res->mPtr || res->mPtrDevice)) {
856 std::cout << "Freeing " << res->mName << ": size " << res->mSize << " (reused " << res->mReuse << ")\n";
857 }
858 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_INDIVIDUAL && res->mReuse < 0) {
859 operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
860 }
861 res->mPtr = nullptr;
862 res->mPtrDevice = nullptr;
863}
864
869
871{
872 if (GetProcessingSettings().keepDisplayMemory || GetProcessingSettings().disableMemoryReuse) {
873 return;
874 }
875 if (mNonPersistentMemoryStack.size() == 0) {
876 GPUFatal("Trying to pop memory state from empty stack");
877 }
878 if (tag != 0 && std::get<4>(mNonPersistentMemoryStack.back()) != tag) {
879 GPUFatal("Tag mismatch when popping non persistent memory from stack : pop %s vs on stack %s", qTag2Str(tag).c_str(), qTag2Str(std::get<4>(mNonPersistentMemoryStack.back())).c_str());
880 }
881 if ((GetProcessingSettings().debugLevel >= 3 || GetProcessingSettings().allocDebugLevel) && (IsGPU() || GetProcessingSettings().forceHostMemoryPoolSize)) {
882 printf("Allocated memory after %30s (%8s) (Stack %zu): ", GPUDataTypes::RECO_STEP_NAMES[getRecoStepNum(step, true)], qTag2Str(std::get<4>(mNonPersistentMemoryStack.back())).c_str(), mNonPersistentMemoryStack.size());
884 printf("%76s", "");
886 }
887 mHostMemoryPoolEnd = std::get<0>(mNonPersistentMemoryStack.back());
889 for (uint32_t i = std::get<2>(mNonPersistentMemoryStack.back()); i < mNonPersistentIndividualAllocations.size(); i++) {
891 if (res->mReuse < 0) {
892 operator delete(res->mPtrDevice, std::align_val_t(GPUCA_BUFFER_ALIGNMENT));
893 }
894 res->mPtr = nullptr;
895 res->mPtrDevice = nullptr;
896 }
899 mNonPersistentMemoryStack.pop_back();
900}
901
903{
905 throw std::runtime_error("temporary memory stack already blocked");
906 }
909}
910
912{
913 if (mNonPersistentMemoryStack.size()) {
914 throw std::runtime_error("cannot unblock while there is stacked memory");
915 }
918 mHostMemoryPoolBlocked = nullptr;
919 mDeviceMemoryPoolBlocked = nullptr;
920}
921
923{
924 mMemoryResources[res].mPtr = ptr;
925}
926
928{
929 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
932 }
933 }
936 mDirectMemoryChunks.clear();
938 mVolatileChunks.clear();
939 mVolatileMemoryStart = nullptr;
940 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
941 mHostMemoryPool = GPUProcessor::alignPointer<GPUCA_MEMALIGN>(mHostMemoryPermanent);
942 mDeviceMemoryPool = GPUProcessor::alignPointer<GPUCA_MEMALIGN>(mDeviceMemoryPermanent);
945 } else {
947 }
948}
949
955
957{
958 printf("Maximum Memory Allocation: Host %'zu / Device %'zu\n", mHostMemoryUsedMax, mDeviceMemoryUsedMax);
959}
960
962{
963 if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
964 printf("Memory Allocation: Host %'13zd / %'13zu (Permanent %'13zd, Data %'13zd, Scratch %'13zd), Device %'13zd / %'13zu, (Permanent %'13zd, Data %'13zd, Scratch %'13zd) %zu chunks\n",
967 mMemoryResources.size());
968 }
969}
970
972{
973 std::map<std::string, std::array<size_t, 3>> sizes;
974 for (uint32_t i = 0; i < mMemoryResources.size(); i++) {
975 auto& res = mMemoryResources[i];
976 if (res.mReuse >= 0) {
977 continue;
978 }
979 auto& x = sizes[res.mName];
980 if (res.mPtr) {
981 x[0] += res.mSize;
982 }
983 if (res.mPtrDevice) {
984 x[1] += res.mSize;
985 }
987 x[2] = 1;
988 }
989 }
990 printf("%59s CPU / %9s GPU\n", "", "");
991 for (auto it = sizes.begin(); it != sizes.end(); it++) {
992 printf("Allocation %30s %s: Size %'14zu / %'14zu\n", it->first.c_str(), it->second[2] ? "P" : " ", it->second[0], it->second[1]);
993 }
995 for (uint32_t i = 0; i < mChains.size(); i++) {
996 mChains[i]->PrintMemoryStatistics();
997 }
998}
999
1001{
1002 if (GetProcessingSettings().noGPUMemoryRegistration) {
1003 return 0;
1004 }
1006 if (retVal == 0) {
1007 mRegisteredMemoryPtrs.emplace(ptr);
1008 }
1009 return retVal;
1010}
1011
1013{
1014 if (GetProcessingSettings().noGPUMemoryRegistration) {
1015 return 0;
1016 }
1017 const auto& pos = mRegisteredMemoryPtrs.find(ptr);
1018 if (pos != mRegisteredMemoryPtrs.end()) {
1021 }
1022 return 1;
1023}
1024
1025namespace o2::gpu::internal
1026{
1027namespace // anonymous
1028{
1029template <class T>
1030constexpr static inline int32_t getStepNum(T step, bool validCheck, int32_t N, const char* err = "Invalid step num")
1031{
1032 static_assert(sizeof(step) == sizeof(uint32_t), "Invalid step enum size");
1033 int32_t retVal = 8 * sizeof(uint32_t) - 1 - CAMath::Clz((uint32_t)step);
1034 if ((uint32_t)step == 0 || retVal >= N) {
1035 if (!validCheck) {
1036 return -1;
1037 }
1038 throw std::runtime_error("Invalid General Step");
1039 }
1040 return retVal;
1041}
1042} // anonymous namespace
1043} // namespace o2::gpu::internal
1044
1045int32_t GPUReconstruction::getRecoStepNum(RecoStep step, bool validCheck) { return internal::getStepNum(step, validCheck, GPUDataTypes::N_RECO_STEPS, "Invalid Reco Step"); }
1046int32_t GPUReconstruction::getGeneralStepNum(GeneralStep step, bool validCheck) { return internal::getStepNum(step, validCheck, GPUDataTypes::N_GENERAL_STEPS, "Invalid General Step"); }
1047
1049{
1050 if (!mInitialized || !GetProcessingSettings().doublePipeline || mMaster != nullptr || !mSlaves.size()) {
1051 throw std::invalid_argument("Cannot start double pipeline mode");
1052 }
1053 if (GetProcessingSettings().debugLevel >= 3) {
1054 GPUInfo("Pipeline worker started");
1055 }
1056 bool terminate = false;
1057 while (!terminate) {
1058 {
1059 std::unique_lock<std::mutex> lk(mPipelineContext->mutex);
1060 mPipelineContext->cond.wait(lk, [this] { return this->mPipelineContext->queue.size() > 0; });
1061 }
1062 GPUReconstructionPipelineQueue* q;
1063 {
1064 std::lock_guard<std::mutex> lk(mPipelineContext->mutex);
1065 q = mPipelineContext->queue.front();
1066 mPipelineContext->queue.pop();
1067 }
1068 if (q->op == 1) {
1069 terminate = 1;
1070 } else {
1071 q->retVal = q->chain->RunChain();
1072 }
1073 {
1074 std::lock_guard<std::mutex> lk(q->m);
1075 q->done = true;
1076 }
1077 q->c.notify_one();
1078 }
1079 if (GetProcessingSettings().debugLevel >= 3) {
1080 GPUInfo("Pipeline worker ended");
1081 }
1082}
1083
1088
1090{
1093 std::unique_ptr<GPUReconstructionPipelineQueue> qu(new GPUReconstructionPipelineQueue);
1094 GPUReconstructionPipelineQueue* q = qu.get();
1095 q->chain = terminate ? nullptr : mChains[0].get();
1096 q->op = terminate ? 1 : 0;
1097 std::unique_lock<std::mutex> lkdone(q->m);
1098 {
1099 std::lock_guard<std::mutex> lkpipe(rec->mPipelineContext->mutex);
1100 if (rec->mPipelineContext->terminate) {
1101 throw std::runtime_error("Must not enqueue work after termination request");
1102 }
1103 rec->mPipelineContext->queue.push(q);
1104 rec->mPipelineContext->terminate = terminate;
1105 rec->mPipelineContext->cond.notify_one();
1106 }
1107 q->c.wait(lkdone, [&q]() { return q->done; });
1108 if (q->retVal) {
1109 return q->retVal;
1110 }
1111 if (terminate) {
1112 return 0;
1113 } else {
1114 return mChains[0]->FinalizePipelinedProcessing();
1115 }
1116}
1117
1119{
1121 std::lock_guard<std::mutex> lk(rec->mPipelineContext->mutex);
1122 return rec->mPipelineContext->queue.size() && rec->mPipelineContext->queue.front()->op == 0 ? rec->mPipelineContext->queue.front()->chain : nullptr;
1123}
1124
1125void GPUReconstruction::PrepareEvent() // TODO: Clean this up, this should not be called from chainTracking but before
1126{
1128 for (uint32_t i = 0; i < mChains.size(); i++) {
1129 mChains[i]->PrepareEvent();
1130 }
1131 for (uint32_t i = 0; i < mProcessors.size(); i++) {
1132 if (mProcessors[i].proc->mAllocateAndInitializeLate) {
1133 continue;
1134 }
1135 (mProcessors[i].proc->*(mProcessors[i].SetMaxData))(mHostConstantMem->ioPtrs);
1136 if (mProcessors[i].proc->mGPUProcessorType != GPUProcessor::PROCESSOR_TYPE_DEVICE && mProcessors[i].proc->mLinkedProcessor) {
1137 (mProcessors[i].proc->mLinkedProcessor->*(mProcessors[i].SetMaxData))(mHostConstantMem->ioPtrs);
1138 }
1139 }
1140 ComputeReuseMax(nullptr);
1141 AllocateRegisteredMemory(nullptr);
1142}
1143
1144int32_t GPUReconstruction::CheckErrorCodes(bool cpuOnly, bool forceShowErrors, std::vector<std::array<uint32_t, 4>>* fillErrors)
1145{
1146 int32_t retVal = 0;
1147 for (uint32_t i = 0; i < mChains.size(); i++) {
1148 if (mChains[i]->CheckErrorCodes(cpuOnly, forceShowErrors, fillErrors)) {
1149 retVal++;
1150 }
1151 }
1152 return retVal;
1153}
1154
1155int32_t GPUReconstruction::GPUChkErrA(const int64_t error, const char* file, int32_t line, bool failOnError)
1156{
1157 if (error == 0 || !GPUChkErrInternal(error, file, line)) {
1158 return 0;
1159 }
1160 if (failOnError) {
1161 if (mInitialized && mInErrorHandling == false) {
1162 mInErrorHandling = true;
1163 CheckErrorCodes(false, true);
1164 }
1165 throw std::runtime_error("GPU Backend Failure");
1166 }
1167 return 1;
1168}
1169
1171{
1172 std::string f;
1173 f = dir;
1174 f += "settings.dump";
1175 DumpStructToFile(mGRPSettings.get(), f.c_str());
1176 for (uint32_t i = 0; i < mChains.size(); i++) {
1177 mChains[i]->DumpSettings(dir);
1178 }
1179}
1180
1181void GPUReconstruction::UpdateDynamicSettings(const GPUSettingsRecDynamic* d)
1182{
1183 UpdateSettings(nullptr, nullptr, d);
1184}
1185
1186void GPUReconstruction::UpdateSettings(const GPUSettingsGRP* g, const GPUSettingsProcessing* p, const GPUSettingsRecDynamic* d)
1187{
1188 if (g) {
1189 *mGRPSettings = *g;
1190 }
1191 if (p) {
1192 mProcessingSettings->debugLevel = p->debugLevel;
1193 mProcessingSettings->resetTimers = p->resetTimers;
1194 }
1195 GPURecoStepConfiguration* w = nullptr;
1197 w = &mRecoSteps;
1198 }
1199 param().UpdateSettings(g, p, w, d);
1200 if (mInitialized) {
1202 }
1203}
1204
1205int32_t GPUReconstruction::ReadSettings(const char* dir)
1206{
1207 std::string f;
1208 f = dir;
1209 f += "settings.dump";
1210 new (mGRPSettings.get()) GPUSettingsGRP;
1211 if (ReadStructFromFile(f.c_str(), mGRPSettings.get())) {
1212 return 1;
1213 }
1215 for (uint32_t i = 0; i < mChains.size(); i++) {
1216 mChains[i]->ReadSettings(dir);
1217 }
1218 return 0;
1219}
1220
1221void GPUReconstruction::SetSettings(float solenoidBzNominalGPU, const GPURecoStepConfiguration* workflow)
1222{
1223#ifdef GPUCA_O2_LIB
1225 config.ReadConfigurableParam(config);
1226 config.configGRP.solenoidBzNominalGPU = solenoidBzNominalGPU;
1227 SetSettings(&config.configGRP, &config.configReconstruction, &config.configProcessing, workflow);
1228#else
1229 GPUSettingsGRP grp;
1230 grp.solenoidBzNominalGPU = solenoidBzNominalGPU;
1231 SetSettings(&grp, nullptr, nullptr, workflow);
1232#endif
1233}
1234
1235void GPUReconstruction::SetSettings(const GPUSettingsGRP* grp, const GPUSettingsRec* rec, const GPUSettingsProcessing* proc, const GPURecoStepConfiguration* workflow)
1236{
1237 if (mInitialized) {
1238 GPUError("Cannot update settings while initialized");
1239 throw std::runtime_error("Settings updated while initialized");
1240 }
1241 *mGRPSettings = *grp;
1242 if (proc) {
1243 *mProcessingSettings = *proc;
1244 }
1245 if (workflow) {
1246 mRecoSteps.steps = workflow->steps;
1248 mRecoSteps.inputs = workflow->inputs;
1249 mRecoSteps.outputs = workflow->outputs;
1250 }
1251 param().SetDefaults(mGRPSettings.get(), rec, proc, workflow);
1252}
1253
1255{
1256 GPUOutputControl outputControl;
1257 outputControl.set(ptr, size);
1258 SetOutputControl(outputControl);
1259}
1260
1264void GPUReconstruction::SetResetTimers(bool reset) { mProcessingSettings->resetTimers = reset; }
1269
1270ThrustVolatileAllocator::ThrustVolatileAllocator(GPUReconstruction* r)
1271{
1272 mAlloc = [&r](size_t n) { return (char*)r->AllocateVolatileDeviceMemory(n); };
1273}
int32_t i
#define GPUCA_BUFFER_ALIGNMENT
uint32_t op
bool done
int32_t retVal
GPUChain * chain
#define GPUCA_NSECTORS
uint16_t pos
Definition RawData.h:3
uint32_t res
Definition RawData.h:0
TBranch * ptr
void set(S v)
Definition bitfield.h:59
bool isSet(const bitfield &v) const
Definition bitfield.h:70
const GPUSettingsDisplay * GetEventDisplayConfig() const
const GPUSettingsQA * GetQAConfig() const
static void dumpConfig(const GPUSettingsRec *rec, const GPUSettingsProcessing *proc, const GPUSettingsQA *qa, const GPUSettingsDisplay *display, const GPUSettingsDeviceBackend *device, const GPURecoStepConfiguration *workflow)
static constexpr const char *const RECO_STEP_NAMES[]
static constexpr int32_t N_RECO_STEPS
static constexpr int32_t N_GENERAL_STEPS
void * SetDevicePointers(void *ptr)
static void computePointerWithAlignment(T *&basePtr, S *&objPtr, size_t nEntries=1)
void InitGPUProcessor(GPUReconstruction *rec, ProcessorType type=PROCESSOR_TYPE_CPU, GPUProcessor *slaveProcessor=nullptr)
ProcessorType mGPUProcessorType
GPURecoStepConfiguration mRecoSteps
std::unordered_set< const void * > mRegisteredMemoryPtrs
int16_t RegisterMemoryAllocationHelper(GPUProcessor *proc, void *(GPUProcessor::*setPtr)(void *), int32_t type, const char *name, const GPUMemoryReuse &re)
std::vector< std::unique_ptr< GPUChain > > mChains
void * AllocateVolatileMemory(size_t size, bool device)
ThrustVolatileAllocator getThrustVolatileDeviceAllocator()
std::unique_ptr< GPUMemorySizeScalers > mMemoryScalers
void AllocateRegisteredForeignMemory(int16_t res, GPUReconstruction *rec, GPUOutputControl *control=nullptr)
void SetInputControl(void *ptr, size_t size)
GPUConstantMem * mDeviceConstantMem
void ConstructGPUProcessor(GPUProcessor *proc)
std::shared_ptr< GPUROOTDumpCore > mROOTDump
void PopNonPersistentMemory(RecoStep step, uint64_t tag)
size_t AllocateRegisteredMemoryHelper(GPUMemoryResource *res, void *&ptr, void *&memorypool, void *memorybase, size_t memorysize, void *(GPUMemoryResource::*SetPointers)(void *), void *&memorypoolend, const char *device)
const GPUSettingsDeviceBackend & GetDeviceBackendSettings() const
void ComputeReuseMax(GPUProcessor *proc)
void SetMemoryExternalInput(int16_t res, void *ptr)
int32_t getGeneralStepNum(GeneralStep step, bool validCheck=true)
static constexpr uint32_t NSECTORS
RecoStepField GetRecoStepsGPU() const
void RegisterGPUDeviceProcessor(GPUProcessor *proc, GPUProcessor *slaveProcessor)
std::vector< GPUReconstruction * > mSlaves
std::vector< std::tuple< void *, void *, size_t, size_t, uint64_t > > mNonPersistentMemoryStack
std::unique_ptr< T > ReadStructFromFile(const char *file)
virtual void GetITSTraits(std::unique_ptr< o2::its::TrackerTraits > *trackerTraits, std::unique_ptr< o2::its::VertexerTraits > *vertexerTraits, std::unique_ptr< o2::its::TimeFrame > *timeFrame)
void UpdateDynamicSettings(const GPUSettingsRecDynamic *d)
std::unique_ptr< GPUSettingsDeviceBackend > mDeviceBackendSettings
std::vector< GPUMemoryResource > mMemoryResources
std::unique_ptr< GPUReconstructionPipelineContext > mPipelineContext
std::unique_ptr< GPUConstantMem > mHostConstantMem
void ResetRegisteredMemoryPointers(GPUProcessor *proc)
void DumpStructToFile(const T *obj, const char *file)
void AllocateRegisteredMemoryInternal(GPUMemoryResource *res, GPUOutputControl *control, GPUReconstruction *recPool)
virtual int32_t registerMemoryForGPU_internal(const void *ptr, size_t size)=0
virtual size_t WriteToConstantMemory(size_t offset, const void *src, size_t size, int32_t stream=-1, gpu_reconstruction_kernels::deviceEvent *ev=nullptr)=0
std::unordered_map< GPUMemoryReuse::ID, MemoryReuseMeta > mMemoryReuse1to1
std::vector< ProcessorData > mProcessors
void * AllocateVolatileDeviceMemory(size_t size)
virtual int32_t InitDevice()=0
void SetSettings(float solenoidBzNominalGPU, const GPURecoStepConfiguration *workflow=nullptr)
const GPUCalibObjectsConst & GetCalib() const
const GPUTrackingInOutPointers GetIOPtrs() const
virtual std::unique_ptr< gpu_reconstruction_kernels::threadContext > GetThreadContext()=0
GPUReconstruction(const GPUReconstruction &)=delete
static constexpr GeometryType geometryType
std::vector< std::unique_ptr< char[], alignedDeleter > > mNonPersistentIndividualDirectAllocations
void FreeRegisteredMemory(GPUProcessor *proc, bool freeCustom=false, bool freePermanent=false)
std::vector< std::unique_ptr< char[], alignedDeleter > > mVolatileChunks
virtual RecoStepField AvailableGPURecoSteps()
static constexpr const char *const IOTYPENAMES[]
std::vector< std::unique_ptr< char[], alignedDeleter > > mDirectMemoryChunks
void UpdateSettings(const GPUSettingsGRP *g, const GPUSettingsProcessing *p=nullptr, const GPUSettingsRecDynamic *d=nullptr)
int32_t CheckErrorCodes(bool cpuOnly=false, bool forceShowErrors=false, std::vector< std::array< uint32_t, 4 > > *fillErrors=nullptr)
const GPUParam & GetParam() const
void ClearAllocatedMemory(bool clearOutputs=true)
static constexpr const char *const GEOMETRY_TYPE_NAMES[]
virtual int32_t ExitDevice()=0
std::unique_ptr< GPUSettingsGRP > mGRPSettings
std::unique_ptr< GPUSettingsProcessing > mProcessingSettings
void PushNonPersistentMemory(uint64_t tag)
int32_t getRecoStepNum(RecoStep step, bool validCheck=true)
virtual int32_t unregisterMemoryForGPU_internal(const void *ptr)=0
void BlockStackedMemory(GPUReconstruction *rec)
const GPUSettingsProcessing & GetProcessingSettings() const
void DumpSettings(const char *dir="")
void * AllocateDirectMemory(size_t size, int32_t type)
int32_t unregisterMemoryForGPU(const void *ptr)
int32_t registerMemoryForGPU(const void *ptr, size_t size)
void SetDebugLevelTmp(int32_t level)
int32_t EnqueuePipeline(bool terminate=false)
std::shared_ptr< GPUReconstructionThreading > mThreading
std::vector< GPUMemoryResource * > mNonPersistentIndividualAllocations
virtual int32_t GPUChkErrInternal(const int64_t error, const char *file, int32_t line) const
int32_t GPUChkErrA(const int64_t error, const char *file, int32_t line, bool failOnError)
size_t AllocateRegisteredMemory(GPUProcessor *proc, bool resetCustom=false)
int32_t ReadSettings(const char *dir="")
void SetOutputControl(const GPUOutputControl &v)
void SetSector(int32_t iSector)
#define TPC_MAX_FRAGMENT_LEN_GPU
#define TPC_MAX_FRAGMENT_LEN_HOST
GLdouble n
Definition glcorearb.h:1982
GLint GLenum GLint x
Definition glcorearb.h:403
const GLfloat * m
Definition glcorearb.h:4066
GLsizeiptr size
Definition glcorearb.h:659
GLuint GLsizei const GLuint const GLintptr const GLsizeiptr * sizes
Definition glcorearb.h:2595
GLuint const GLchar * name
Definition glcorearb.h:781
GLdouble f
Definition glcorearb.h:310
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLint GLint GLsizei GLint GLenum GLenum type
Definition glcorearb.h:275
typedef void(APIENTRYP PFNGLCULLFACEPROC)(GLenum mode)
GLboolean GLboolean g
Definition glcorearb.h:1233
GLint level
Definition glcorearb.h:275
GLboolean r
Definition glcorearb.h:1233
GLenum GLfloat param
Definition glcorearb.h:271
GLboolean GLboolean GLboolean GLboolean a
Definition glcorearb.h:1233
GLubyte GLubyte GLubyte GLubyte w
Definition glcorearb.h:852
std::unique_ptr< GPUDisplayFrontendInterface > eventDisplay
GPUReconstruction * rec
std::string qTag2Str(const T tag)
Definition strtag.h:35
GPUTPCTracker tpcTrackers[GPUCA_NSECTORS]
GPUTPCClusterFinder tpcClusterer[GPUCA_NSECTORS]
GPUCalibObjectsConst calibObjects
void set(void *p, size_t s)
std::function< void *(size_t)> allocator
void SetDefaults(float solenoidBz)
Definition GPUParam.cxx:33
void UpdateSettings(const GPUSettingsGRP *g, const GPUSettingsProcessing *p=nullptr, const GPURecoStepConfiguration *w=nullptr, const GPUSettingsRecDynamic *d=nullptr)
Definition GPUParam.cxx:121
GPUDataTypes::RecoStepField stepsGPUMask
GPUDataTypes::InOutTypeField outputs
GPUDataTypes::RecoStepField steps
GPUDataTypes::InOutTypeField inputs
std::queue< GPUReconstructionPipelineQueue * > queue